multi: add new information extraction method

Adds `curl_off_t curl_multi_get_offt(CURLM *multi_handle, CURLMinfo_offt
info)` to the multi interface with enums:

* CURLMINFO_XFERS_CURRENT: current number of transfers
* CURLMINFO_XFERS_RUNNING: number of running transfers
* CURLMINFO_XFERS_PENDING: number of pending transfers
* CURLMINFO_XFERS_DONE: number of finished transfers to read
* CURLMINFO_XFERS_ADDED: total number of transfers added, ever

Add documentation for functions and info enums.

Add use in the curl command line tool to replace two static
variables counting the same "from the outside".

refs #17870
Closes #17992

.github/scripts/spellcheck.curl

@@ -132,6 +132,7 @@ curl_multi_timeout
 curl_multi_setopt
 curl_multi_assign
 curl_multi_get_handles
+curl_multi_get_offt
 curl_pushheader_bynum
 curl_pushheader_byname
 curl_multi_waitfds

docs/libcurl/Makefile.inc

@@ -75,6 +75,7 @@ man_MANS = \
  curl_multi_cleanup.3 \
  curl_multi_fdset.3 \
  curl_multi_get_handles.3 \
+ curl_multi_get_offt.3 \
  curl_multi_info_read.3 \
  curl_multi_init.3 \
  curl_multi_perform.3 \

docs/libcurl/curl_multi_get_offt.md

@@ -0,0 +1,102 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Title: curl_multi_get_offt
+Section: 3
+Source: libcurl
+See-also:
+  - curl_multi_add_handle (3)
+  - curl_multi_remove_handle (3)
+Protocol:
+  - All
+Added-in: 8.16.0
+---
+
+# NAME
+
+curl_multi_get_offt - extract information from a multi handle
+
+# SYNOPSIS
+
+~~~c
+#include <curl/curl.h>
+
+CURLMcode curl_multi_get_offt(CURLM *multi_handle,
+                              CURLMinfo_offt info,
+                              curl_off_t *pvalue);
+~~~
+
+# DESCRIPTION
+
+Get the *info* kept in the *multi* handle for `CURLMI_OFFT_*`.
+If the *info* is not applicable, this function returns CURLM_UNKNOWN_OPTION.
+
+# OPTIONS
+
+The following information can be extracted:
+
+## CURLMINFO_XFERS_CURRENT
+
+The number of easy handles currently added to the multi. This does not
+count handles removed. It does count internal handles that get
+added for tasks (like resolving via DoH, for example).
+
+For the total number of easy handles ever added to the multi, see
+*CURLMINFO_XFERS_ADDED*.
+
+## CURLMINFO_XFERS_RUNNING
+
+The number of easy handles currently running, e.g. where the transfer
+has started but not finished yet.
+
+## CURLMINFO_XFERS_PENDING
+
+The number of current easy handles waiting to start. An added transfer
+might become pending for various reasons: a connection limit forces it
+to wait, resolving DNS is not finished or it is not clear if an existing,
+matching connection may allow multiplexing (HTTP/2 or HTTP/3).
+
+## CURLMINFO_XFERS_DONE
+
+The number of easy handles currently finished, but not yet processed
+via curl_multi_info_read(3).
+
+## CURLMINFO_XFERS_ADDED
+
+The cumulative number of all easy handles added to the multi, ever.
+This includes internal handles added for tasks (like resolving
+via DoH, for example).
+
+For the current number of easy handles managed by the multi, use
+*CURLMINFO_XFERS_CURRENT*.
+
+# %PROTOCOLS%
+
+# EXAMPLE
+
+~~~c
+int main(void)
+{
+  /* init a multi stack */
+  CURLM *multi = curl_multi_init();
+  CURL *curl = curl_easy_init();
+  curl_off_t n;
+
+  if(curl) {
+    /* add the transfer */
+    curl_multi_add_handle(multi, curl);
+
+    curl_multi_get_offt(multi, CURLMINFO_XFERS_ADDED, &n);
+    /* on successful add, n is 1 */
+  }
+}
+~~~
+
+# %AVAILABILITY%
+
+# RETURN VALUE
+
+This function returns a CURLMcode indicating success or error.
+
+CURLM_OK (0) means everything was OK, non-zero means an error occurred,
+see libcurl-errors(3).

docs/libcurl/symbols-in-versions

@@ -553,6 +553,12 @@ CURLM_RECURSIVE_API_CALL        7.59.0
 CURLM_UNKNOWN_OPTION            7.15.4
 CURLM_UNRECOVERABLE_POLL        7.84.0
 CURLM_WAKEUP_FAILURE            7.68.0
+CURLMINFO_NONE                  8.16.0
+CURLMINFO_XFERS_CURRENT         8.16.0
+CURLMINFO_XFERS_RUNNING         8.16.0
+CURLMINFO_XFERS_PENDING         8.16.0
+CURLMINFO_XFERS_DONE            8.16.0
+CURLMINFO_XFERS_ADDED           8.16.0
 CURLMIMEOPT_FORMESCAPE          7.81.0
 CURLMOPT_CHUNK_LENGTH_PENALTY_SIZE 7.30.0
 CURLMOPT_CONTENT_LENGTH_PENALTY_SIZE 7.30.0

include/curl/multi.h

@@ -448,6 +448,36 @@ CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle,
  */
 CURL_EXTERN CURL **curl_multi_get_handles(CURLM *multi_handle);
 
+
+typedef enum {
+  CURLMINFO_NONE, /* first, never use this */
+  /* The number of easy handles currently managed by the multi handle,
+   * e.g. have been added but not yet removed. */
+  CURLMINFO_XFERS_CURRENT = 1,
+  /* The number of easy handles running, e.g. not done and not queueing. */
+  CURLMINFO_XFERS_RUNNING = 2,
+  /* The number of easy handles waiting to start, e.g. for a connection
+   * to become available due to limits on parallelism, max connections
+   * or other factors. */
+  CURLMINFO_XFERS_PENDING = 3,
+  /* The number of easy handles finished, waiting for their results to
+   * be read via `curl_multi_info_read()`. */
+  CURLMINFO_XFERS_DONE = 4,
+  /* The total number of easy handles added to the multi handle, ever. */
+  CURLMINFO_XFERS_ADDED = 5
+} CURLMinfo_offt;
+
+/*
+ * Name:    curl_multi_get_offt()
+ *
+ * Desc:    Retrieves a numeric value for the `CURLMINFO_*` enums.
+ *
+ * Returns: CULRM_OK or error when value could not be obtained.
+ */
+CURL_EXTERN CURLMcode curl_multi_get_offt(CURLM *multi_handle,
+                                          CURLMinfo_offt info,
+                                          curl_off_t *pvalue);
+
 /*
  * Name: curl_push_callback
  *

lib/libcurl.def

@@ -54,6 +54,7 @@ curl_multi_assign
 curl_multi_cleanup
 curl_multi_fdset
 curl_multi_get_handles
+curl_multi_get_offt
 curl_multi_info_read
 curl_multi_init
 curl_multi_perform

lib/multi.c

@@ -479,6 +479,7 @@ CURLMcode curl_multi_add_handle(CURLM *m, CURL *d)
   /* add the easy handle to the process set */
   Curl_uint_bset_add(&multi->process, data->mid);
   ++multi->xfers_alive;
+  ++multi->xfers_total_ever;
 
   Curl_cpool_xfer_init(data);
   multi_warn_debug(multi, data);
@@ -3741,6 +3742,43 @@ CURL **curl_multi_get_handles(CURLM *m)
   return a;
 }
 
+CURLMcode curl_multi_get_offt(CURLM *m,
+                              CURLMinfo_offt info,
+                              curl_off_t *pvalue)
+{
+  struct Curl_multi *multi = m;
+
+  if(!GOOD_MULTI_HANDLE(multi))
+    return CURLM_BAD_HANDLE;
+  if(!pvalue)
+    return CURLM_BAD_FUNCTION_ARGUMENT;
+
+  switch(info) {
+  case CURLMINFO_XFERS_CURRENT: {
+    unsigned int n = Curl_uint_tbl_count(&multi->xfers);
+    if(n && multi->admin)
+      --n;
+    *pvalue = (curl_off_t)n;
+    return CURLM_OK;
+  }
+  case CURLMINFO_XFERS_RUNNING:
+    *pvalue = (curl_off_t)Curl_uint_bset_count(&multi->process);
+    return CURLM_OK;
+  case CURLMINFO_XFERS_PENDING:
+    *pvalue = (curl_off_t)Curl_uint_bset_count(&multi->pending);
+    return CURLM_OK;
+  case CURLMINFO_XFERS_DONE:
+    *pvalue = (curl_off_t)Curl_uint_bset_count(&multi->msgsent);
+    return CURLM_OK;
+  case CURLMINFO_XFERS_ADDED:
+    *pvalue = multi->xfers_total_ever;
+    return CURLM_OK;
+  default:
+    *pvalue = -1;
+    return CURLM_UNKNOWN_OPTION;
+  }
+}
+
 CURLcode Curl_multi_xfer_buf_borrow(struct Curl_easy *data,
                                     char **pbuf, size_t *pbuflen)
 {

lib/multihandle.h

@@ -89,6 +89,7 @@ struct Curl_multi {
 
   unsigned int xfers_alive; /* amount of added transfers that have
                                not yet reached COMPLETE state */
+  curl_off_t xfers_total_ever; /* total of added transfers, ever. */
   struct uint_tbl xfers; /* transfers added to this multi */
   /* Each transfer's mid may be present in at most one of these */
   struct uint_bset process; /* transfer being processed */

lib/uint-bset.c

@@ -79,8 +79,9 @@ UNITTEST unsigned int Curl_uint_bset_capacity(struct uint_bset *bset)
 {
   return bset->nslots * 64;
 }
+#endif
 
-UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset)
+unsigned int Curl_uint_bset_count(struct uint_bset *bset)
 {
   unsigned int i;
   unsigned int n = 0;
@@ -90,7 +91,6 @@ UNITTEST unsigned int Curl_uint_bset_count(struct uint_bset *bset)
   }
   return n;
 }
-#endif
 
 bool Curl_uint_bset_empty(struct uint_bset *bset)
 {

scripts/singleuse.pl

@@ -111,6 +111,7 @@ my %api = (
     'curl_multi_cleanup' => 'API',
     'curl_multi_fdset' => 'API',
     'curl_multi_get_handles' => 'API',
+    'curl_multi_get_offt' => 'API',
     'curl_multi_info_read' => 'API',
     'curl_multi_init' => 'API',
     'curl_multi_perform' => 'API',

src/tool_operate.c

@@ -207,7 +207,6 @@ static curl_off_t VmsSpecialSize(const char *name,
 
 struct per_transfer *transfers; /* first node */
 static struct per_transfer *transfersl; /* last node */
-static curl_off_t all_pers;
 
 /* add_per_transfer creates a new 'per_transfer' node in the linked
    list of transfers */
@@ -229,8 +228,6 @@ static CURLcode add_per_transfer(struct per_transfer **per)
     transfersl = p;
   }
   *per = p;
-  all_xfers++; /* count total number of transfers added */
-  all_pers++;
 
   return CURLE_OK;
 }
@@ -259,7 +256,6 @@ static struct per_transfer *del_per_transfer(struct per_transfer *per)
     transfersl = p;
 
   free(per);
-  all_pers--;
 
   return n;
 }
@@ -1420,9 +1416,17 @@ static CURLcode add_parallel_transfers(struct GlobalConfig *global,
   CURLMcode mcode;
   bool sleeping = FALSE;
   char *errorbuf;
+  curl_off_t nxfers;
+
   *addedp = FALSE;
   *morep = FALSE;
-  if(all_pers < (global->parallel_max*2)) {
+  mcode = curl_multi_get_offt(multi, CURLMINFO_XFERS_CURRENT, &nxfers);
+  if(mcode) {
+    DEBUGASSERT(0);
+    return CURLE_UNKNOWN_OPTION;
+  }
+
+  if(nxfers < (curl_off_t)(global->parallel_max*2)) {
     bool skipped = FALSE;
     do {
       result = create_transfer(global, share, addedp, &skipped);
@@ -1762,7 +1766,7 @@ static CURLcode check_finished(struct parastate *s)
   CURLMsg *msg;
   bool checkmore = FALSE;
   struct GlobalConfig *global = s->global;
-  progress_meter(global, &s->start, FALSE);
+  progress_meter(global, s->multi, &s->start, FALSE);
   do {
     msg = curl_multi_info_read(s->multi, &rc);
     if(msg) {
@@ -1887,7 +1891,7 @@ static CURLcode parallel_transfers(struct GlobalConfig *global,
         result = check_finished(s);
     }
 
-    (void)progress_meter(global, &s->start, TRUE);
+    (void)progress_meter(global, s->multi, &s->start, TRUE);
   }
 
   /* Make sure to return some kind of error if there was a multi problem */

src/tool_progress.c

@@ -134,8 +134,6 @@ static curl_off_t all_ultotal = 0;
 static curl_off_t all_dlalready = 0;
 static curl_off_t all_ulalready = 0;
 
-curl_off_t all_xfers = 0;   /* current total */
-
 struct speedcount {
   curl_off_t dl;
   curl_off_t ul;
@@ -151,6 +149,7 @@ static struct speedcount speedstore[SPEEDCNT];
   |  6 --   9.9G     0     2     2   0:00:40  0:00:02  0:00:37 4087M
 */
 bool progress_meter(struct GlobalConfig *global,
+                    CURLM *multi,
                     struct curltime *start,
                     bool final)
 {
@@ -182,9 +181,10 @@ bool progress_meter(struct GlobalConfig *global,
     struct per_transfer *per;
     curl_off_t all_dlnow = 0;
     curl_off_t all_ulnow = 0;
+    curl_off_t xfers_added = 0;
+    curl_off_t xfers_running = 0;
     bool dlknown = TRUE;
     bool ulknown = TRUE;
-    curl_off_t all_running = 0; /* in progress */
     curl_off_t speed = 0;
     unsigned int i;
     stamp = now;
@@ -210,8 +210,6 @@ bool progress_meter(struct GlobalConfig *global,
         all_ultotal += per->ultotal;
         per->ultotal_added = TRUE;
       }
-      if(per->added)
-        all_running++;
     }
     if(dlknown && all_dltotal)
       msnprintf(dlpercen, sizeof(dlpercen), "%3" CURL_FORMAT_CURL_OFF_T,
@@ -274,6 +272,8 @@ bool progress_meter(struct GlobalConfig *global,
     }
     time2str(time_spent, spent);
 
+    (void)curl_multi_get_offt(multi, CURLMINFO_XFERS_ADDED, &xfers_added);
+    (void)curl_multi_get_offt(multi, CURLMINFO_XFERS_RUNNING, &xfers_running);
     fprintf(tool_stderr,
             "\r"
             "%-3s " /* percent downloaded */
@@ -292,8 +292,8 @@ bool progress_meter(struct GlobalConfig *global,
             ulpercen,  /* 3 letters */
             max5data(all_dlnow, buffer[0]),
             max5data(all_ulnow, buffer[1]),
-            all_xfers,
-            all_running,
+            xfers_added,
+            xfers_running,
             time_total,
             time_spent,
             time_left,

src/tool_progress.h

@@ -32,10 +32,9 @@ int xferinfo_cb(void *clientp,
                 curl_off_t ulnow);
 
 bool progress_meter(struct GlobalConfig *global,
+                    CURLM *multi,
                     struct curltime *start,
                     bool final);
 void progress_finalize(struct per_transfer *per);
 
-extern curl_off_t all_xfers;   /* total number */
-
 #endif /* HEADER_CURL_TOOL_PROGRESS_H */

tests/data/test1135

@@ -109,6 +109,7 @@ curl_multi_timeout
 curl_multi_setopt
 curl_multi_assign
 curl_multi_get_handles
+curl_multi_get_offt
 curl_pushheader_bynum
 curl_pushheader_byname
 curl_multi_waitfds