progress: count amount of data "delivered" to application

... and apply the CURLOPT_MAXFILESIZE limit (if set) on that as well.
This effectively protects the user against "zip bombs".

Test case 1618 verifies using a 14 byte brotli payload that otherwise
explodes to 102400 zero bytes.

Closes #20787

docs/KNOWN_RISKS.md

@@ -144,3 +144,6 @@ and secure algorithms.
 When asking curl or libcurl to automatically decompress data on arrival, there
 is a risk that the size of the output from the decompression process ends up
 many times larger than the input data size.
+
+Since curl 8.20.0, users can mitigate this risk by setting the max filesize
+option that also covers the decompressed size.

docs/cmdline-opts/max-filesize.md

@@ -37,3 +37,6 @@ threshold during transfer.
 Starting in curl 8.19.0, the maximum size can be specified using a fraction as
 in `2.5M` for two and a half megabytes. It only works with a period (`.`)
 delimiter, independent of what your locale might prefer.
+
+Since 8.20.0, this option also stops ongoing transfers that would reach this
+threshold due to automatic decompression using --compressed.

docs/cmdline-opts/write-out.md

@@ -192,6 +192,11 @@ known as "http_code"). (Added in 7.18.2)
 ## `scheme`
 The URL scheme (sometimes called protocol) that was effectively used. (Added in 7.52.0)
 
+## `size_delivered`
+The total amount of data that were saved or written to stdout. When
+--compressed is used, this is likely different than `size_download`. Includes
+the headers in the count if --include is used.
+
 ## `size_download`
 The total amount of bytes that were downloaded. This is the size of the
 body/data that was transferred, excluding headers.

docs/libcurl/curl_easy_getinfo.md

@@ -304,6 +304,10 @@ RTSP session ID. See CURLINFO_RTSP_SESSION_ID(3)
 
 The scheme used for the connection. See CURLINFO_SCHEME(3)
 
+## CURLINFO_SIZE_DELIVERED
+
+Number of bytes passed to the write callback. See CURLINFO_SIZE_DELIVERED(3)
+
 ## CURLINFO_SIZE_DOWNLOAD
 
 (**Deprecated**) Number of bytes downloaded. See CURLINFO_SIZE_DOWNLOAD(3)

docs/libcurl/opts/CURLINFO_SIZE_DELIVERED.md

@@ -0,0 +1,78 @@
+---
+c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+SPDX-License-Identifier: curl
+Title: CURLINFO_SIZE_DELIVERED
+Section: 3
+Source: libcurl
+See-also:
+  - CURLINFO_SIZE_DOWNLOAD_T (3)
+  - CURLINFO_CONTENT_LENGTH_DOWNLOAD_T (3)
+  - CURLOPT_MAXFILESIZE (3)
+  - curl_easy_getinfo (3)
+  - curl_easy_setopt (3)
+Protocol:
+  - All
+Added-in: 8.20.0
+---
+
+# NAME
+
+CURLINFO_SIZE_DELIVERED - number of delivered bytes
+
+# SYNOPSIS
+
+~~~c
+#include <curl/curl.h>
+
+CURLcode curl_easy_getinfo(CURL *handle, CURLINFO_SIZE_DELIVERED,
+                           curl_off_t *dlp);
+~~~
+
+# DESCRIPTION
+
+Pass a pointer to a *curl_off_t* to receive the total amount of bytes that
+were passed on to the write callback in the download. The amount is only for
+the latest transfer and gets reset again for each new transfer. This counts
+actual payload data, what's also commonly called body. All meta and header
+data is excluded from this amount (unless CURLOPT_HEADER(3) is set).
+
+The delivered size may differ from the size retrieved with
+CURLINFO_SIZE_DOWNLOAD_T(3) when CURLOPT_ACCEPT_ENCODING(3) is used for
+automatic data decompression, as this is then the size of the uncompressed
+body while CURLINFO_SIZE_DOWNLOAD_T(3) returns the size of the download.
+
+# %PROTOCOLS%
+
+# EXAMPLE
+
+~~~c
+int main(void)
+{
+  CURL *curl = curl_easy_init();
+  if(curl) {
+    CURLcode result;
+    curl_easy_setopt(curl, CURLOPT_URL, "https://example.com");
+
+    /* Perform the request */
+    result = curl_easy_perform(curl);
+
+    if(result == CURLE_OK) {
+      /* check the size */
+      curl_off_t dl;
+      result = curl_easy_getinfo(curl, CURLINFO_SIZE_DELIVERED, &dl);
+      if(result == CURLE_OK) {
+        printf("Stored %" CURL_FORMAT_CURL_OFF_T " bytes\n", dl);
+      }
+    }
+  }
+}
+~~~
+
+# %AVAILABILITY%
+
+# RETURN VALUE
+
+curl_easy_setopt(3) returns a CURLcode indicating success or error.
+
+CURLE_OK (0) means everything was OK, non-zero means an error occurred, see
+libcurl-errors(3).

docs/libcurl/opts/CURLOPT_MAXFILESIZE.md

@@ -41,6 +41,9 @@ If you want a limit above 2GB, use CURLOPT_MAXFILESIZE_LARGE(3).
 Since 8.4.0, this option also stops ongoing transfers if they reach this
 threshold.
 
+Since 8.20.0, this option also stops ongoing transfers that would reach this
+threshold due to automatic decompression using CURLOPT_ACCEPT_ENCODING(3).
+
 # DEFAULT
 
 0, meaning disabled.

docs/libcurl/opts/CURLOPT_MAXFILESIZE_LARGE.md

@@ -42,6 +42,9 @@ ends up being larger than this given limit.
 Since 8.4.0, this option also stops ongoing transfers if they reach this
 threshold.
 
+Since 8.20.0, this option also stops ongoing transfers that would reach this
+threshold due to automatic decompression using CURLOPT_ACCEPT_ENCODING(3).
+
 # DEFAULT
 
 0, meaning disabled.

docs/libcurl/opts/Makefile.inc

@@ -83,6 +83,7 @@ man_MANS =                                      \
   CURLINFO_RTSP_SERVER_CSEQ.3                   \
   CURLINFO_RTSP_SESSION_ID.3                    \
   CURLINFO_SCHEME.3                             \
+  CURLINFO_SIZE_DELIVERED.3                     \
   CURLINFO_SIZE_DOWNLOAD.3                      \
   CURLINFO_SIZE_DOWNLOAD_T.3                    \
   CURLINFO_SIZE_UPLOAD.3                        \

docs/libcurl/symbols-in-versions

@@ -492,6 +492,7 @@ CURLINFO_RTSP_CSEQ_RECV         7.20.0
 CURLINFO_RTSP_SERVER_CSEQ       7.20.0
 CURLINFO_RTSP_SESSION_ID        7.20.0
 CURLINFO_SCHEME                 7.52.0
+CURLINFO_SIZE_DELIVERED         8.20.0
 CURLINFO_SIZE_DOWNLOAD          7.4.1         7.55.0
 CURLINFO_SIZE_DOWNLOAD_T        7.55.0
 CURLINFO_SIZE_UPLOAD            7.4.1         7.55.0

include/curl/curl.h

@@ -2991,7 +2991,8 @@ typedef enum {
   CURLINFO_EARLYDATA_SENT_T = CURLINFO_OFF_T + 68,
   CURLINFO_HTTPAUTH_USED    = CURLINFO_LONG + 69,
   CURLINFO_PROXYAUTH_USED   = CURLINFO_LONG + 70,
-  CURLINFO_LASTONE          = 70
+  CURLINFO_SIZE_DELIVERED   = CURLINFO_OFF_T + 71,
+  CURLINFO_LASTONE          = 71
 } CURLINFO;
 
 /* CURLINFO_RESPONSE_CODE is the new name for the option previously known as

lib/cw-out.c

@@ -31,7 +31,7 @@
 #include "transfer.h"
 #include "cw-out.h"
 #include "cw-pause.h"
-
+#include "progress.h"
 
 /**
  * OVERALL DESIGN of this client writer
@@ -228,8 +228,8 @@ static CURLcode cw_out_ptr_flush(struct cw_out_ctx *ctx,
   curl_write_callback wcb = NULL;
   void *wcb_data;
   size_t max_write, min_write;
-  size_t wlen, nwritten;
-  CURLcode result;
+  size_t wlen, nwritten = 0;
+  CURLcode result = CURLE_OK;
 
   /* If we errored once, we do not invoke the client callback again */
   if(ctx->errored)
@@ -253,10 +253,15 @@ static CURLcode cw_out_ptr_flush(struct cw_out_ctx *ctx,
       if(!flush_all && blen < min_write)
         break;
       wlen = max_write ? CURLMIN(blen, max_write) : blen;
-      result = cw_out_cb_write(ctx, data, wcb, wcb_data, otype,
-                               buf, wlen, &nwritten);
+      if(otype == CW_OUT_BODY)
+        result = Curl_pgrs_deliver_check(data, wlen);
+      if(!result)
+        result = cw_out_cb_write(ctx, data, wcb, wcb_data, otype,
+                                 buf, wlen, &nwritten);
       if(result)
         return result;
+      if(otype == CW_OUT_BODY)
+        Curl_pgrs_deliver_inc(data, nwritten);
       *pconsumed += nwritten;
       blen -= nwritten;
       buf += nwritten;

lib/getinfo.c

@@ -415,6 +415,9 @@ static CURLcode getinfo_offt(struct Curl_easy *data, CURLINFO info,
   case CURLINFO_FILETIME_T:
     *param_offt = (curl_off_t)data->info.filetime;
     break;
+  case CURLINFO_SIZE_DELIVERED:
+    *param_offt = data->progress.deliver;
+    break;
   case CURLINFO_SIZE_UPLOAD_T:
     *param_offt = data->progress.ul.cur_size;
     break;

lib/progress.c

@@ -213,6 +213,7 @@ void Curl_pgrsReset(struct Curl_easy *data)
   Curl_pgrsSetUploadSize(data, -1);
   Curl_pgrsSetDownloadSize(data, -1);
   data->progress.speeder_c = 0; /* reset speed records */
+  data->progress.deliver = 0;
   pgrs_speedinit(data);
 }
 
@@ -341,6 +342,26 @@ void Curl_pgrsStartNow(struct Curl_easy *data)
   p->ul_size_known = FALSE;
 }
 
+/* check that the 'delta' amount of bytes are okay to deliver to the
+   application, or return error if not. */
+CURLcode Curl_pgrs_deliver_check(struct Curl_easy *data, size_t delta)
+{
+  if(data->set.max_filesize &&
+     ((curl_off_t)delta > data->set.max_filesize - data->progress.deliver)) {
+    failf(data, "Would have exceeded max file size");
+    return CURLE_FILESIZE_EXCEEDED;
+  }
+  return CURLE_OK;
+}
+
+/* this counts how much data is delivered to the application, which
+   in compressed cases may differ from downloaded amount */
+void Curl_pgrs_deliver_inc(struct Curl_easy *data, size_t delta)
+{
+  data->progress.deliver += delta;
+}
+
+
 void Curl_pgrs_download_inc(struct Curl_easy *data, size_t delta)
 {
   if(delta) {

lib/progress.h

@@ -50,7 +50,8 @@ int Curl_pgrsDone(struct Curl_easy *data);
 void Curl_pgrsStartNow(struct Curl_easy *data);
 void Curl_pgrsSetDownloadSize(struct Curl_easy *data, curl_off_t size);
 void Curl_pgrsSetUploadSize(struct Curl_easy *data, curl_off_t size);
-
+CURLcode Curl_pgrs_deliver_check(struct Curl_easy *data, size_t delta);
+void Curl_pgrs_deliver_inc(struct Curl_easy *data, size_t delta);
 void Curl_pgrs_download_inc(struct Curl_easy *data, size_t delta);
 void Curl_pgrs_upload_inc(struct Curl_easy *data, size_t delta);
 void Curl_pgrsSetUploadCounter(struct Curl_easy *data, curl_off_t size);

lib/urldata.h

@@ -551,6 +551,7 @@ struct Progress {
                       force redraw at next call */
   struct pgrs_dir ul;
   struct pgrs_dir dl;
+  curl_off_t deliver; /* amount of data delivered to application */
 
   curl_off_t current_speed; /* uses the currently fastest transfer */
   curl_off_t earlydata_sent;

src/tool_writeout.c

@@ -461,6 +461,8 @@ static const struct writeoutvar variables[] = {
   { "remote_port", VAR_PRIMARY_PORT, CURLINFO_PRIMARY_PORT, writeLong },
   { "response_code", VAR_HTTP_CODE, CURLINFO_RESPONSE_CODE, writeLong },
   { "scheme", VAR_SCHEME, CURLINFO_SCHEME, writeString },
+  { "size_delivered", VAR_SIZE_DELIVERED, CURLINFO_SIZE_DELIVERED,
+    writeOffset },
   { "size_download", VAR_SIZE_DOWNLOAD, CURLINFO_SIZE_DOWNLOAD_T,
     writeOffset },
   { "size_header", VAR_HEADER_SIZE, CURLINFO_HEADER_SIZE, writeLong },

src/tool_writeout.h

@@ -90,6 +90,7 @@ typedef enum {
   VAR_REFERER,
   VAR_REQUEST_SIZE,
   VAR_SCHEME,
+  VAR_SIZE_DELIVERED,
   VAR_SIZE_DOWNLOAD,
   VAR_SIZE_UPLOAD,
   VAR_SPEED_DOWNLOAD,

tests/data/Makefile.am

@@ -213,7 +213,7 @@ test1580 test1581 test1582 test1583 test1584 test1585 test1586 \
 test1590 test1591 test1592 test1593 test1594 test1595 test1596 test1597 \
 test1598 test1599 test1600 test1601 test1602 test1603 test1604 test1605 \
 test1606 test1607 test1608 test1609 test1610 test1611 test1612 test1613 \
-test1614 test1615 test1616 test1617 \
+test1614 test1615 test1616 test1617 test1618 \
 test1620 test1621 test1622 test1623 test1624 test1625 test1626 test1627 \
 \
 test1630 test1631 test1632 test1633 test1634 test1635 test1636 test1637 \

tests/data/test1618

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="US-ASCII"?>
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+compressed
+brotli
+</keywords>
+</info>
+
+<reply>
+
+<data crlf="headers" nonewline="yes">
+HTTP/1.1 200 OK
+Date: Mon, 29 Nov 2004 21:56:53 GMT
+Vary: Accept-Encoding
+Content-Encoding: br
+Content-Length: 14
+
+%hex[%81%fa%7f%0c%fc%13%00%f1%58%20%90%7b%18%00]hex%
+</data>
+
+<datacheck crlf="headers">
+HTTP/1.1 200 OK
+Date: Mon, 29 Nov 2004 21:56:53 GMT
+Vary: Accept-Encoding
+Content-Encoding: br
+Content-Length: 14
+
+</datacheck>
+
+</reply>
+
+# Client-side
+<client>
+<features>
+brotli
+</features>
+<server>
+http
+</server>
+<name>
+HTTP GET brotli compression bomb
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --compressed --max-filesize=1000
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strippart>
+s/^Accept-Encoding: [a-zA-Z, ]*/Accept-Encoding: xxx/
+</strippart>
+<protocol crlf="headers">
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+Accept-Encoding: xxx
+
+</protocol>
+<errorcode>
+63
+</errorcode>
+</verify>
+</testcase>

tests/data/test220

@@ -50,7 +50,7 @@ http
 HTTP GET gzip compressed content
 </name>
 <command>
-http://%HOSTIP:%HTTPPORT/%TESTNUMBER --compressed
+http://%HOSTIP:%HTTPPORT/%TESTNUMBER --compressed -s -w '%{stderr}%{size_delivered}\n'
 </command>
 </client>
 
@@ -67,5 +67,8 @@ Accept: */*
 Accept-Encoding: xxx
 
 </protocol>
+<stderr mode="text">
+24
+</stderr>
 </verify>
 </testcase>

tests/data/test970

@@ -57,7 +57,7 @@ Accept: */*
 
 </protocol>
 <stdout nonewline="yes">
-{"certs":"","conn_id":0,"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"num_retries":0,"proxy_ssl_verify_result":0,"proxy_used":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"http","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_posttransfer":0.000013,"time_pretransfer":0.000013,"time_queue":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"tls_earlydata":0,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.fragment":null,"url.host":"127.0.0.1","url.options":null,"url.password":null,"url.path":"/%TESTNUMBER","url.port":"%HTTPPORT","url.query":null,"url.scheme":"http","url.user":null,"url.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urle.fragment":null,"urle.host":"127.0.0.1","urle.options":null,"urle.password":null,"urle.path":"/%TESTNUMBER","urle.port":"%HTTPPORT","urle.query":null,"urle.scheme":"http","urle.user":null,"urle.zoneid":null,"urlnum":0,"xfer_id":0,"curl_version":"curl-unit-test-fake-version"}
+{"certs":"","conn_id":0,"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"num_retries":0,"proxy_ssl_verify_result":0,"proxy_used":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"http","size_delivered":445,"size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_posttransfer":0.000013,"time_pretransfer":0.000013,"time_queue":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"tls_earlydata":0,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.fragment":null,"url.host":"127.0.0.1","url.options":null,"url.password":null,"url.path":"/%TESTNUMBER","url.port":"%HTTPPORT","url.query":null,"url.scheme":"http","url.user":null,"url.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urle.fragment":null,"urle.host":"127.0.0.1","urle.options":null,"urle.password":null,"urle.path":"/%TESTNUMBER","urle.port":"%HTTPPORT","urle.query":null,"urle.scheme":"http","urle.user":null,"urle.zoneid":null,"urlnum":0,"xfer_id":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>

tests/data/test972

@@ -58,7 +58,7 @@ Accept: */*
 
 </protocol>
 <stdout mode="text">
-{"certs":"","conn_id":0,"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"num_retries":0,"proxy_ssl_verify_result":0,"proxy_used":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"http","size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_posttransfer":0.000013,"time_pretransfer":0.000013,"time_queue":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"tls_earlydata":0,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.fragment":null,"url.host":"127.0.0.1","url.options":null,"url.password":null,"url.path":"/%TESTNUMBER","url.port":"%HTTPPORT","url.query":null,"url.scheme":"http","url.user":null,"url.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urle.fragment":null,"urle.host":"127.0.0.1","urle.options":null,"urle.password":null,"urle.path":"/%TESTNUMBER","urle.port":"%HTTPPORT","urle.query":null,"urle.scheme":"http","urle.user":null,"urle.zoneid":null,"urlnum":0,"xfer_id":0,"curl_version":"curl-unit-test-fake-version"}
+{"certs":"","conn_id":0,"content_type":"text/html","errormsg":null,"exitcode":0,"filename_effective":"%LOGDIR/out%TESTNUMBER","ftp_entry_path":null,"http_code":200,"http_connect":0,"http_version":"1.1","local_ip":"127.0.0.1","local_port":13,"method":"GET","num_certs":0,"num_connects":1,"num_headers":9,"num_redirects":0,"num_retries":0,"proxy_ssl_verify_result":0,"proxy_used":0,"redirect_url":null,"referer":null,"remote_ip":"%HOSTIP","remote_port":%HTTPPORT,"response_code":200,"scheme":"http","size_delivered":445,"size_download":445,"size_header":4019,"size_request":4019,"size_upload":0,"speed_download":13,"speed_upload":13,"ssl_verify_result":0,"time_appconnect":0.000013,"time_connect":0.000013,"time_namelookup":0.000013,"time_posttransfer":0.000013,"time_pretransfer":0.000013,"time_queue":0.000013,"time_redirect":0.000013,"time_starttransfer":0.000013,"time_total":0.000013,"tls_earlydata":0,"url":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","url.fragment":null,"url.host":"127.0.0.1","url.options":null,"url.password":null,"url.path":"/%TESTNUMBER","url.port":"%HTTPPORT","url.query":null,"url.scheme":"http","url.user":null,"url.zoneid":null,"url_effective":"http://%HOSTIP:%HTTPPORT/%TESTNUMBER","urle.fragment":null,"urle.host":"127.0.0.1","urle.options":null,"urle.password":null,"urle.path":"/%TESTNUMBER","urle.port":"%HTTPPORT","urle.query":null,"urle.scheme":"http","urle.user":null,"urle.zoneid":null,"urlnum":0,"xfer_id":0,"curl_version":"curl-unit-test-fake-version"}
 </stdout>
 </verify>
 </testcase>