http: unfold response headers earlier

Make the low-level HTTP header "builder" unfold headers so that
everything else can keep pretending folding does not exist.

This code no longer tries to reduce repeated leading whitespace (in the
continued folded header) to a single one. To avoid having to have a
special state for that.

Adjusted two test cases accordingly

Closes #19949

lib/headers.c

@@ -215,55 +215,6 @@ static CURLcode namevalue(char *header, size_t hlen, unsigned int type,
   return CURLE_OK;
 }
 
-static CURLcode unfold_value(struct Curl_easy *data, const char *value,
-                             size_t vlen)  /* length of the incoming header */
-{
-  struct Curl_header_store *hs;
-  struct Curl_header_store *newhs;
-  size_t olen; /* length of the old value */
-  size_t oalloc; /* length of the old name + value + separator */
-  size_t offset;
-  DEBUGASSERT(data->state.prevhead);
-  hs = data->state.prevhead;
-  olen = strlen(hs->value);
-  offset = hs->value - hs->buffer;
-  oalloc = olen + offset + 1;
-
-  /* skip all trailing space letters */
-  while(vlen && ISBLANK(value[vlen - 1]))
-    vlen--;
-
-  /* save only one leading space */
-  while((vlen > 1) && ISBLANK(value[0]) && ISBLANK(value[1])) {
-    vlen--;
-    value++;
-  }
-
-  /* since this header block might move in the realloc below, it needs to
-     first be unlinked from the list and then re-added again after the
-     realloc */
-  Curl_node_remove(&hs->node);
-
-  /* new size = struct + new value length + old name+value length */
-  newhs = Curl_saferealloc(hs, sizeof(*hs) + vlen + oalloc + 1);
-  if(!newhs)
-    return CURLE_OUT_OF_MEMORY;
-  /* ->name and ->value point into ->buffer (to keep the header allocation
-     in a single memory block), which now potentially have moved. Adjust
-     them. */
-  newhs->name = newhs->buffer;
-  newhs->value = &newhs->buffer[offset];
-
-  /* put the data at the end of the previous data, not the newline */
-  memcpy(&newhs->value[olen], value, vlen);
-  newhs->value[olen + vlen] = 0; /* null-terminate at newline */
-
-  /* insert this node into the list of headers */
-  Curl_llist_append(&data->state.httphdrs, newhs, &newhs->node);
-  data->state.prevhead = newhs;
-  return CURLE_OK;
-}
-
 /*
  * Curl_headers_push() gets passed a full HTTP header to store. It gets called
  * immediately before the header callback. The header is CRLF, CR or LF
@@ -292,20 +243,14 @@ CURLcode Curl_headers_push(struct Curl_easy *data, const char *header,
     /* neither CR nor LF as terminator is not a valid header */
     return CURLE_WEIRD_SERVER_REPLY;
 
-  if((header[0] == ' ') || (header[0] == '\t')) {
-    if(data->state.prevhead)
-      /* line folding, append value to the previous header's value */
-      return unfold_value(data, header, hlen);
-    else {
-      /* cannot unfold without a previous header. Instead of erroring, just
-         pass the leading blanks. */
-      while(hlen && ISBLANK(*header)) {
-        header++;
-        hlen--;
-      }
-      if(!hlen)
-        return CURLE_WEIRD_SERVER_REPLY;
+  if(ISBLANK(header[0])) {
+    /* pass leading blanks */
+    while(hlen && ISBLANK(*header)) {
+      header++;
+      hlen--;
     }
+    if(!hlen)
+      return CURLE_WEIRD_SERVER_REPLY;
   }
   if(Curl_llist_count(&data->state.httphdrs) >= MAX_HTTP_RESP_HEADER_COUNT) {
     failf(data, "Too many response headers, %d is max",

lib/http.c

@@ -111,6 +111,8 @@ static CURLcode http_statusline(struct Curl_easy *data,
                                 struct connectdata *conn);
 static CURLcode http_target(struct Curl_easy *data, struct dynbuf *req);
 static CURLcode http_useragent(struct Curl_easy *data);
+static CURLcode http_write_header(struct Curl_easy *data,
+                                  const char *hd, size_t hdlen);
 
 /*
  * HTTP handler interface.
@@ -1575,6 +1577,10 @@ CURLcode Curl_http_done(struct Curl_easy *data,
   data->state.authhost.multipass = FALSE;
   data->state.authproxy.multipass = FALSE;
 
+  if(curlx_dyn_len(&data->state.headerb)) {
+    (void)http_write_header(data, curlx_dyn_ptr(&data->state.headerb),
+                            curlx_dyn_len(&data->state.headerb));
+  }
   curlx_dyn_reset(&data->state.headerb);
 
   if(status)
@@ -2947,6 +2953,7 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
   /* make sure the header buffer is reset - if there are leftovers from a
      previous transfer */
   curlx_dyn_reset(&data->state.headerb);
+  data->state.maybe_folded = FALSE;
 
   if(!data->conn->bits.reuse) {
     result = http_check_new_conn(data);
@@ -4283,6 +4290,18 @@ static CURLcode http_rw_hd(struct Curl_easy *data,
   return CURLE_OK;
 }
 
+/* cut off the newline characters */
+static void unfold_header(struct Curl_easy *data)
+{
+  size_t len = curlx_dyn_len(&data->state.headerb);
+  char *hd = curlx_dyn_ptr(&data->state.headerb);
+  if(len && (hd[len -1] == '\n'))
+    len--;
+  if(len && (hd[len -1] == '\r'))
+    len--;
+  curlx_dyn_setlen(&data->state.headerb, len);
+}
+
 /*
  * Read any HTTP header lines from the server and pass them to the client app.
  */
@@ -4296,10 +4315,32 @@ static CURLcode http_parse_headers(struct Curl_easy *data,
   char *end_ptr;
   bool leftover_body = FALSE;
 
+  /* we have bytes for the next header, make sure it is not a folded header
+     before passing it on */
+  if(data->state.maybe_folded && blen) {
+    if(ISBLANK(buf[0])) {
+      /* folded, remove the trailing newlines and append the next header */
+      unfold_header(data);
+    }
+    else {
+      /* the header data we hold is a complete header, pass it on */
+      size_t ignore_this;
+      result = http_rw_hd(data, curlx_dyn_ptr(&data->state.headerb),
+                          curlx_dyn_len(&data->state.headerb),
+                          NULL, 0, &ignore_this);
+      curlx_dyn_reset(&data->state.headerb);
+      if(result)
+        return result;
+    }
+    data->state.maybe_folded = FALSE;
+  }
+
   /* header line within buffer loop */
   *pconsumed = 0;
   while(blen && k->header) {
     size_t consumed;
+    size_t hlen;
+    char *hd;
 
     end_ptr = memchr(buf, '\n', blen);
     if(!end_ptr) {
@@ -4350,11 +4391,12 @@ static CURLcode http_parse_headers(struct Curl_easy *data,
      * We now have a FULL header line in 'headerb'.
      *****/
 
+    hlen = curlx_dyn_len(&data->state.headerb);
+    hd = curlx_dyn_ptr(&data->state.headerb);
+
     if(!k->headerline) {
-      /* the first read header */
-      statusline st = checkprotoprefix(data, conn,
-                                       curlx_dyn_ptr(&data->state.headerb),
-                                       curlx_dyn_len(&data->state.headerb));
+      /* the first read "header", the status line */
+      statusline st = checkprotoprefix(data, conn, hd, hlen);
       if(st == STATUS_BAD) {
         streamclose(conn, "bad HTTP: No end-of-message indicator");
         /* this is not the beginning of a protocol first header line.
@@ -4372,10 +4414,25 @@ static CURLcode http_parse_headers(struct Curl_easy *data,
         goto out;
       }
     }
+    else {
+      if(hlen && !ISNEWLINE(hd[0])) {
+        /* this is NOT the header separator */
 
-    result = http_rw_hd(data, curlx_dyn_ptr(&data->state.headerb),
-                        curlx_dyn_len(&data->state.headerb),
-                        buf, blen, &consumed);
+        /* if we have bytes for the next header, check for folding */
+        if(blen && ISBLANK(buf[0])) {
+          /* remove the trailing CRLF and append the next header */
+          unfold_header(data);
+          continue;
+        }
+        else if(!blen) {
+          /* this might be a folded header so deal with it in next invoke */
+          data->state.maybe_folded = TRUE;
+          break;
+        }
+      }
+    }
+
+    result = http_rw_hd(data, hd, hlen, buf, blen, &consumed);
     /* We are done with this line. We reset because response
      * processing might switch to HTTP/2 and that might call us
      * directly again. */

lib/urldata.h

@@ -1133,6 +1133,7 @@ struct UrlState {
   BIT(http_hd_te); /* Added HTTP header TE: */
   BIT(http_hd_upgrade); /* Added HTTP header Upgrade: */
   BIT(http_hd_h2_settings); /* Added HTTP header H2Settings: */
+  BIT(maybe_folded);
 #endif
 };
 

tests/data/test1274

@@ -51,14 +51,11 @@ Accept: */*
 <file name="%LOGDIR/out%TESTNUMBER" crlf="yes">
 HTTP/1.1 200 OK
 Date: Tue, 09 Nov 2010 14:49:00 GMT
-Server: test-server/
- fake
- folded
+Server: test-server/ fake folded
 Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
 ETag: "21025-dc7-39462498"
 Content-Length: 6
-Connection:%repeat[46 x  ]%
-   close
+Connection:%repeat[46 x  ]%   close
 
 </file>
 </verify>

tests/data/test1940

@@ -59,7 +59,7 @@ http://%HOSTIP:%HTTPPORT/%TESTNUMBER
 - Set-Cookie == onecookie=data; (0/3)
 - Set-Cookie == secondcookie=2data; (1/3)
 - Set-Cookie == cookie3=data3; (2/3)
- Fold == is folding a line
+ Fold == is folding a        line
  Blank ==%SP
  Blank2 ==%SP
 </stdout>