mirror of
https://github.com/curl/curl.git
synced 2026-04-11 12:01:42 +08:00
async-thrdd: use thread queue for resolving
Use a thread queue and pool for asnyc threaded DNS resolves. Add pytest test_21_* for verification. Add `CURLMOPT_RESOLVE_THREADS_MAX` to allow applications to resize the thread pool used. Add `CURLMOPT_QUICK_EXIT` to allow applications to skip thread joins when cleaning up a multi handle. Multi handles in `curl_easy_perform()` inherit this from `CURLOPT_QUICK_EXIT`. Add several debug environment variables for testing. Closes #20936
This commit is contained in:
parent
507e7be573
commit
39036c9021
@ -100,6 +100,14 @@ Pointer to pass to push callback. See CURLMOPT_PUSHDATA(3)
|
|||||||
|
|
||||||
Callback that approves or denies server pushes. See CURLMOPT_PUSHFUNCTION(3)
|
Callback that approves or denies server pushes. See CURLMOPT_PUSHFUNCTION(3)
|
||||||
|
|
||||||
|
## CURLMOPT_QUICK_EXIT
|
||||||
|
|
||||||
|
Enable a quicker cleanup of the multi handle. See CURLMOPT_QUICK_EXIT(3)
|
||||||
|
|
||||||
|
## CURLMOPT_RESOLVE_THREADS_MAX
|
||||||
|
|
||||||
|
Max threads used for threaded DNS resolver. See CURLMOPT_RESOLVE_THREADS_MAX(3)
|
||||||
|
|
||||||
## CURLMOPT_SOCKETDATA
|
## CURLMOPT_SOCKETDATA
|
||||||
|
|
||||||
Custom pointer passed to the socket callback. See CURLMOPT_SOCKETDATA(3)
|
Custom pointer passed to the socket callback. See CURLMOPT_SOCKETDATA(3)
|
||||||
|
|||||||
@ -173,3 +173,20 @@ Make a blocking, graceful shutdown of all remaining connections when
|
|||||||
a multi handle is destroyed. This implicitly triggers for easy handles
|
a multi handle is destroyed. This implicitly triggers for easy handles
|
||||||
that are run via easy_perform. The value of the environment variable
|
that are run via easy_perform. The value of the environment variable
|
||||||
gives the shutdown timeout in milliseconds.
|
gives the shutdown timeout in milliseconds.
|
||||||
|
|
||||||
|
## `CURL_DBG_RESOLV_MAX_THREADS`
|
||||||
|
|
||||||
|
Overrides the maximum number of threads for resolver.
|
||||||
|
|
||||||
|
## `CURL_DBG_RESOLV_DELAY`
|
||||||
|
|
||||||
|
Makes ever threaded resolve experience an initial delay in milliseconds.
|
||||||
|
|
||||||
|
## `CURL_DBG_RESOLV_FAIL_DELAY`
|
||||||
|
|
||||||
|
With a threaded resolver, delay each lookup by the given milliseconds
|
||||||
|
and give a negative answer.
|
||||||
|
|
||||||
|
## `CURL_DBG_RESOLV_FAIL_IPV6`
|
||||||
|
|
||||||
|
Make libcurl fail a resolve for IPv6 only.
|
||||||
|
|||||||
60
docs/libcurl/opts/CURLMOPT_QUICK_EXIT.md
Normal file
60
docs/libcurl/opts/CURLMOPT_QUICK_EXIT.md
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
---
|
||||||
|
c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||||
|
SPDX-License-Identifier: curl
|
||||||
|
Title: CURLMOPT_QUICK_EXIT
|
||||||
|
Section: 3
|
||||||
|
Source: libcurl
|
||||||
|
See-also:
|
||||||
|
- CURLOPT_QUICK_EXIT (3)
|
||||||
|
Protocol:
|
||||||
|
- All
|
||||||
|
Added-in: 8.20.0
|
||||||
|
---
|
||||||
|
|
||||||
|
# NAME
|
||||||
|
|
||||||
|
CURLOPT_QUICK_EXIT - allow libcurl to exit quickly
|
||||||
|
|
||||||
|
# SYNOPSIS
|
||||||
|
|
||||||
|
~~~c
|
||||||
|
#include <curl/curl.h>
|
||||||
|
|
||||||
|
CURLMcode curl_multi_setopt(CURLM *handle, CURLMOPT_QUICK_EXIT,
|
||||||
|
long value);
|
||||||
|
~~~
|
||||||
|
|
||||||
|
# DESCRIPTION
|
||||||
|
|
||||||
|
Pass a long as a parameter, 1L meaning that when recovering from a timeout,
|
||||||
|
libcurl should skip lengthy cleanups that are intended to avoid all kinds of
|
||||||
|
leaks (threads etc.), as the caller program is about to call exit() anyway.
|
||||||
|
This allows for a swift termination after a DNS timeout for example, by
|
||||||
|
canceling and/or forgetting about a resolver thread, at the expense of a
|
||||||
|
possible (though short-lived) leak of associated resources.
|
||||||
|
|
||||||
|
# DEFAULT
|
||||||
|
|
||||||
|
20.
|
||||||
|
|
||||||
|
# %PROTOCOLS%
|
||||||
|
|
||||||
|
# EXAMPLE
|
||||||
|
|
||||||
|
~~~c
|
||||||
|
int main(void)
|
||||||
|
{
|
||||||
|
CURLM *m = curl_multi_init();
|
||||||
|
/* do not join threads when cleaning up this multi handle */
|
||||||
|
curl_multi_setopt(m, CURLMOPT_QUICK_EXIT, 1L);
|
||||||
|
}
|
||||||
|
~~~
|
||||||
|
|
||||||
|
# %AVAILABILITY%
|
||||||
|
|
||||||
|
# RETURN VALUE
|
||||||
|
|
||||||
|
curl_multi_setopt(3) returns a CURLMcode indicating success or error.
|
||||||
|
|
||||||
|
CURLM_OK (0) means everything was OK, non-zero means an error occurred, see
|
||||||
|
libcurl-errors(3).
|
||||||
75
docs/libcurl/opts/CURLMOPT_RESOLVE_THREADS_MAX.md
Normal file
75
docs/libcurl/opts/CURLMOPT_RESOLVE_THREADS_MAX.md
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
---
|
||||||
|
c: Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||||
|
SPDX-License-Identifier: curl
|
||||||
|
Title: CURLMOPT_RESOLVE_THREADS_MAX
|
||||||
|
Section: 3
|
||||||
|
Source: libcurl
|
||||||
|
See-also:
|
||||||
|
- CURLOPT_IPRESOLVE (3)
|
||||||
|
- CURLOPT_RESOLVE (3)
|
||||||
|
Protocol:
|
||||||
|
- All
|
||||||
|
Added-in: 8.20.0
|
||||||
|
---
|
||||||
|
|
||||||
|
# NAME
|
||||||
|
|
||||||
|
CURLMOPT_RESOLVE_THREADS_MAX - max threads for threaded DNS resolver
|
||||||
|
|
||||||
|
# SYNOPSIS
|
||||||
|
|
||||||
|
~~~c
|
||||||
|
#include <curl/curl.h>
|
||||||
|
|
||||||
|
CURLMcode curl_multi_setopt(CURLM *handle, CURLMOPT_RESOLVE_THREADS_MAX,
|
||||||
|
long amount);
|
||||||
|
~~~
|
||||||
|
|
||||||
|
# DESCRIPTION
|
||||||
|
|
||||||
|
Pass a long for the **amount**. The set number is used as the maximum number
|
||||||
|
of threads to be used for the threaded DNS resolver. It has to be a
|
||||||
|
positive number in the range of 32 bits.
|
||||||
|
|
||||||
|
When libcurl is built with a threaded resolver, which is the default on
|
||||||
|
many systems, it uses a thread pool to lookup addresses and other
|
||||||
|
properties of hostnames so other transfers are not blocked by this.
|
||||||
|
|
||||||
|
Threads are started on demand to perform the resolving and shut down
|
||||||
|
again after a period of inactivity. When the maximum number of threads
|
||||||
|
is reached, outstanding resolves are held in a queue and served when
|
||||||
|
a thread becomes available.
|
||||||
|
|
||||||
|
The default maximum is expected to work fine for many situations. Application
|
||||||
|
may override it using this option for the multi handle.
|
||||||
|
|
||||||
|
Changing this value while there are resolves in progress is possible.
|
||||||
|
Increasing the value takes effect right away. Lowering the value does
|
||||||
|
not close down any resolves, but ends threads above the new maximum
|
||||||
|
once the resolving is done.
|
||||||
|
|
||||||
|
# DEFAULT
|
||||||
|
|
||||||
|
20.
|
||||||
|
|
||||||
|
# %PROTOCOLS%
|
||||||
|
|
||||||
|
# EXAMPLE
|
||||||
|
|
||||||
|
~~~c
|
||||||
|
int main(void)
|
||||||
|
{
|
||||||
|
CURLM *m = curl_multi_init();
|
||||||
|
/* never use more than 5 threads for resolving */
|
||||||
|
curl_multi_setopt(m, CURLMOPT_RESOLVE_THREADS_MAX, 5L);
|
||||||
|
}
|
||||||
|
~~~
|
||||||
|
|
||||||
|
# %AVAILABILITY%
|
||||||
|
|
||||||
|
# RETURN VALUE
|
||||||
|
|
||||||
|
curl_multi_setopt(3) returns a CURLMcode indicating success or error.
|
||||||
|
|
||||||
|
CURLM_OK (0) means everything was OK, non-zero means an error occurred, see
|
||||||
|
libcurl-errors(3).
|
||||||
@ -122,6 +122,8 @@ man_MANS = \
|
|||||||
CURLMOPT_PIPELINING_SITE_BL.3 \
|
CURLMOPT_PIPELINING_SITE_BL.3 \
|
||||||
CURLMOPT_PUSHDATA.3 \
|
CURLMOPT_PUSHDATA.3 \
|
||||||
CURLMOPT_PUSHFUNCTION.3 \
|
CURLMOPT_PUSHFUNCTION.3 \
|
||||||
|
CURLMOPT_QUICK_EXIT.3 \
|
||||||
|
CURLMOPT_RESOLVE_THREADS_MAX.3 \
|
||||||
CURLMOPT_SOCKETDATA.3 \
|
CURLMOPT_SOCKETDATA.3 \
|
||||||
CURLMOPT_SOCKETFUNCTION.3 \
|
CURLMOPT_SOCKETFUNCTION.3 \
|
||||||
CURLMOPT_TIMERDATA.3 \
|
CURLMOPT_TIMERDATA.3 \
|
||||||
|
|||||||
@ -579,6 +579,8 @@ CURLMOPT_PIPELINING_SERVER_BL 7.30.0
|
|||||||
CURLMOPT_PIPELINING_SITE_BL 7.30.0
|
CURLMOPT_PIPELINING_SITE_BL 7.30.0
|
||||||
CURLMOPT_PUSHDATA 7.44.0
|
CURLMOPT_PUSHDATA 7.44.0
|
||||||
CURLMOPT_PUSHFUNCTION 7.44.0
|
CURLMOPT_PUSHFUNCTION 7.44.0
|
||||||
|
CURLMOPT_QUICK_EXIT 8.20.0
|
||||||
|
CURLMOPT_RESOLVE_THREADS_MAX 8.20.0
|
||||||
CURLMOPT_SOCKETDATA 7.15.4
|
CURLMOPT_SOCKETDATA 7.15.4
|
||||||
CURLMOPT_SOCKETFUNCTION 7.15.4
|
CURLMOPT_SOCKETFUNCTION 7.15.4
|
||||||
CURLMOPT_TIMERDATA 7.16.0
|
CURLMOPT_TIMERDATA 7.16.0
|
||||||
|
|||||||
@ -403,6 +403,12 @@ typedef enum {
|
|||||||
/* This is the argument passed to the notify callback */
|
/* This is the argument passed to the notify callback */
|
||||||
CURLOPT(CURLMOPT_NOTIFYDATA, CURLOPTTYPE_OBJECTPOINT, 19),
|
CURLOPT(CURLMOPT_NOTIFYDATA, CURLOPTTYPE_OBJECTPOINT, 19),
|
||||||
|
|
||||||
|
/* maximum number of threads used with threaded DNS resolver */
|
||||||
|
CURLOPT(CURLMOPT_RESOLVE_THREADS_MAX, CURLOPTTYPE_LONG, 20),
|
||||||
|
|
||||||
|
/* set to 1L for not joining threads when multi is cleaned up */
|
||||||
|
CURLOPT(CURLMOPT_QUICK_EXIT, CURLOPTTYPE_LONG, 21),
|
||||||
|
|
||||||
CURLMOPT_LASTENTRY /* the last unused */
|
CURLMOPT_LASTENTRY /* the last unused */
|
||||||
} CURLMoption;
|
} CURLMoption;
|
||||||
|
|
||||||
|
|||||||
906
lib/asyn-thrdd.c
906
lib/asyn-thrdd.c
File diff suppressed because it is too large
Load Diff
51
lib/asyn.h
51
lib/asyn.h
@ -32,6 +32,7 @@
|
|||||||
struct Curl_easy;
|
struct Curl_easy;
|
||||||
struct Curl_dns_entry;
|
struct Curl_dns_entry;
|
||||||
struct Curl_resolv_async;
|
struct Curl_resolv_async;
|
||||||
|
struct Curl_multi;
|
||||||
|
|
||||||
#ifdef CURLRES_ASYNCH
|
#ifdef CURLRES_ASYNCH
|
||||||
|
|
||||||
@ -169,39 +170,12 @@ CURLcode Curl_async_ares_set_dns_local_ip6(struct Curl_easy *data);
|
|||||||
#endif /* USE_RESOLV_ARES */
|
#endif /* USE_RESOLV_ARES */
|
||||||
|
|
||||||
#ifdef USE_RESOLV_THREADED
|
#ifdef USE_RESOLV_THREADED
|
||||||
/* async resolving implementation using POSIX threads */
|
|
||||||
#include "curl_threads.h"
|
|
||||||
|
|
||||||
/* Context for threaded address resolver */
|
struct async_thrdd_item;
|
||||||
struct async_thrdd_addr_ctx {
|
|
||||||
curl_thread_t thread_hnd;
|
|
||||||
char *hostname; /* hostname to resolve, Curl_async.hostname
|
|
||||||
duplicate */
|
|
||||||
curl_mutex_t mutx;
|
|
||||||
#ifndef CURL_DISABLE_SOCKETPAIR
|
|
||||||
curl_socket_t sock_pair[2]; /* eventfd/pipes/socket pair */
|
|
||||||
#endif
|
|
||||||
struct Curl_addrinfo *res;
|
|
||||||
#ifdef HAVE_GETADDRINFO
|
|
||||||
struct addrinfo hints;
|
|
||||||
#endif
|
|
||||||
struct curltime start;
|
|
||||||
timediff_t interval_end;
|
|
||||||
unsigned int poll_interval;
|
|
||||||
int port;
|
|
||||||
int sock_error;
|
|
||||||
int ref_count;
|
|
||||||
BIT(thrd_done);
|
|
||||||
BIT(do_abort);
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Context for threaded resolver */
|
/* Context for threaded resolver */
|
||||||
struct async_thrdd_ctx {
|
struct async_thrdd_ctx {
|
||||||
/* `addr` is a pointer since this memory is shared with a started
|
struct async_thrdd_item *resolved;
|
||||||
* thread. Since threads cannot be killed, we use reference counting
|
|
||||||
* so that we can "release" our pointer to this memory while the
|
|
||||||
* thread is still running. */
|
|
||||||
struct async_thrdd_addr_ctx *addr;
|
|
||||||
#if defined(USE_HTTPSRR) && defined(USE_ARES)
|
#if defined(USE_HTTPSRR) && defined(USE_ARES)
|
||||||
struct {
|
struct {
|
||||||
ares_channel channel;
|
ares_channel channel;
|
||||||
@ -210,6 +184,8 @@ struct async_thrdd_ctx {
|
|||||||
BIT(done);
|
BIT(done);
|
||||||
} rr;
|
} rr;
|
||||||
#endif
|
#endif
|
||||||
|
BIT(queued);
|
||||||
|
BIT(done);
|
||||||
};
|
};
|
||||||
|
|
||||||
void Curl_async_thrdd_shutdown(struct Curl_easy *data,
|
void Curl_async_thrdd_shutdown(struct Curl_easy *data,
|
||||||
@ -217,6 +193,18 @@ void Curl_async_thrdd_shutdown(struct Curl_easy *data,
|
|||||||
void Curl_async_thrdd_destroy(struct Curl_easy *data,
|
void Curl_async_thrdd_destroy(struct Curl_easy *data,
|
||||||
struct Curl_resolv_async *async);
|
struct Curl_resolv_async *async);
|
||||||
|
|
||||||
|
CURLcode Curl_async_thrdd_multi_init(struct Curl_multi *multi,
|
||||||
|
uint32_t min_threads,
|
||||||
|
uint32_t max_threads,
|
||||||
|
uint32_t idle_time_ms);
|
||||||
|
void Curl_async_thrdd_multi_destroy(struct Curl_multi *multi, bool join);
|
||||||
|
void Curl_async_thrdd_multi_process(struct Curl_multi *multi);
|
||||||
|
|
||||||
|
CURLcode Curl_async_thrdd_multi_set_props(struct Curl_multi *multi,
|
||||||
|
uint32_t min_threads,
|
||||||
|
uint32_t max_threads,
|
||||||
|
uint32_t idle_time_ms);
|
||||||
|
|
||||||
#endif /* USE_RESOLV_THREADED */
|
#endif /* USE_RESOLV_THREADED */
|
||||||
|
|
||||||
#ifndef CURL_DISABLE_DOH
|
#ifndef CURL_DISABLE_DOH
|
||||||
@ -248,10 +236,15 @@ struct Curl_resolv_async {
|
|||||||
#ifndef CURL_DISABLE_DOH
|
#ifndef CURL_DISABLE_DOH
|
||||||
struct doh_probes *doh; /* DoH specific data for this request */
|
struct doh_probes *doh; /* DoH specific data for this request */
|
||||||
#endif
|
#endif
|
||||||
|
struct curltime start;
|
||||||
|
timediff_t interval_end;
|
||||||
|
timediff_t timeout_ms;
|
||||||
|
uint32_t poll_interval;
|
||||||
uint32_t id; /* unique id per easy handle of the resolve operation */
|
uint32_t id; /* unique id per easy handle of the resolve operation */
|
||||||
/* what is being resolved */
|
/* what is being resolved */
|
||||||
uint16_t port;
|
uint16_t port;
|
||||||
uint8_t ip_version;
|
uint8_t ip_version;
|
||||||
|
uint8_t transport;
|
||||||
char hostname[1];
|
char hostname[1];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -530,7 +530,8 @@ CURLcode Curl_parse_interface(const char *input,
|
|||||||
|
|
||||||
#ifndef CURL_DISABLE_BINDLOCAL
|
#ifndef CURL_DISABLE_BINDLOCAL
|
||||||
static CURLcode bindlocal(struct Curl_easy *data, struct connectdata *conn,
|
static CURLcode bindlocal(struct Curl_easy *data, struct connectdata *conn,
|
||||||
curl_socket_t sockfd, int af, unsigned int scope)
|
curl_socket_t sockfd, int af, unsigned int scope,
|
||||||
|
uint8_t transport)
|
||||||
{
|
{
|
||||||
struct Curl_sockaddr_storage sa;
|
struct Curl_sockaddr_storage sa;
|
||||||
struct sockaddr *sock = (struct sockaddr *)&sa; /* bind to this address */
|
struct sockaddr *sock = (struct sockaddr *)&sa; /* bind to this address */
|
||||||
@ -648,7 +649,7 @@ static CURLcode bindlocal(struct Curl_easy *data, struct connectdata *conn,
|
|||||||
ip_version = CURL_IPRESOLVE_V6;
|
ip_version = CURL_IPRESOLVE_V6;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
(void)Curl_resolv_blocking(data, host, 80, ip_version, &h);
|
(void)Curl_resolv_blocking(data, host, 80, ip_version, transport, &h);
|
||||||
if(h) {
|
if(h) {
|
||||||
int h_af = h->addr->ai_family;
|
int h_af = h->addr->ai_family;
|
||||||
/* convert the resolved address, sizeof myhost >= INET_ADDRSTRLEN */
|
/* convert the resolved address, sizeof myhost >= INET_ADDRSTRLEN */
|
||||||
@ -1143,7 +1144,8 @@ static CURLcode cf_socket_open(struct Curl_cfilter *cf,
|
|||||||
#endif
|
#endif
|
||||||
) {
|
) {
|
||||||
result = bindlocal(data, cf->conn, ctx->sock, ctx->addr.family,
|
result = bindlocal(data, cf->conn, ctx->sock, ctx->addr.family,
|
||||||
Curl_ipv6_scope(&ctx->addr.curl_sa_addr));
|
Curl_ipv6_scope(&ctx->addr.curl_sa_addr),
|
||||||
|
ctx->transport);
|
||||||
if(result) {
|
if(result) {
|
||||||
if(result == CURLE_UNSUPPORTED_PROTOCOL) {
|
if(result == CURLE_UNSUPPORTED_PROTOCOL) {
|
||||||
/* The address family is not supported on this interface.
|
/* The address family is not supported on this interface.
|
||||||
|
|||||||
@ -700,6 +700,18 @@ unsigned char Curl_conn_get_transport(struct Curl_easy *data,
|
|||||||
return Curl_conn_cf_get_transport(cf, data);
|
return Curl_conn_cf_get_transport(cf, data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int Curl_socktype_for_transport(uint8_t transport)
|
||||||
|
{
|
||||||
|
switch(transport) {
|
||||||
|
case TRNSPRT_TCP:
|
||||||
|
return SOCK_STREAM;
|
||||||
|
case TRNSPRT_UNIX:
|
||||||
|
return SOCK_STREAM;
|
||||||
|
default: /* UDP and QUIC */
|
||||||
|
return SOCK_DGRAM;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const char *Curl_conn_get_alpn_negotiated(struct Curl_easy *data,
|
const char *Curl_conn_get_alpn_negotiated(struct Curl_easy *data,
|
||||||
struct connectdata *conn)
|
struct connectdata *conn)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -345,6 +345,8 @@ bool Curl_conn_cf_needs_flush(struct Curl_cfilter *cf,
|
|||||||
unsigned char Curl_conn_cf_get_transport(struct Curl_cfilter *cf,
|
unsigned char Curl_conn_cf_get_transport(struct Curl_cfilter *cf,
|
||||||
struct Curl_easy *data);
|
struct Curl_easy *data);
|
||||||
|
|
||||||
|
int Curl_socktype_for_transport(uint8_t transport);
|
||||||
|
|
||||||
const char *Curl_conn_cf_get_alpn_negotiated(struct Curl_cfilter *cf,
|
const char *Curl_conn_cf_get_alpn_negotiated(struct Curl_cfilter *cf,
|
||||||
struct Curl_easy *data);
|
struct Curl_easy *data);
|
||||||
|
|
||||||
|
|||||||
@ -779,8 +779,9 @@ static CURLcode easy_perform(struct Curl_easy *data, bool events)
|
|||||||
if(multi->in_callback)
|
if(multi->in_callback)
|
||||||
return CURLE_RECURSIVE_API_CALL;
|
return CURLE_RECURSIVE_API_CALL;
|
||||||
|
|
||||||
/* Copy the MAXCONNECTS option to the multi handle */
|
/* Copy relevant easy options to the multi handle */
|
||||||
curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, (long)data->set.maxconnects);
|
curl_multi_setopt(multi, CURLMOPT_MAXCONNECTS, (long)data->set.maxconnects);
|
||||||
|
curl_multi_setopt(multi, CURLMOPT_QUICK_EXIT, (long)data->set.quick_exit);
|
||||||
|
|
||||||
data->multi_easy = NULL; /* pretend it does not exist */
|
data->multi_easy = NULL; /* pretend it does not exist */
|
||||||
mresult = curl_multi_add_handle(multi, data);
|
mresult = curl_multi_add_handle(multi, data);
|
||||||
|
|||||||
@ -1063,6 +1063,7 @@ static CURLcode ftp_port_resolve_host(struct Curl_easy *data,
|
|||||||
|
|
||||||
*resp = NULL;
|
*resp = NULL;
|
||||||
result = Curl_resolv_blocking(data, host, 0, conn->ip_version,
|
result = Curl_resolv_blocking(data, host, 0, conn->ip_version,
|
||||||
|
Curl_conn_get_transport(data, conn),
|
||||||
dns_entryp);
|
dns_entryp);
|
||||||
if(result)
|
if(result)
|
||||||
failf(data, "failed to resolve the address provided to PORT: %s", host);
|
failf(data, "failed to resolve the address provided to PORT: %s", host);
|
||||||
@ -2163,6 +2164,7 @@ static CURLcode ftp_state_pasv_resp(struct Curl_easy *data,
|
|||||||
|
|
||||||
(void)Curl_resolv_blocking(data, host_name, ipquad.remote_port,
|
(void)Curl_resolv_blocking(data, host_name, ipquad.remote_port,
|
||||||
is_ipv6 ? CURL_IPRESOLVE_V6 : CURL_IPRESOLVE_V4,
|
is_ipv6 ? CURL_IPRESOLVE_V6 : CURL_IPRESOLVE_V4,
|
||||||
|
Curl_conn_get_transport(data, conn),
|
||||||
&dns);
|
&dns);
|
||||||
/* we connect to the proxy's port */
|
/* we connect to the proxy's port */
|
||||||
connectport = (unsigned short)ipquad.remote_port;
|
connectport = (unsigned short)ipquad.remote_port;
|
||||||
@ -2187,7 +2189,9 @@ static CURLcode ftp_state_pasv_resp(struct Curl_easy *data,
|
|||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
|
||||||
(void)Curl_resolv_blocking(data, newhost, newport, conn->ip_version, &dns);
|
(void)Curl_resolv_blocking(data, newhost, newport, conn->ip_version,
|
||||||
|
Curl_conn_get_transport(data, conn),
|
||||||
|
&dns);
|
||||||
connectport = newport; /* we connect to the remote port */
|
connectport = newport; /* we connect to the remote port */
|
||||||
|
|
||||||
if(!dns) {
|
if(!dns) {
|
||||||
|
|||||||
54
lib/hostip.c
54
lib/hostip.c
@ -373,7 +373,9 @@ static bool can_resolve_ip_version(struct Curl_easy *data, int ip_version)
|
|||||||
static CURLcode hostip_async_new(struct Curl_easy *data,
|
static CURLcode hostip_async_new(struct Curl_easy *data,
|
||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version)
|
uint8_t ip_version,
|
||||||
|
uint8_t transport,
|
||||||
|
timediff_t timeout_ms)
|
||||||
{
|
{
|
||||||
struct Curl_resolv_async *async;
|
struct Curl_resolv_async *async;
|
||||||
size_t hostlen = strlen(hostname);
|
size_t hostlen = strlen(hostname);
|
||||||
@ -389,6 +391,9 @@ static CURLcode hostip_async_new(struct Curl_easy *data,
|
|||||||
async->id = data->state.next_async_id++;
|
async->id = data->state.next_async_id++;
|
||||||
async->port = port;
|
async->port = port;
|
||||||
async->ip_version = ip_version;
|
async->ip_version = ip_version;
|
||||||
|
async->transport = transport;
|
||||||
|
async->start = *Curl_pgrs_now(data);
|
||||||
|
async->timeout_ms = timeout_ms;
|
||||||
if(hostlen)
|
if(hostlen)
|
||||||
memcpy(async->hostname, hostname, hostlen);
|
memcpy(async->hostname, hostname, hostlen);
|
||||||
|
|
||||||
@ -401,6 +406,8 @@ static CURLcode hostip_resolv(struct Curl_easy *data,
|
|||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version,
|
uint8_t ip_version,
|
||||||
|
uint8_t transport,
|
||||||
|
timediff_t timeout_ms,
|
||||||
bool allowDOH,
|
bool allowDOH,
|
||||||
struct Curl_dns_entry **entry)
|
struct Curl_dns_entry **entry)
|
||||||
{
|
{
|
||||||
@ -415,6 +422,8 @@ static CURLcode hostip_resolv(struct Curl_easy *data,
|
|||||||
#ifdef USE_CURL_ASYNC
|
#ifdef USE_CURL_ASYNC
|
||||||
if(data->state.async)
|
if(data->state.async)
|
||||||
Curl_async_destroy(data);
|
Curl_async_destroy(data);
|
||||||
|
#else
|
||||||
|
(void)timeout_ms;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifndef CURL_DISABLE_DOH
|
#ifndef CURL_DISABLE_DOH
|
||||||
@ -463,7 +472,8 @@ static CURLcode hostip_resolv(struct Curl_easy *data,
|
|||||||
int st;
|
int st;
|
||||||
#ifdef CURLRES_ASYNCH
|
#ifdef CURLRES_ASYNCH
|
||||||
if(!data->state.async) {
|
if(!data->state.async) {
|
||||||
result = hostip_async_new(data, hostname, port, ip_version);
|
result = hostip_async_new(data, hostname, port, ip_version,
|
||||||
|
transport, timeout_ms);
|
||||||
if(result)
|
if(result)
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
@ -501,7 +511,8 @@ static CURLcode hostip_resolv(struct Curl_easy *data,
|
|||||||
#ifndef CURL_DISABLE_DOH
|
#ifndef CURL_DISABLE_DOH
|
||||||
else if(!Curl_is_ipaddr(hostname) && allowDOH && data->set.doh) {
|
else if(!Curl_is_ipaddr(hostname) && allowDOH && data->set.doh) {
|
||||||
if(!data->state.async) {
|
if(!data->state.async) {
|
||||||
result = hostip_async_new(data, hostname, port, ip_version);
|
result = hostip_async_new(data, hostname, port, ip_version,
|
||||||
|
transport, timeout_ms);
|
||||||
if(result)
|
if(result)
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
@ -518,7 +529,8 @@ static CURLcode hostip_resolv(struct Curl_easy *data,
|
|||||||
|
|
||||||
#ifdef CURLRES_ASYNCH
|
#ifdef CURLRES_ASYNCH
|
||||||
if(!data->state.async) {
|
if(!data->state.async) {
|
||||||
result = hostip_async_new(data, hostname, port, ip_version);
|
result = hostip_async_new(data, hostname, port, ip_version,
|
||||||
|
transport, timeout_ms);
|
||||||
if(result)
|
if(result)
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
@ -526,7 +538,7 @@ static CURLcode hostip_resolv(struct Curl_easy *data,
|
|||||||
respwait = TRUE;
|
respwait = TRUE;
|
||||||
#else
|
#else
|
||||||
respwait = FALSE; /* no async waiting here */
|
respwait = FALSE; /* no async waiting here */
|
||||||
addr = Curl_sync_getaddrinfo(data, hostname, port, ip_version);
|
addr = Curl_sync_getaddrinfo(data, hostname, port, ip_version, transport);
|
||||||
if(addr)
|
if(addr)
|
||||||
result = CURLE_OK;
|
result = CURLE_OK;
|
||||||
#endif
|
#endif
|
||||||
@ -577,13 +589,15 @@ CURLcode Curl_resolv_blocking(struct Curl_easy *data,
|
|||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version,
|
uint8_t ip_version,
|
||||||
|
uint8_t transport,
|
||||||
struct Curl_dns_entry **pdns)
|
struct Curl_dns_entry **pdns)
|
||||||
{
|
{
|
||||||
CURLcode result;
|
CURLcode result;
|
||||||
DEBUGASSERT(hostname && *hostname);
|
DEBUGASSERT(hostname && *hostname);
|
||||||
*pdns = NULL;
|
*pdns = NULL;
|
||||||
/* We cannot do a blocking resolve using DoH currently */
|
/* We cannot do a blocking resolve using DoH currently */
|
||||||
result = hostip_resolv(data, hostname, port, ip_version, FALSE, pdns);
|
result = hostip_resolv(data, hostname, port, ip_version,
|
||||||
|
transport, 0, FALSE, pdns);
|
||||||
switch(result) {
|
switch(result) {
|
||||||
case CURLE_OK:
|
case CURLE_OK:
|
||||||
DEBUGASSERT(*pdns);
|
DEBUGASSERT(*pdns);
|
||||||
@ -619,7 +633,8 @@ static CURLcode resolv_alarm_timeout(struct Curl_easy *data,
|
|||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version,
|
uint8_t ip_version,
|
||||||
timediff_t timeoutms,
|
uint8_t transport,
|
||||||
|
timediff_t timeout_ms,
|
||||||
struct Curl_dns_entry **entry)
|
struct Curl_dns_entry **entry)
|
||||||
{
|
{
|
||||||
#ifdef HAVE_SIGACTION
|
#ifdef HAVE_SIGACTION
|
||||||
@ -636,14 +651,14 @@ static CURLcode resolv_alarm_timeout(struct Curl_easy *data,
|
|||||||
CURLcode result;
|
CURLcode result;
|
||||||
|
|
||||||
DEBUGASSERT(hostname && *hostname);
|
DEBUGASSERT(hostname && *hostname);
|
||||||
DEBUGASSERT(timeoutms > 0);
|
DEBUGASSERT(timeout_ms > 0);
|
||||||
DEBUGASSERT(!data->set.no_signal);
|
DEBUGASSERT(!data->set.no_signal);
|
||||||
#ifndef CURL_DISABLE_DOH
|
#ifndef CURL_DISABLE_DOH
|
||||||
DEBUGASSERT(!data->set.doh);
|
DEBUGASSERT(!data->set.doh);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
*entry = NULL;
|
*entry = NULL;
|
||||||
timeout = (timeoutms > LONG_MAX) ? LONG_MAX : (long)timeoutms;
|
timeout = (timeout_ms > LONG_MAX) ? LONG_MAX : (long)timeout_ms;
|
||||||
if(timeout < 1000) {
|
if(timeout < 1000) {
|
||||||
/* The alarm() function only provides integer second resolution, so if
|
/* The alarm() function only provides integer second resolution, so if
|
||||||
we want to wait less than one second we must bail out already now. */
|
we want to wait less than one second we must bail out already now. */
|
||||||
@ -696,7 +711,8 @@ static CURLcode resolv_alarm_timeout(struct Curl_easy *data,
|
|||||||
|
|
||||||
/* Perform the actual name resolution. This might be interrupted by an
|
/* Perform the actual name resolution. This might be interrupted by an
|
||||||
* alarm if it takes too long. */
|
* alarm if it takes too long. */
|
||||||
result = hostip_resolv(data, hostname, port, ip_version, TRUE, entry);
|
result = hostip_resolv(data, hostname, port, ip_version, transport,
|
||||||
|
timeout_ms, TRUE, entry);
|
||||||
|
|
||||||
clean_up:
|
clean_up:
|
||||||
if(!prev_alarm)
|
if(!prev_alarm)
|
||||||
@ -772,33 +788,35 @@ CURLcode Curl_resolv(struct Curl_easy *data,
|
|||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version,
|
uint8_t ip_version,
|
||||||
timediff_t timeoutms,
|
uint8_t transport,
|
||||||
|
timediff_t timeout_ms,
|
||||||
struct Curl_dns_entry **entry)
|
struct Curl_dns_entry **entry)
|
||||||
{
|
{
|
||||||
DEBUGASSERT(hostname && *hostname);
|
DEBUGASSERT(hostname && *hostname);
|
||||||
*entry = NULL;
|
*entry = NULL;
|
||||||
|
|
||||||
if(timeoutms < 0)
|
if(timeout_ms < 0)
|
||||||
/* got an already expired timeout */
|
/* got an already expired timeout */
|
||||||
return CURLE_OPERATION_TIMEDOUT;
|
return CURLE_OPERATION_TIMEDOUT;
|
||||||
|
|
||||||
#ifdef USE_ALARM_TIMEOUT
|
#ifdef USE_ALARM_TIMEOUT
|
||||||
if(timeoutms && data->set.no_signal) {
|
if(timeout_ms && data->set.no_signal) {
|
||||||
/* Cannot use ALARM when signals are disabled */
|
/* Cannot use ALARM when signals are disabled */
|
||||||
timeoutms = 0;
|
timeout_ms = 0;
|
||||||
}
|
}
|
||||||
if(timeoutms && !Curl_doh_wanted(data)) {
|
if(timeout_ms && !Curl_doh_wanted(data)) {
|
||||||
return resolv_alarm_timeout(data, hostname, port, ip_version,
|
return resolv_alarm_timeout(data, hostname, port, ip_version,
|
||||||
timeoutms, entry);
|
transport, timeout_ms, entry);
|
||||||
}
|
}
|
||||||
#endif /* !USE_ALARM_TIMEOUT */
|
#endif /* !USE_ALARM_TIMEOUT */
|
||||||
|
|
||||||
#ifndef CURLRES_ASYNCH
|
#ifndef CURLRES_ASYNCH
|
||||||
if(timeoutms)
|
if(timeout_ms)
|
||||||
infof(data, "timeout on name lookup is not supported");
|
infof(data, "timeout on name lookup is not supported");
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
return hostip_resolv(data, hostname, port, ip_version, TRUE, entry);
|
return hostip_resolv(data, hostname, port, ip_version, transport,
|
||||||
|
timeout_ms, TRUE, entry);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -91,13 +91,15 @@ CURLcode Curl_resolv(struct Curl_easy *data,
|
|||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version,
|
uint8_t ip_version,
|
||||||
timediff_t timeoutms,
|
uint8_t transport,
|
||||||
|
timediff_t timeout_ms,
|
||||||
struct Curl_dns_entry **pdns);
|
struct Curl_dns_entry **pdns);
|
||||||
|
|
||||||
CURLcode Curl_resolv_blocking(struct Curl_easy *data,
|
CURLcode Curl_resolv_blocking(struct Curl_easy *data,
|
||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version,
|
uint8_t ip_version,
|
||||||
|
uint8_t transport,
|
||||||
struct Curl_dns_entry **pdns);
|
struct Curl_dns_entry **pdns);
|
||||||
|
|
||||||
CURLcode Curl_resolv_timeout(struct Curl_easy *data,
|
CURLcode Curl_resolv_timeout(struct Curl_easy *data,
|
||||||
@ -127,7 +129,8 @@ CURLcode Curl_resolver_error(struct Curl_easy *data, const char *detail);
|
|||||||
struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version);
|
uint8_t ip_version,
|
||||||
|
uint8_t transport);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|||||||
@ -70,11 +70,13 @@
|
|||||||
struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version)
|
uint8_t ip_version,
|
||||||
|
uint8_t transport)
|
||||||
{
|
{
|
||||||
struct Curl_addrinfo *ai = NULL;
|
struct Curl_addrinfo *ai = NULL;
|
||||||
|
|
||||||
(void)ip_version;
|
(void)ip_version;
|
||||||
|
(void)transport;
|
||||||
|
|
||||||
ai = Curl_ipv4_resolve_r(hostname, port);
|
ai = Curl_ipv4_resolve_r(hostname, port);
|
||||||
if(!ai)
|
if(!ai)
|
||||||
|
|||||||
@ -65,7 +65,8 @@
|
|||||||
struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
||||||
const char *hostname,
|
const char *hostname,
|
||||||
uint16_t port,
|
uint16_t port,
|
||||||
uint8_t ip_version)
|
uint8_t ip_version,
|
||||||
|
uint8_t transport)
|
||||||
{
|
{
|
||||||
struct addrinfo hints;
|
struct addrinfo hints;
|
||||||
struct Curl_addrinfo *res;
|
struct Curl_addrinfo *res;
|
||||||
@ -83,8 +84,7 @@ struct Curl_addrinfo *Curl_sync_getaddrinfo(struct Curl_easy *data,
|
|||||||
|
|
||||||
memset(&hints, 0, sizeof(hints));
|
memset(&hints, 0, sizeof(hints));
|
||||||
hints.ai_family = pf;
|
hints.ai_family = pf;
|
||||||
hints.ai_socktype =
|
hints.ai_socktype = (transport == TRNSPRT_TCP) ?
|
||||||
(Curl_conn_get_transport(data, data->conn) == TRNSPRT_TCP) ?
|
|
||||||
SOCK_STREAM : SOCK_DGRAM;
|
SOCK_STREAM : SOCK_DGRAM;
|
||||||
|
|
||||||
#ifndef USE_RESOLVE_ON_IPS
|
#ifndef USE_RESOLVE_ON_IPS
|
||||||
|
|||||||
48
lib/multi.c
48
lib/multi.c
@ -307,10 +307,24 @@ struct Curl_multi *Curl_multi_handle(uint32_t xfer_table_size,
|
|||||||
goto error;
|
goto error;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef USE_RESOLV_THREADED
|
||||||
|
if(xfer_table_size < CURL_XFER_TABLE_SIZE) { /* easy multi */
|
||||||
|
if(Curl_async_thrdd_multi_init(multi, 0, 2, 10))
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
else { /* real multi handle */
|
||||||
|
if(Curl_async_thrdd_multi_init(multi, 0, 20, 2000))
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
return multi;
|
return multi;
|
||||||
|
|
||||||
error:
|
error:
|
||||||
|
|
||||||
|
#ifdef USE_RESOLV_THREADED
|
||||||
|
Curl_async_thrdd_multi_destroy(multi, TRUE);
|
||||||
|
#endif
|
||||||
Curl_multi_ev_cleanup(multi);
|
Curl_multi_ev_cleanup(multi);
|
||||||
Curl_hash_destroy(&multi->proto_hash);
|
Curl_hash_destroy(&multi->proto_hash);
|
||||||
Curl_dnscache_destroy(&multi->dnscache);
|
Curl_dnscache_destroy(&multi->dnscache);
|
||||||
@ -2522,6 +2536,9 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi,
|
|||||||
Curl_uint32_bset_remove(&multi->dirty, data->mid);
|
Curl_uint32_bset_remove(&multi->dirty, data->mid);
|
||||||
|
|
||||||
if(data == multi->admin) {
|
if(data == multi->admin) {
|
||||||
|
#ifdef USE_RESOLV_THREADED
|
||||||
|
Curl_async_thrdd_multi_process(multi);
|
||||||
|
#endif
|
||||||
Curl_cshutdn_perform(&multi->cshutdn, multi->admin, sigpipe_ctx);
|
Curl_cshutdn_perform(&multi->cshutdn, multi->admin, sigpipe_ctx);
|
||||||
return CURLM_OK;
|
return CURLM_OK;
|
||||||
}
|
}
|
||||||
@ -2951,6 +2968,9 @@ CURLMcode curl_multi_cleanup(CURLM *m)
|
|||||||
} while(Curl_uint32_tbl_next(&multi->xfers, mid, &mid, &entry));
|
} while(Curl_uint32_tbl_next(&multi->xfers, mid, &mid, &entry));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef USE_RESOLV_THREADED
|
||||||
|
Curl_async_thrdd_multi_destroy(multi, !multi->quick_exit);
|
||||||
|
#endif
|
||||||
Curl_cpool_destroy(&multi->cpool);
|
Curl_cpool_destroy(&multi->cpool);
|
||||||
Curl_cshutdn_destroy(&multi->cshutdn, multi->admin);
|
Curl_cshutdn_destroy(&multi->cshutdn, multi->admin);
|
||||||
if(multi->admin) {
|
if(multi->admin) {
|
||||||
@ -3347,6 +3367,34 @@ CURLMcode curl_multi_setopt(CURLM *m, CURLMoption option, ...)
|
|||||||
case CURLMOPT_NOTIFYDATA:
|
case CURLMOPT_NOTIFYDATA:
|
||||||
multi->ntfy.ntfy_cb_data = va_arg(param, void *);
|
multi->ntfy.ntfy_cb_data = va_arg(param, void *);
|
||||||
break;
|
break;
|
||||||
|
case CURLMOPT_RESOLVE_THREADS_MAX:
|
||||||
|
#ifdef USE_RESOLV_THREADED
|
||||||
|
uarg = va_arg(param, long);
|
||||||
|
if((uarg <= 0) || (uarg > UINT32_MAX))
|
||||||
|
mresult = CURLM_BAD_FUNCTION_ARGUMENT;
|
||||||
|
else {
|
||||||
|
CURLcode result = Curl_async_thrdd_multi_set_props(
|
||||||
|
multi, 0, (uint32_t)uarg, 2000);
|
||||||
|
switch(result) {
|
||||||
|
case CURLE_OK:
|
||||||
|
mresult = CURLM_OK;
|
||||||
|
break;
|
||||||
|
case CURLE_BAD_FUNCTION_ARGUMENT:
|
||||||
|
mresult = CURLM_BAD_FUNCTION_ARGUMENT;
|
||||||
|
break;
|
||||||
|
case CURLE_OUT_OF_MEMORY:
|
||||||
|
mresult = CURLM_OUT_OF_MEMORY;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
mresult = CURLM_INTERNAL_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
break;
|
||||||
|
case CURLMOPT_QUICK_EXIT:
|
||||||
|
multi->quick_exit = va_arg(param, long) ? 1 : 0;
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
mresult = CURLM_UNKNOWN_OPTION;
|
mresult = CURLM_UNKNOWN_OPTION;
|
||||||
break;
|
break;
|
||||||
|
|||||||
@ -109,6 +109,9 @@ struct Curl_multi {
|
|||||||
|
|
||||||
struct Curl_dnscache dnscache; /* DNS cache */
|
struct Curl_dnscache dnscache; /* DNS cache */
|
||||||
struct Curl_ssl_scache *ssl_scache; /* TLS session pool */
|
struct Curl_ssl_scache *ssl_scache; /* TLS session pool */
|
||||||
|
#ifdef USE_RESOLV_THREADED
|
||||||
|
struct curl_thrdq *resolv_thrdq;
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef USE_LIBPSL
|
#ifdef USE_LIBPSL
|
||||||
/* PSL cache. */
|
/* PSL cache. */
|
||||||
@ -186,6 +189,7 @@ struct Curl_multi {
|
|||||||
BIT(xfer_buf_borrowed); /* xfer_buf is currently being borrowed */
|
BIT(xfer_buf_borrowed); /* xfer_buf is currently being borrowed */
|
||||||
BIT(xfer_ulbuf_borrowed); /* xfer_ulbuf is currently being borrowed */
|
BIT(xfer_ulbuf_borrowed); /* xfer_ulbuf is currently being borrowed */
|
||||||
BIT(xfer_sockbuf_borrowed); /* xfer_sockbuf is currently being borrowed */
|
BIT(xfer_sockbuf_borrowed); /* xfer_sockbuf is currently being borrowed */
|
||||||
|
BIT(quick_exit); /* do not join threads on cleanup */
|
||||||
#ifdef DEBUGBUILD
|
#ifdef DEBUGBUILD
|
||||||
BIT(warned); /* true after user warned of DEBUGBUILD */
|
BIT(warned); /* true after user warned of DEBUGBUILD */
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@ -324,7 +324,9 @@ static CURLproxycode socks4_resolving(struct socks_state *sx,
|
|||||||
DEBUGASSERT(sx->hostname && *sx->hostname);
|
DEBUGASSERT(sx->hostname && *sx->hostname);
|
||||||
|
|
||||||
result = Curl_resolv(data, sx->hostname, sx->remote_port,
|
result = Curl_resolv(data, sx->hostname, sx->remote_port,
|
||||||
cf->conn->ip_version, 0, &dns);
|
cf->conn->ip_version,
|
||||||
|
Curl_conn_cf_get_transport(cf, data),
|
||||||
|
0, &dns);
|
||||||
if(result == CURLE_AGAIN) {
|
if(result == CURLE_AGAIN) {
|
||||||
CURL_TRC_CF(data, cf, "SOCKS4 non-blocking resolve of %s", sx->hostname);
|
CURL_TRC_CF(data, cf, "SOCKS4 non-blocking resolve of %s", sx->hostname);
|
||||||
return CURLPX_OK;
|
return CURLPX_OK;
|
||||||
@ -853,7 +855,9 @@ static CURLproxycode socks5_resolving(struct socks_state *sx,
|
|||||||
DEBUGASSERT(sx->hostname && *sx->hostname);
|
DEBUGASSERT(sx->hostname && *sx->hostname);
|
||||||
|
|
||||||
result = Curl_resolv(data, sx->hostname, sx->remote_port,
|
result = Curl_resolv(data, sx->hostname, sx->remote_port,
|
||||||
cf->conn->ip_version, 0, &dns);
|
cf->conn->ip_version,
|
||||||
|
Curl_conn_cf_get_transport(cf, data),
|
||||||
|
0, &dns);
|
||||||
if(result == CURLE_AGAIN) {
|
if(result == CURLE_AGAIN) {
|
||||||
CURL_TRC_CF(data, cf, "SOCKS5 non-blocking resolve of %s", sx->hostname);
|
CURL_TRC_CF(data, cf, "SOCKS5 non-blocking resolve of %s", sx->hostname);
|
||||||
return CURLPX_OK;
|
return CURLPX_OK;
|
||||||
|
|||||||
@ -121,7 +121,8 @@ static CURL_THREAD_RETURN_T CURL_STDCALL thrdslot_run(void *arg)
|
|||||||
tpool->fn_return(item, tpool->aborted ? NULL : tpool->fn_user_data);
|
tpool->fn_return(item, tpool->aborted ? NULL : tpool->fn_user_data);
|
||||||
}
|
}
|
||||||
|
|
||||||
if(tpool->aborted)
|
if(tpool->aborted ||
|
||||||
|
(Curl_llist_count(&tpool->slots) > tpool->max_threads))
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
tslot->idle = TRUE;
|
tslot->idle = TRUE;
|
||||||
@ -235,6 +236,63 @@ static bool thrdpool_unlink(struct curl_thrdpool *tpool, bool locked)
|
|||||||
return TRUE;
|
return TRUE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static CURLcode thrdpool_signal(struct curl_thrdpool *tpool,
|
||||||
|
uint32_t nthreads)
|
||||||
|
{
|
||||||
|
struct Curl_llist_node *e, *n;
|
||||||
|
CURLcode result = CURLE_OK;
|
||||||
|
|
||||||
|
DEBUGASSERT(!tpool->aborted);
|
||||||
|
thrdpool_join_zombies(tpool);
|
||||||
|
|
||||||
|
for(e = Curl_llist_head(&tpool->slots); e && nthreads; e = n) {
|
||||||
|
struct thrdslot *tslot = Curl_node_elem(e);
|
||||||
|
n = Curl_node_next(e);
|
||||||
|
if(tslot->idle) {
|
||||||
|
Curl_cond_signal(&tslot->await);
|
||||||
|
--nthreads;
|
||||||
|
}
|
||||||
|
else if(!tslot->starttime.tv_sec && !tslot->starttime.tv_usec) {
|
||||||
|
/* starting thread, queries for work soon. */
|
||||||
|
--nthreads;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while(nthreads && !result &&
|
||||||
|
Curl_llist_count(&tpool->slots) < tpool->max_threads) {
|
||||||
|
result = thrdslot_start(tpool);
|
||||||
|
if(result)
|
||||||
|
break;
|
||||||
|
--nthreads;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
CURLcode Curl_thrdpool_set_props(struct curl_thrdpool *tpool,
|
||||||
|
uint32_t min_threads,
|
||||||
|
uint32_t max_threads,
|
||||||
|
uint32_t idle_time_ms)
|
||||||
|
{
|
||||||
|
CURLcode result = CURLE_OK;
|
||||||
|
size_t running;
|
||||||
|
|
||||||
|
if(!max_threads || (min_threads > max_threads))
|
||||||
|
return CURLE_BAD_FUNCTION_ARGUMENT;
|
||||||
|
|
||||||
|
Curl_mutex_acquire(&tpool->lock);
|
||||||
|
tpool->min_threads = min_threads;
|
||||||
|
tpool->max_threads = max_threads;
|
||||||
|
tpool->idle_time_ms = idle_time_ms;
|
||||||
|
running = Curl_llist_count(&tpool->slots);
|
||||||
|
if(tpool->min_threads > running) {
|
||||||
|
result = thrdpool_signal(tpool, tpool->min_threads - (uint32_t)running);
|
||||||
|
}
|
||||||
|
Curl_mutex_release(&tpool->lock);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
CURLcode Curl_thrdpool_create(struct curl_thrdpool **ptpool,
|
CURLcode Curl_thrdpool_create(struct curl_thrdpool **ptpool,
|
||||||
const char *name,
|
const char *name,
|
||||||
uint32_t min_threads,
|
uint32_t min_threads,
|
||||||
@ -257,9 +315,6 @@ CURLcode Curl_thrdpool_create(struct curl_thrdpool **ptpool,
|
|||||||
Curl_cond_init(&tpool->await);
|
Curl_cond_init(&tpool->await);
|
||||||
Curl_llist_init(&tpool->slots, NULL);
|
Curl_llist_init(&tpool->slots, NULL);
|
||||||
Curl_llist_init(&tpool->zombies, NULL);
|
Curl_llist_init(&tpool->zombies, NULL);
|
||||||
tpool->min_threads = min_threads;
|
|
||||||
tpool->max_threads = max_threads;
|
|
||||||
tpool->idle_time_ms = idle_time_ms;
|
|
||||||
tpool->fn_take = fn_take;
|
tpool->fn_take = fn_take;
|
||||||
tpool->fn_process = fn_process;
|
tpool->fn_process = fn_process;
|
||||||
tpool->fn_return = fn_return;
|
tpool->fn_return = fn_return;
|
||||||
@ -269,10 +324,8 @@ CURLcode Curl_thrdpool_create(struct curl_thrdpool **ptpool,
|
|||||||
if(!tpool->name)
|
if(!tpool->name)
|
||||||
goto out;
|
goto out;
|
||||||
|
|
||||||
if(tpool->min_threads)
|
result = Curl_thrdpool_set_props(tpool, min_threads, max_threads,
|
||||||
result = Curl_thrdpool_signal(tpool, tpool->min_threads);
|
idle_time_ms);
|
||||||
else
|
|
||||||
result = CURLE_OK;
|
|
||||||
|
|
||||||
out:
|
out:
|
||||||
if(result && tpool) {
|
if(result && tpool) {
|
||||||
@ -316,35 +369,10 @@ void Curl_thrdpool_destroy(struct curl_thrdpool *tpool, bool join)
|
|||||||
|
|
||||||
CURLcode Curl_thrdpool_signal(struct curl_thrdpool *tpool, uint32_t nthreads)
|
CURLcode Curl_thrdpool_signal(struct curl_thrdpool *tpool, uint32_t nthreads)
|
||||||
{
|
{
|
||||||
struct Curl_llist_node *e, *n;
|
CURLcode result;
|
||||||
CURLcode result = CURLE_OK;
|
|
||||||
|
|
||||||
Curl_mutex_acquire(&tpool->lock);
|
Curl_mutex_acquire(&tpool->lock);
|
||||||
DEBUGASSERT(!tpool->aborted);
|
result = thrdpool_signal(tpool, nthreads);
|
||||||
|
|
||||||
thrdpool_join_zombies(tpool);
|
|
||||||
|
|
||||||
for(e = Curl_llist_head(&tpool->slots); e && nthreads; e = n) {
|
|
||||||
struct thrdslot *tslot = Curl_node_elem(e);
|
|
||||||
n = Curl_node_next(e);
|
|
||||||
if(tslot->idle) {
|
|
||||||
Curl_cond_signal(&tslot->await);
|
|
||||||
--nthreads;
|
|
||||||
}
|
|
||||||
else if(!tslot->starttime.tv_sec && !tslot->starttime.tv_usec) {
|
|
||||||
/* starting thread, queries for work soon. */
|
|
||||||
--nthreads;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
while(nthreads && !result &&
|
|
||||||
Curl_llist_count(&tpool->slots) < tpool->max_threads) {
|
|
||||||
result = thrdslot_start(tpool);
|
|
||||||
if(result)
|
|
||||||
break;
|
|
||||||
--nthreads;
|
|
||||||
}
|
|
||||||
|
|
||||||
Curl_mutex_release(&tpool->lock);
|
Curl_mutex_release(&tpool->lock);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -91,6 +91,12 @@ CURLcode Curl_thrdpool_signal(struct curl_thrdpool *tpool, uint32_t nthreads);
|
|||||||
CURLcode Curl_thrdpool_await_idle(struct curl_thrdpool *tpool,
|
CURLcode Curl_thrdpool_await_idle(struct curl_thrdpool *tpool,
|
||||||
uint32_t timeout_ms);
|
uint32_t timeout_ms);
|
||||||
|
|
||||||
|
/* Change the properties of a threadpool. */
|
||||||
|
CURLcode Curl_thrdpool_set_props(struct curl_thrdpool *tpool,
|
||||||
|
uint32_t min_threads,
|
||||||
|
uint32_t max_threads,
|
||||||
|
uint32_t idle_time_ms);
|
||||||
|
|
||||||
#ifdef CURLVERBOSE
|
#ifdef CURLVERBOSE
|
||||||
void Curl_thrdpool_trace(struct curl_thrdpool *tpool,
|
void Curl_thrdpool_trace(struct curl_thrdpool *tpool,
|
||||||
struct Curl_easy *data,
|
struct Curl_easy *data,
|
||||||
|
|||||||
@ -282,16 +282,20 @@ CURLcode Curl_thrdq_send(struct curl_thrdq *tqueue, void *item,
|
|||||||
result = CURLE_OUT_OF_MEMORY;
|
result = CURLE_OUT_OF_MEMORY;
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
|
item = NULL;
|
||||||
Curl_llist_append(&tqueue->sendq, qitem, &qitem->node);
|
Curl_llist_append(&tqueue->sendq, qitem, &qitem->node);
|
||||||
result = CURLE_OK;
|
|
||||||
signals = Curl_llist_count(&tqueue->sendq);
|
signals = Curl_llist_count(&tqueue->sendq);
|
||||||
|
result = CURLE_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
out:
|
out:
|
||||||
Curl_mutex_release(&tqueue->lock);
|
Curl_mutex_release(&tqueue->lock);
|
||||||
/* Signal thread pool unlocked to avoid deadlocks */
|
/* Signal thread pool unlocked to avoid deadlocks. Since we added
|
||||||
|
* item to the queue already, it might have been taken for processing
|
||||||
|
* already. Any error in signalling the pool cannot be reported to
|
||||||
|
* the caller since it needs to give up ownership of item. */
|
||||||
if(!result && signals)
|
if(!result && signals)
|
||||||
result = Curl_thrdpool_signal(tqueue->tpool, (uint32_t)signals);
|
(void)Curl_thrdpool_signal(tqueue->tpool, (uint32_t)signals);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -357,6 +361,27 @@ CURLcode Curl_thrdq_await_done(struct curl_thrdq *tqueue,
|
|||||||
return Curl_thrdpool_await_idle(tqueue->tpool, timeout_ms);
|
return Curl_thrdpool_await_idle(tqueue->tpool, timeout_ms);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CURLcode Curl_thrdq_set_props(struct curl_thrdq *tqueue,
|
||||||
|
uint32_t max_len,
|
||||||
|
uint32_t min_threads,
|
||||||
|
uint32_t max_threads,
|
||||||
|
uint32_t idle_time_ms)
|
||||||
|
{
|
||||||
|
CURLcode result;
|
||||||
|
size_t signals;
|
||||||
|
|
||||||
|
Curl_mutex_acquire(&tqueue->lock);
|
||||||
|
tqueue->send_max_len = max_len;
|
||||||
|
signals = Curl_llist_count(&tqueue->sendq);
|
||||||
|
Curl_mutex_release(&tqueue->lock);
|
||||||
|
|
||||||
|
result = Curl_thrdpool_set_props(tqueue->tpool, min_threads,
|
||||||
|
max_threads, idle_time_ms);
|
||||||
|
if(!result && signals)
|
||||||
|
result = Curl_thrdpool_signal(tqueue->tpool, (uint32_t)signals);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef CURLVERBOSE
|
#ifdef CURLVERBOSE
|
||||||
void Curl_thrdq_trace(struct curl_thrdq *tqueue,
|
void Curl_thrdq_trace(struct curl_thrdq *tqueue,
|
||||||
struct Curl_easy *data,
|
struct Curl_easy *data,
|
||||||
|
|||||||
@ -103,6 +103,12 @@ void Curl_thrdq_clear(struct curl_thrdq *tqueue,
|
|||||||
CURLcode Curl_thrdq_await_done(struct curl_thrdq *tqueue,
|
CURLcode Curl_thrdq_await_done(struct curl_thrdq *tqueue,
|
||||||
uint32_t timeout_ms);
|
uint32_t timeout_ms);
|
||||||
|
|
||||||
|
CURLcode Curl_thrdq_set_props(struct curl_thrdq *tqueue,
|
||||||
|
uint32_t max_len, /* 0 for unlimited */
|
||||||
|
uint32_t min_threads,
|
||||||
|
uint32_t max_threads,
|
||||||
|
uint32_t idle_time_ms);
|
||||||
|
|
||||||
#ifdef CURLVERBOSE
|
#ifdef CURLVERBOSE
|
||||||
void Curl_thrdq_trace(struct curl_thrdq *tqueue,
|
void Curl_thrdq_trace(struct curl_thrdq *tqueue,
|
||||||
struct Curl_easy *data,
|
struct Curl_easy *data,
|
||||||
|
|||||||
@ -3039,7 +3039,8 @@ static CURLcode resolve_server(struct Curl_easy *data,
|
|||||||
}
|
}
|
||||||
|
|
||||||
result = Curl_resolv(data, ehost->name, eport,
|
result = Curl_resolv(data, ehost->name, eport,
|
||||||
conn->ip_version, timeout_ms, pdns);
|
conn->ip_version, conn->transport_wanted,
|
||||||
|
timeout_ms, pdns);
|
||||||
DEBUGASSERT(!result || !*pdns);
|
DEBUGASSERT(!result || !*pdns);
|
||||||
if(!result) { /* resolved right away, either sync or from dnscache */
|
if(!result) { /* resolved right away, either sync or from dnscache */
|
||||||
DEBUGASSERT(*pdns);
|
DEBUGASSERT(*pdns);
|
||||||
|
|||||||
@ -1867,6 +1867,9 @@ static CURLcode parallel_transfers(CURLSH *share)
|
|||||||
if(!s->multi)
|
if(!s->multi)
|
||||||
return CURLE_OUT_OF_MEMORY;
|
return CURLE_OUT_OF_MEMORY;
|
||||||
|
|
||||||
|
#ifndef DEBUGBUILD
|
||||||
|
(void)curl_multi_setopt(s->multi, CURLMOPT_QUICK_EXIT, 1L);
|
||||||
|
#endif
|
||||||
(void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYFUNCTION, mnotify);
|
(void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYFUNCTION, mnotify);
|
||||||
(void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYDATA, s);
|
(void)curl_multi_setopt(s->multi, CURLMOPT_NOTIFYDATA, s);
|
||||||
(void)curl_multi_notify_enable(s->multi, CURLMNOTIFY_INFO_READ);
|
(void)curl_multi_notify_enable(s->multi, CURLMNOTIFY_INFO_READ);
|
||||||
|
|||||||
@ -72,7 +72,7 @@ via: 1.1 nghttpx
|
|||||||
s/^server: nghttpx.*\r?\n//
|
s/^server: nghttpx.*\r?\n//
|
||||||
</stripfile>
|
</stripfile>
|
||||||
<limits>
|
<limits>
|
||||||
Allocations: 155
|
Allocations: 160
|
||||||
Maximum allocated: 1800000
|
Maximum allocated: 1800000
|
||||||
</limits>
|
</limits>
|
||||||
</verify>
|
</verify>
|
||||||
|
|||||||
@ -63,6 +63,7 @@ EXTRA_DIST = \
|
|||||||
test_18_methods.py \
|
test_18_methods.py \
|
||||||
test_19_shutdown.py \
|
test_19_shutdown.py \
|
||||||
test_20_websockets.py \
|
test_20_websockets.py \
|
||||||
|
test_21_resolve.py \
|
||||||
test_30_vsftpd.py \
|
test_30_vsftpd.py \
|
||||||
test_31_vsftpds.py \
|
test_31_vsftpds.py \
|
||||||
test_32_ftps_vsftpd.py \
|
test_32_ftps_vsftpd.py \
|
||||||
|
|||||||
124
tests/http/test_21_resolve.py
Normal file
124
tests/http/test_21_resolve.py
Normal file
@ -0,0 +1,124 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
#***************************************************************************
|
||||||
|
# _ _ ____ _
|
||||||
|
# Project ___| | | | _ \| |
|
||||||
|
# / __| | | | |_) | |
|
||||||
|
# | (__| |_| | _ <| |___
|
||||||
|
# \___|\___/|_| \_\_____|
|
||||||
|
#
|
||||||
|
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
|
||||||
|
#
|
||||||
|
# This software is licensed as described in the file COPYING, which
|
||||||
|
# you should have received as part of this distribution. The terms
|
||||||
|
# are also available at https://curl.se/docs/copyright.html.
|
||||||
|
#
|
||||||
|
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
|
||||||
|
# copies of the Software, and permit persons to whom the Software is
|
||||||
|
# furnished to do so, under the terms of the COPYING file.
|
||||||
|
#
|
||||||
|
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
|
||||||
|
# KIND, either express or implied.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: curl
|
||||||
|
#
|
||||||
|
###########################################################################
|
||||||
|
#
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import timedelta
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from testenv import Env, CurlClient, LocalClient
|
||||||
|
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(condition=not Env.curl_is_debug(), reason="needs curl debug")
|
||||||
|
@pytest.mark.skipif(condition=Env.curl_uses_lib('c-ares'), reason="c-ares resolver skipped")
|
||||||
|
@pytest.mark.skipif(condition=not Env.curl_has_feature('AsynchDNS'), reason="needs AsynchDNS")
|
||||||
|
class TestResolve:
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True, scope='class')
|
||||||
|
def _class_scope(self, env, httpd):
|
||||||
|
indir = httpd.docs_dir
|
||||||
|
env.make_data_file(indir=indir, fname="data-0k", fsize=0)
|
||||||
|
|
||||||
|
# use .invalid host name that should never resolv
|
||||||
|
def test_21_01_resolv_invalid_one(self, env: Env, httpd, nghttpx):
|
||||||
|
count = 1
|
||||||
|
run_env = os.environ.copy()
|
||||||
|
run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = '5'
|
||||||
|
curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
|
||||||
|
url = f'https://test-{count}.http.curl.invalid/'
|
||||||
|
r = curl.http_download(urls=[url], with_stats=True)
|
||||||
|
r.check_exit_code(6)
|
||||||
|
r.check_stats(count=count, http_status=0, exitcode=6)
|
||||||
|
|
||||||
|
# use .invalid host name, one after the other
|
||||||
|
@pytest.mark.parametrize("delay_ms", [1, 50])
|
||||||
|
def test_21_02_resolv_invalid_serial(self, env: Env, delay_ms, httpd, nghttpx):
|
||||||
|
count = 10
|
||||||
|
run_env = os.environ.copy()
|
||||||
|
run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
|
||||||
|
curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
|
||||||
|
urls = [ f'https://test-{i}.http.curl.invalid/' for i in range(count)]
|
||||||
|
r = curl.http_download(urls=urls, with_stats=True)
|
||||||
|
r.check_exit_code(6)
|
||||||
|
r.check_stats(count=count, http_status=0, exitcode=6)
|
||||||
|
|
||||||
|
# use .invalid host name, parallel
|
||||||
|
@pytest.mark.parametrize("delay_ms", [1, 50])
|
||||||
|
def test_21_03_resolv_invalid_parallel(self, env: Env, delay_ms, httpd, nghttpx):
|
||||||
|
count = 20
|
||||||
|
run_env = os.environ.copy()
|
||||||
|
run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
|
||||||
|
curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
|
||||||
|
urls = [ f'https://test-{i}.http.curl.invalid/' for i in range(count)]
|
||||||
|
r = curl.http_download(urls=urls, with_stats=True, extra_args=[
|
||||||
|
'--parallel'
|
||||||
|
])
|
||||||
|
r.check_exit_code(6)
|
||||||
|
r.check_stats(count=count, http_status=0, exitcode=6)
|
||||||
|
|
||||||
|
# resolve first url with ipv6 only and fail that, resolve second
|
||||||
|
# with ipv*, should succeed.
|
||||||
|
def test_21_04_resolv_inv_v6(self, env: Env, httpd):
|
||||||
|
count = 2
|
||||||
|
run_env = os.environ.copy()
|
||||||
|
run_env['CURL_DBG_RESOLV_FAIL_IPV6'] = '1'
|
||||||
|
url = f'https://localhost:{env.https_port}/'
|
||||||
|
client = LocalClient(name='cli_hx_download', env=env, run_env=run_env)
|
||||||
|
if not client.exists():
|
||||||
|
pytest.skip(f'example client not built: {client.name}')
|
||||||
|
dfiles = [client.download_file(i) for i in range(count)]
|
||||||
|
self._clean_files(dfiles)
|
||||||
|
# let the first URL resolve via ipv6 only, which we force to fail
|
||||||
|
r = client.run(args=[
|
||||||
|
'-n', f'{count}', '-6', '-C', env.ca.cert_file, url
|
||||||
|
])
|
||||||
|
r.check_exit_code(6)
|
||||||
|
assert not os.path.exists(dfiles[0])
|
||||||
|
assert os.path.exists(dfiles[1])
|
||||||
|
|
||||||
|
# use .invalid host name, parallel, single resolve thread
|
||||||
|
def test_21_05_resolv_single_thread(self, env: Env, httpd, nghttpx):
|
||||||
|
count = 10
|
||||||
|
delay_ms = 50
|
||||||
|
run_env = os.environ.copy()
|
||||||
|
run_env['CURL_DBG_RESOLV_FAIL_DELAY'] = f'{delay_ms}'
|
||||||
|
run_env['CURL_DBG_RESOLV_MAX_THREADS'] = '1'
|
||||||
|
curl = CurlClient(env=env, run_env=run_env, force_resolv=False)
|
||||||
|
urls = [ f'https://test-{i}.http.curl.invalid/' for i in range(count)]
|
||||||
|
r = curl.http_download(urls=urls, with_stats=True, extra_args=[
|
||||||
|
'--parallel', '-6'
|
||||||
|
])
|
||||||
|
r.check_exit_code(6)
|
||||||
|
r.check_stats(count=count, http_status=0, exitcode=6)
|
||||||
|
assert r.duration > timedelta(milliseconds=count * delay_ms), f'{r}'
|
||||||
|
|
||||||
|
def _clean_files(self, files):
|
||||||
|
for file in files:
|
||||||
|
if os.path.exists(file):
|
||||||
|
os.remove(file)
|
||||||
@ -630,6 +630,7 @@ class CurlClient:
|
|||||||
with_dtrace: bool = False,
|
with_dtrace: bool = False,
|
||||||
with_perf: bool = False,
|
with_perf: bool = False,
|
||||||
with_flame: bool = False,
|
with_flame: bool = False,
|
||||||
|
force_resolv: bool = True,
|
||||||
socks_args: Optional[List[str]] = None):
|
socks_args: Optional[List[str]] = None):
|
||||||
self.env = env
|
self.env = env
|
||||||
self._timeout = timeout if timeout else env.test_timeout
|
self._timeout = timeout if timeout else env.test_timeout
|
||||||
@ -659,6 +660,7 @@ class CurlClient:
|
|||||||
self._silent = silent
|
self._silent = silent
|
||||||
self._run_env = run_env
|
self._run_env = run_env
|
||||||
self._server_addr = server_addr if server_addr else '127.0.0.1'
|
self._server_addr = server_addr if server_addr else '127.0.0.1'
|
||||||
|
self._force_resolv = force_resolv
|
||||||
self._rmrf(self._run_dir)
|
self._rmrf(self._run_dir)
|
||||||
self._mkpath(self._run_dir)
|
self._mkpath(self._run_dir)
|
||||||
|
|
||||||
@ -1086,7 +1088,6 @@ class CurlClient:
|
|||||||
def _raw(self, urls, intext='', timeout=None, options=None, insecure=False,
|
def _raw(self, urls, intext='', timeout=None, options=None, insecure=False,
|
||||||
alpn_proto: Optional[str] = None,
|
alpn_proto: Optional[str] = None,
|
||||||
url_options=None,
|
url_options=None,
|
||||||
force_resolve=True,
|
|
||||||
with_stats=False,
|
with_stats=False,
|
||||||
with_headers=True,
|
with_headers=True,
|
||||||
def_tracing=True,
|
def_tracing=True,
|
||||||
@ -1094,9 +1095,8 @@ class CurlClient:
|
|||||||
with_tcpdump=False):
|
with_tcpdump=False):
|
||||||
args = self._complete_args(
|
args = self._complete_args(
|
||||||
urls=urls, timeout=timeout, options=options, insecure=insecure,
|
urls=urls, timeout=timeout, options=options, insecure=insecure,
|
||||||
alpn_proto=alpn_proto, force_resolve=force_resolve,
|
alpn_proto=alpn_proto, with_headers=with_headers,
|
||||||
with_headers=with_headers, def_tracing=def_tracing,
|
def_tracing=def_tracing, url_options=url_options)
|
||||||
url_options=url_options)
|
|
||||||
r = self._run(args, intext=intext, with_stats=with_stats,
|
r = self._run(args, intext=intext, with_stats=with_stats,
|
||||||
with_profile=with_profile, with_tcpdump=with_tcpdump)
|
with_profile=with_profile, with_tcpdump=with_tcpdump)
|
||||||
if r.exit_code == 0 and with_headers:
|
if r.exit_code == 0 and with_headers:
|
||||||
@ -1104,8 +1104,7 @@ class CurlClient:
|
|||||||
return r
|
return r
|
||||||
|
|
||||||
def _complete_args(self, urls, timeout=None, options=None,
|
def _complete_args(self, urls, timeout=None, options=None,
|
||||||
insecure=False, force_resolve=True,
|
insecure=False, alpn_proto: Optional[str] = None,
|
||||||
alpn_proto: Optional[str] = None,
|
|
||||||
url_options=None,
|
url_options=None,
|
||||||
with_headers: bool = True,
|
with_headers: bool = True,
|
||||||
def_tracing: bool = True):
|
def_tracing: bool = True):
|
||||||
@ -1115,6 +1114,8 @@ class CurlClient:
|
|||||||
|
|
||||||
if options is not None and '--resolve' in options:
|
if options is not None and '--resolve' in options:
|
||||||
force_resolve = False
|
force_resolve = False
|
||||||
|
else:
|
||||||
|
force_resolve = self._force_resolv
|
||||||
|
|
||||||
args = [self._curl, "-s", "--path-as-is"]
|
args = [self._curl, "-s", "--path-as-is"]
|
||||||
if 'CURL_TEST_EVENT' in os.environ:
|
if 'CURL_TEST_EVENT' in os.environ:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user