pytest: enable in 5 GHA jobs, fix fallouts

They take about 4.5 minutes of CI time in GHA/macos. Also: - autotools: improve `caddy`, `vsftpd` detection. Bringing it closer to cmake. - autotools: fix `--with-test-caddy=no`, `--with-test-vsftps=no`, `--with-test-nghttpx=no` options. - cmake: sync `nghttpx` default with autotools. - pytest: disable failing mbedTLS tests on macOS CI. - pytest: disable failing earlydata tests on macOS CI. - GHA/macos: keep vsftpd pytests disabled due to lengthy run times. - pytest: fix test_05_04 for LibreSSL. Ref: #17367 Authored-by: Stefan Eissing Remaining issues: - some unidentified tests taking a long time with mbedTLS: `================= 462 passed, 278 skipped in 347.93s (0:05:47) =================` Ref: https://github.com/curl/curl/actions/runs/15073354301/job/42374999041#step:17:1536 Workaround: not enabling pytest for mbedTLS jobs - 17 FTP tests taking a long time (affecting all TLS backends): without vsftpd: `====================== 496 passed, 244 skipped in 56.15s =======================` Ref: https://github.com/curl/curl/actions/runs/15073354301/job/42374998230#step:17:1536 with vsftpd: `================= 513 passed, 227 skipped in 409.37s (0:06:49) =================` Ref: https://github.com/curl/curl/actions/runs/15073678568/job/42376039672?pr=17362#step:17:1537 Workaround: force-disable vsftpd. - 100 tests failing with SecureTransport. Let's ignore that due to imminent deprecation. Ref: https://github.com/curl/curl/actions/runs/15055652333/job/42320873732#step:17:15362 Follow-up to 30ef79ed93 #17295 Follow-up to 9147903366 #16518 Closes #17362
2026-04-11 12:01:42 +08:00 · 2025-05-15 22:11:06 +02:00 · 2025-05-15 22:11:06 +02:00 · 42331cb48a
commit 42331cb48a
parent 60f8f6d5cb
9 changed files with 68 additions and 16 deletions
--- a/.github/workflows/macos.yml
+++ b/.github/workflows/macos.yml
@ -116,9 +116,11 @@ jobs:
          - name: 'LibreSSL +examples'
            compiler: clang
            install: libressl
+            install_steps: pytest
            configure: --enable-debug --with-openssl=$(brew --prefix libressl)
          - name: 'OpenSSL'
            compiler: clang
+            install_steps: pytest
            configure: --enable-debug --with-openssl=$(brew --prefix openssl)
          - name: 'OpenSSL event-based'
            compiler: clang
@ -149,6 +151,7 @@ jobs:
            generate: -DOPENSSL_ROOT_DIR=$(brew --prefix libressl) -DENABLE_ARES=ON -DCURL_USE_GSSAPI=ON -DGSS_ROOT_DIR=$(brew --prefix heimdal) -DCURL_DISABLE_LDAP=ON
          - name: 'wolfSSL !ldap brotli zstd'
            install: brotli wolfssl zstd
+            install_steps: pytest
            generate: -DCURL_USE_WOLFSSL=ON -DCURL_DISABLE_LDAP=ON -DUSE_ECH=ON
          - name: 'mbedTLS openldap brotli zstd'
            install: brotli mbedtls zstd openldap
@ -237,7 +240,7 @@ jobs:
            for _chkprefill in '' ${{ matrix.build.chkprefill }}; do
              options=''
              [ -n '${{ matrix.build.macos-version-min }}' ] && options+=' -DCMAKE_OSX_DEPLOYMENT_TARGET=${{ matrix.build.macos-version-min }}'
-              [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' -DTEST_NGHTTPX= -DHTTPD_NGHTTPX='
+              [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' -DVSFTPD=NO'  # Skip ~20 tests that stretch run time by 7x on macOS
              [ "${_chkprefill}" = '_chkprefill' ] && options+=' -D_CURL_PREFILL=OFF'
              cmake -B "bld${_chkprefill}" -G Ninja -D_CURL_PREFILL=ON \
                -DCMAKE_INSTALL_PREFIX="$HOME/curl" \
@ -260,7 +263,7 @@ jobs:
              CFLAGS+=" --sysroot=${sysroot}"
            fi
            [ -n '${{ matrix.build.macos-version-min }}' ] && CFLAGS+=' -mmacosx-version-min=${{ matrix.build.macos-version-min }}'
-            [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' --with-test-nghttpx= ac_cv_path_HTTPD_NGHTTPX='
+            [[ '${{ matrix.build.install_steps }}' = *'pytest'* ]] && options+=' --with-test-vsftpd=no'  # Skip ~20 tests that stretch run time by 7x on macOS
            mkdir bld && cd bld && ../configure --enable-unity --enable-test-bundles --enable-warnings --enable-werror \
              --disable-dependency-tracking \
              --with-libpsl=$(brew --prefix libpsl) \
--- a/configure.ac
+++ b/configure.ac
@ -299,27 +299,39 @@ TEST_NGHTTPX=nghttpx
 AC_ARG_WITH(test-nghttpx,dnl
 AS_HELP_STRING([--with-test-nghttpx=PATH],[where to find nghttpx for testing]),
  TEST_NGHTTPX=$withval
-  if test X"$OPT_TEST_NGHTTPX" = "Xno"; then
+  if test X"$TEST_NGHTTPX" = "Xno"; then
    TEST_NGHTTPX=""
  fi
 )
 AC_SUBST(TEST_NGHTTPX)

-CADDY=/usr/bin/caddy
+if test -x /usr/bin/caddy; then
+  CADDY=/usr/bin/caddy
+elif test -x /usr/local/bin/caddy; then
+  CADDY=/usr/local/bin/caddy
+elif test -x `brew --prefix`/bin/caddy; then
+  CADDY=`brew --prefix`/bin/caddy
+fi
 AC_ARG_WITH(test-caddy,dnl
 AS_HELP_STRING([--with-test-caddy=PATH],[where to find caddy for testing]),
  CADDY=$withval
-  if test X"$OPT_CADDY" = "Xno"; then
+  if test X"$CADDY" = "Xno"; then
    CADDY=""
  fi
 )
 AC_SUBST(CADDY)

-VSFTPD=/usr/sbin/vsftpd
+if test -x /usr/sbin/vsftpd; then
+  VSFTPD=/usr/sbin/vsftpd
+elif test -x /usr/local/sbin/vsftpd; then
+  VSFTPD=/usr/local/sbin/vsftpd
+elif test -x `brew --prefix`/sbin/vsftpd; then
+  VSFTPD=`brew --prefix`/sbin/vsftpd
+fi
 AC_ARG_WITH(test-vsftpd,dnl
 AS_HELP_STRING([--with-test-vsftpd=PATH],[where to find vsftpd for testing]),
  VSFTPD=$withval
-  if test X"$OPT_VSFTPD" = "Xno"; then
+  if test X"$VSFTPD" = "Xno"; then
    VSFTPD=""
  fi
 )
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@ -23,7 +23,7 @@
 ###########################################################################
 find_program(TEST_NGHTTPX "nghttpx")
 if(NOT TEST_NGHTTPX)
-  set(TEST_NGHTTPX "nghttpx")
+  set(TEST_NGHTTPX "")
 endif()
 mark_as_advanced(TEST_NGHTTPX)
 # Consumed variables: TEST_NGHTTPX
--- a/tests/http/test_02_download.py
+++ b/tests/http/test_02_download.py
@ -30,6 +30,7 @@ import logging
 import math
 import os
 import re
+import sys
 from datetime import timedelta
 import pytest

@ -76,6 +77,9 @@ class TestDownload:
    def test_02_03_download_sequential(self, env: Env, httpd, nghttpx, proto):
        if proto == 'h3' and not env.have_h3():
            pytest.skip("h3 not supported")
+        if (proto == 'http/1.1' or proto == 'h2') and env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        count = 10
        curl = CurlClient(env=env)
        urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
@ -87,6 +91,9 @@ class TestDownload:
    def test_02_04_download_parallel(self, env: Env, httpd, nghttpx, proto):
        if proto == 'h3' and not env.have_h3():
            pytest.skip("h3 not supported")
+        if proto == 'h2' and env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        count = 10
        max_parallel = 5
        curl = CurlClient(env=env)
@ -109,6 +116,9 @@ class TestDownload:
            pytest.skip("h3 not supported")
        if proto == 'h3' and env.curl_uses_lib('msh3'):
            pytest.skip("msh3 shaky here")
+        if proto == 'h2' and env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        count = 200
        curl = CurlClient(env=env)
        urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]'
@ -126,6 +136,9 @@ class TestDownload:
    def test_02_06_download_many_parallel(self, env: Env, httpd, nghttpx, proto):
        if proto == 'h3' and not env.have_h3():
            pytest.skip("h3 not supported")
+        if proto == 'h2' and env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        count = 200
        max_parallel = 50
        curl = CurlClient(env=env)
@ -591,6 +604,8 @@ class TestDownload:
        if proto == 'h3' and \
           (not env.have_h3() or not env.curl_can_h3_early_data()):
            pytest.skip("h3 not supported")
+        if proto != 'h3' and sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('failing on macOS CI runners')
        count = 2
        docname = 'data-10k'
        # we want this test to always connect to nghttpx, since it is
@ -635,6 +650,8 @@ class TestDownload:
    @pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
    @pytest.mark.parametrize("max_host_conns", [0, 1, 5])
    def test_02_33_max_host_conns(self, env: Env, httpd, nghttpx, proto, max_host_conns):
+        if not env.curl_is_debug():
+            pytest.skip('only works for curl debug builds')
        if proto == 'h3' and not env.have_h3():
            pytest.skip("h3 not supported")
        count = 50
@ -671,6 +688,8 @@ class TestDownload:
    @pytest.mark.parametrize("proto", ['http/1.1', 'h2'])
    @pytest.mark.parametrize("max_total_conns", [0, 1, 5])
    def test_02_34_max_total_conns(self, env: Env, httpd, nghttpx, proto, max_total_conns):
+        if not env.curl_is_debug():
+            pytest.skip('only works for curl debug builds')
        if proto == 'h3' and not env.have_h3():
            pytest.skip("h3 not supported")
        count = 50
--- a/tests/http/test_05_errors.py
+++ b/tests/http/test_05_errors.py
@ -118,10 +118,11 @@ class TestErrors:
        url = f'https://{env.authority_for(env.domain1, proto)}'\
            f'/curltest/shutdown_unclean?id=[0-{count-1}]&chunks=4'
        r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[
-            '--parallel',
+            '--parallel', '--trace-config', 'ssl'
        ])
        if proto == 'http/1.0' and not env.curl_uses_lib('wolfssl') and \
-                (env.curl_is_debug() or not env.curl_uses_lib('openssl')):
+                (env.curl_is_debug() or
+                not env.curl_uses_any_libs(['openssl', 'libressl'])):
            # we are inconsistent if we fail or not in missing TLS shutdown
            # openssl code ignore such errors intentionally in non-debug builds
            r.check_exit_code(56)
--- a/tests/http/test_07_upload.py
+++ b/tests/http/test_07_upload.py
@ -29,6 +29,7 @@ import filecmp
 import logging
 import os
 import re
+import sys
 import pytest
 from typing import List, Union

@ -708,6 +709,8 @@ class TestUpload:
        if proto == 'h3' and \
           (not env.have_h3() or not env.curl_can_h3_early_data()):
            pytest.skip("h3 not supported")
+        if proto != 'h3' and sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('failing on macOS CI runners')
        count = 2
        # we want this test to always connect to nghttpx, since it is
        # the only server we have that supports TLS earlydata
--- a/tests/http/test_10_proxy.py
+++ b/tests/http/test_10_proxy.py
@ -28,6 +28,7 @@ import filecmp
 import logging
 import os
 import re
+import sys
 import pytest

 from testenv import Env, CurlClient, ExecResult
@ -179,6 +180,9 @@ class TestProxy:
                                  tunnel, fname, fcount):
        if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
            pytest.skip('only supported with nghttp2')
+        if env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        count = fcount
        curl = CurlClient(env=env)
        url = f'https://localhost:{env.https_port}/{fname}?[0-{count-1}]'
@ -209,6 +213,9 @@ class TestProxy:
                                    tunnel, fname, fcount):
        if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
            pytest.skip('only supported with nghttp2')
+        if env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        count = fcount
        srcfile = os.path.join(httpd.docs_dir, fname)
        curl = CurlClient(env=env)
@ -255,6 +262,9 @@ class TestProxy:
        # url twice via https: proxy separated with '--next', will reuse
        if tunnel == 'h2' and not env.curl_uses_lib('nghttp2'):
            pytest.skip('only supported with nghttp2')
+        if env.curl_uses_lib('mbedtls') and \
+           sys.platform.startswith('darwin') and env.ci_run:
+            pytest.skip('mbedtls 3.6.3 fails this test on macOS CI runners')
        curl = CurlClient(env=env)
        url = f'https://localhost:{env.https_port}/data.json'
        proxy_args = curl.get_proxy_args(tunnel=True, proto=tunnel)
--- a/tests/http/test_20_websockets.py
+++ b/tests/http/test_20_websockets.py
@ -49,7 +49,7 @@ class TestWebsockets:
        'ws': socket.SOCK_STREAM,
    }

-    def check_alive(self, env, port, timeout=5):
+    def check_alive(self, env, port, timeout=Env.SERVER_TIMEOUT):
        curl = CurlClient(env=env)
        url = f'http://localhost:{port}/'
        end = datetime.now() + timedelta(seconds=timeout)
--- a/tests/http/testenv/env.py
+++ b/tests/http/testenv/env.py
@ -33,7 +33,7 @@ import subprocess
 import tempfile
 from configparser import ConfigParser, ExtendedInterpolation
 from datetime import timedelta
-from typing import Optional, Dict
+from typing import Optional, Dict, List

 import pytest
 from filelock import FileLock
@ -329,6 +329,13 @@ class Env:
    def curl_uses_lib(libname: str) -> bool:
        return libname.lower() in Env.CONFIG.curl_props['libs']

+    @staticmethod
+    def curl_uses_any_libs(libs: List[str]) -> bool:
+        for libname in libs:
+            if libname.lower() in Env.CONFIG.curl_props['libs']:
+                return True
+        return False
+
    @staticmethod
    def curl_uses_ossl_quic() -> bool:
        if Env.have_h3_curl():
@ -389,10 +396,7 @@ class Env:

    @staticmethod
    def curl_can_early_data() -> bool:
-        return Env.curl_uses_lib('gnutls') or \
-            Env.curl_uses_lib('wolfssl') or \
-            Env.curl_uses_lib('quictls') or \
-            Env.curl_uses_lib('openssl')
+        return Env.curl_uses_any_libs(['gnutls', 'wolfssl', 'quictls', 'openssl'])

    @staticmethod
    def curl_can_h3_early_data() -> bool: