From 5172ba5475cffc525c2338dfa63f818e11e80a42 Mon Sep 17 00:00:00 2001 From: Viktor Szakats Date: Thu, 26 Mar 2026 15:59:23 +0100 Subject: [PATCH] GHA: try workaround for slow Azure Ubuntu distro server It's been going on for almost a year, chances seem slim to have this fixed upstream. The observed issue is that on GitHub-supplied CI runner images, sometimes, `apt-get install` using the stock distro server `http://azure.archive.ubuntu.com/ubuntu` sees extreme slowness while downloading data, then reaching the job timeout and aborting, making CI red and necessitating a manual restart of the failed job(s). In majority of cases the install step takes no longer than 8-40 seconds (download + install), while in the failing case it takes 10-20+ minutes just to download. Earlier I slimmed down installs to avoid unnecessary packages, which mitigated, but did not completely fix the issue. Example: https://github.com/curl/curl/actions/runs/23599596928/job/68725576899?pr=21105 Follow-up to 0455d8772a1af20ce63c46c5738582aa9b1b8441 #18509 Closes #21107 --- .github/workflows/codeql.yml | 4 +++- .github/workflows/configure-vs-cmake.yml | 4 +++- .github/workflows/distcheck.yml | 3 ++- .github/workflows/http3-linux.yml | 6 ++++-- .github/workflows/linux.yml | 9 ++++++--- .github/workflows/non-native.yml | 4 +++- .github/workflows/windows.yml | 4 +++- 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3b4a067c24..c1921a692b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -73,7 +73,9 @@ jobs: run: | sudo rm -f /etc/apt/sources.list.d/{azure-cli.sources,microsoft-prod.list,ondrej-ubuntu-php-noble.sources} sudo apt-get -o Dpkg::Use-Pty=0 update - sudo apt-get -o Dpkg::Use-Pty=0 install libpsl-dev libbrotli-dev libidn2-dev libssh2-1-dev libssh-dev \ + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 60 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install \ + libpsl-dev libbrotli-dev libidn2-dev libssh2-1-dev libssh-dev \ libnghttp2-dev libldap-dev libkrb5-dev libgnutls28-dev libwolfssl-dev HOMEBREW_NO_AUTO_UPDATE=1 /home/linuxbrew/.linuxbrew/bin/brew install c-ares gsasl libnghttp3 libngtcp2 mbedtls rustls-ffi diff --git a/.github/workflows/configure-vs-cmake.yml b/.github/workflows/configure-vs-cmake.yml index 3c8f0249f8..f0dc948786 100644 --- a/.github/workflows/configure-vs-cmake.yml +++ b/.github/workflows/configure-vs-cmake.yml @@ -144,7 +144,9 @@ jobs: TRIPLET: 'x86_64-w64-mingw32' steps: - name: 'install packages' - run: sudo apt-get -o Dpkg::Use-Pty=0 install gcc-mingw-w64-x86-64-win32 + run: | + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 30 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install gcc-mingw-w64-x86-64-win32 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/.github/workflows/distcheck.yml b/.github/workflows/distcheck.yml index d09f45d433..5e2cb02547 100644 --- a/.github/workflows/distcheck.yml +++ b/.github/workflows/distcheck.yml @@ -288,7 +288,8 @@ jobs: sha256sum pkg.bin && sha256sum pkg.bin | grep -qwF -- "${OLD_CMAKE_SHA256_WIN_INTEL}" && unzip -q pkg.bin && rm -f pkg.bin printf '%s' ~/cmake-"${OLD_CMAKE_VERSION}"-win64-x64/bin/cmake.exe > ~/old-cmake-path.txt elif [[ "${MATRIX_IMAGE}" = *'ubuntu'* ]]; then - sudo apt-get -o Dpkg::Use-Pty=0 install libpsl-dev libssl-dev + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 30 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install libpsl-dev libssl-dev cd ~ curl --disable --fail --silent --show-error --connect-timeout 15 --max-time 60 --retry 3 --retry-connrefused \ --location "https://github.com/Kitware/CMake/releases/download/v${OLD_CMAKE_VERSION}/cmake-${OLD_CMAKE_VERSION}-Linux-aarch64.tar.gz" --output pkg.bin diff --git a/.github/workflows/http3-linux.yml b/.github/workflows/http3-linux.yml index 6ad55a8892..5995394e20 100644 --- a/.github/workflows/http3-linux.yml +++ b/.github/workflows/http3-linux.yml @@ -186,7 +186,8 @@ jobs: run: | sudo rm -f /etc/apt/sources.list.d/{azure-cli.sources,microsoft-prod.list,ondrej-ubuntu-php-noble.sources} sudo apt-get -o Dpkg::Use-Pty=0 update - sudo apt-get -o Dpkg::Use-Pty=0 install \ + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 60 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install \ libtool autoconf automake pkgconf \ libbrotli-dev libzstd-dev zlib1g-dev \ libev-dev \ @@ -489,7 +490,8 @@ jobs: run: | sudo rm -f /etc/apt/sources.list.d/{azure-cli.sources,microsoft-prod.list,ondrej-ubuntu-php-noble.sources} sudo apt-get -o Dpkg::Use-Pty=0 update - sudo apt-get -o Dpkg::Use-Pty=0 install \ + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 45 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install \ libtool autoconf automake pkgconf \ libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libidn2-0-dev libldap-dev libuv1-dev valgrind \ ${INSTALL_PACKAGES} \ diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f1ae2bcac9..206d21d244 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -429,7 +429,8 @@ jobs: run: | sudo rm -f /etc/apt/sources.list.d/{azure-cli.sources,microsoft-prod.list,ondrej-ubuntu-php-noble.sources} sudo apt-get -o Dpkg::Use-Pty=0 update - sudo apt-get -o Dpkg::Use-Pty=0 install \ + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 45 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install \ libtool autoconf automake pkgconf \ libpsl-dev zlib1g-dev libbrotli-dev libzstd-dev \ ${INSTALL_PACKAGES} \ @@ -450,7 +451,8 @@ jobs: sudo rm -f /etc/apt/sources.list.d/{azure-cli.sources,microsoft-prod.list,ondrej-ubuntu-php-noble.sources} sudo dpkg --add-architecture i386 sudo apt-get -o Dpkg::Use-Pty=0 update - sudo apt-get -o Dpkg::Use-Pty=0 install \ + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 45 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install \ libtool autoconf automake pkgconf stunnel4 \ libpsl-dev:i386 libbrotli-dev:i386 libzstd-dev:i386 \ ${MATRIX_INSTALL_PACKAGES} @@ -790,7 +792,8 @@ jobs: --compressed https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | \ sudo tee /etc/apt/trusted.gpg.d/intel-sw.asc >/dev/null sudo add-apt-repository 'deb https://apt.repos.intel.com/oneapi all main' - sudo apt-get -o Dpkg::Use-Pty=0 install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 90 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic source /opt/intel/oneapi/setvars.sh printenv >> "$GITHUB_ENV" diff --git a/.github/workflows/non-native.yml b/.github/workflows/non-native.yml index cb12569b3b..49a351cfcf 100644 --- a/.github/workflows/non-native.yml +++ b/.github/workflows/non-native.yml @@ -348,7 +348,9 @@ jobs: fail-fast: false steps: - name: 'install packages' - run: sudo apt-get -o Dpkg::Use-Pty=0 install libfl2 + run: | + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 30 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install libfl2 - name: 'cache compiler (djgpp)' uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 6bee3d2eb3..a5d96ef0a6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -740,7 +740,9 @@ jobs: - name: 'install packages' env: MATRIX_INSTALL_PACKAGES: '${{ matrix.install_packages }}' - run: sudo apt-get -o Dpkg::Use-Pty=0 install gcc-mingw-w64-x86-64-win32 ${MATRIX_INSTALL_PACKAGES} + run: | + printf '#!/bin/sh\nfor i in 1 2 3; do timeout 30 "$@" && break; echo "Error: slow server, retry"; sleep 1; done' > "$HOME"/my-apt; chmod +x "$HOME"/my-apt + sudo "$HOME"/my-apt apt-get -o Dpkg::Use-Pty=0 install gcc-mingw-w64-x86-64-win32 ${MATRIX_INSTALL_PACKAGES} - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: