autotools: add nettle library detection via pkg-config (for GnuTLS)

Also:
- fix to restore full state when gnutls canary function is not found.
- fix indentation.

Closes #19703

.github/workflows/http3-linux.yml

@@ -409,8 +409,7 @@ jobs:
             install_packages: libp11-kit-dev
             install_steps: skipall
             PKG_CONFIG_PATH: /home/runner/nettle/build/lib64/pkgconfig:/home/runner/gnutls/build/lib/pkgconfig:/home/runner/nghttp3/build/lib/pkgconfig:/home/runner/ngtcp2/build/lib/pkgconfig:/home/runner/nghttp2/build/lib/pkgconfig
-            LDFLAGS: -Wl,-rpath,/home/runner/gnutls/build/lib -Wl,-rpath,/home/runner/nettle/build/lib64 -L/home/runner/nettle/build/lib64 -Wl,-rpath,/home/runner/ngtcp2/build/lib
-            CPPFLAGS: -I/home/runner/nettle/build/include
+            LDFLAGS: -Wl,-rpath,/home/runner/gnutls/build/lib -Wl,-rpath,/home/runner/nettle/build/lib64 -Wl,-rpath,/home/runner/ngtcp2/build/lib
             configure: >-
               --with-gnutls=/home/runner/gnutls/build --with-ngtcp2 --enable-ssls-export
 
@@ -640,7 +639,6 @@ jobs:
 
       - name: 'configure'
         env:
-          CPPFLAGS: '${{ matrix.build.CPPFLAGS }}'
           LDFLAGS: '${{ matrix.build.LDFLAGS }}'
           MATRIX_CONFIGURE: '${{ matrix.build.configure }}'
           MATRIX_GENERATE: '${{ matrix.build.generate }}'

m4/curl-gnutls.m4

@@ -100,18 +100,20 @@ if test "x$OPT_GNUTLS" != xno; then
 
       dnl this function is selected since it was introduced in 3.1.10
       AC_CHECK_LIB(gnutls, gnutls_x509_crt_get_dn2,
-        [
+      [
         AC_DEFINE(USE_GNUTLS, 1, [if GnuTLS is enabled])
         GNUTLS_ENABLED=1
         USE_GNUTLS="yes"
         ssl_msg="GnuTLS"
         QUIC_ENABLED=yes
         test gnutls != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes
-        ],
-        [
-          LIBS="$CLEANLIBS"
-          CPPFLAGS="$CLEANCPPFLAGS"
-        ])
+      ],
+      [
+        LIBS="$CLEANLIBS"
+        CPPFLAGS="$CLEANCPPFLAGS"
+        LDFLAGS="$CLEANLDFLAGS"
+        LDFLAGSPC="$CLEANLDFLAGSPC"
+      ])
 
       if test "x$USE_GNUTLS" = "xyes"; then
         AC_MSG_NOTICE([detected GnuTLS version $version])
@@ -127,9 +129,8 @@ if test "x$OPT_GNUTLS" != xno; then
             AC_MSG_NOTICE([Added $gtlslib to CURL_LIBRARY_PATH])
           fi
         fi
-        LIBCURL_PC_REQUIRES_PRIVATE="$LIBCURL_PC_REQUIRES_PRIVATE gnutls nettle"
+        LIBCURL_PC_REQUIRES_PRIVATE="$LIBCURL_PC_REQUIRES_PRIVATE gnutls"
       fi
-
     fi
 
   fi dnl GNUTLS not disabled
@@ -147,12 +148,60 @@ if test "$GNUTLS_ENABLED" = "1"; then
 
   # If not, try linking directly to both of them to see if they are available
   if test "$USE_GNUTLS_NETTLE" = ""; then
-    AC_CHECK_LIB(nettle, nettle_MD5Init, [ USE_GNUTLS_NETTLE=1 ])
+
+    dnl this is with no particular path given
+    CURL_CHECK_PKGCONFIG(nettle)
+
+    if test "$PKGCONFIG" != "no" ; then
+      addlib=`$PKGCONFIG --libs-only-l nettle`
+      addld=`$PKGCONFIG --libs-only-L nettle`
+      addcflags=`$PKGCONFIG --cflags-only-I nettle`
+      version=`$PKGCONFIG --modversion nettle`
+      gtlslib=`echo $addld | $SED -e 's/^-L//'`
+
+      if test -n "$addlib"; then
+
+        CLEANLIBS="$LIBS"
+        CLEANCPPFLAGS="$CPPFLAGS"
+        CLEANLDFLAGS="$LDFLAGS"
+        CLEANLDFLAGSPC="$LDFLAGSPC"
+
+        LIBS="$addlib $LIBS"
+        LDFLAGS="$LDFLAGS $addld"
+        LDFLAGSPC="$LDFLAGSPC $addld"
+        if test "$addcflags" != "-I/usr/include"; then
+          CPPFLAGS="$CPPFLAGS $addcflags"
+        fi
+
+        AC_CHECK_LIB(nettle, nettle_MD5Init,
+        [
+          USE_GNUTLS_NETTLE=1
+        ],
+        [
+          LIBS="$CLEANLIBS"
+          CPPFLAGS="$CLEANCPPFLAGS"
+          LDFLAGS="$CLEANLDFLAGS"
+          LDFLAGSPC="$CLEANLDFLAGSPC"
+        ])
+
+        if test "$USE_GNUTLS_NETTLE" = "1"; then
+          if test -z "$version"; then
+            version="unknown"
+          fi
+          AC_MSG_NOTICE([detected nettle version $version])
+        fi
+      fi
+    fi
+    if test "$USE_GNUTLS_NETTLE" = ""; then
+      AC_MSG_ERROR([GnuTLS found, but nettle was not found])
+    fi
+  else
+    LIBS="-lnettle $LIBS"
   fi
-  if test "$USE_GNUTLS_NETTLE" = ""; then
-    AC_MSG_ERROR([GnuTLS found, but nettle was not found])
+
+  if test "$USE_GNUTLS_NETTLE" = "1"; then
+    LIBCURL_PC_REQUIRES_PRIVATE="$LIBCURL_PC_REQUIRES_PRIVATE nettle"
   fi
-  LIBS="-lnettle $LIBS"
 
   dnl ---
   dnl We require GnuTLS with SRP support.