scorecard: flame graphs and documentation

Add `--flame` option to scorecard.py for generating flame graphs.
Add documentation in docs/internal/SCORECARD.md on how to use this.

Closes #17792

.github/scripts/spellcheck.words

@@ -205,6 +205,7 @@ DoT
 doxygen
 drftpd
 dsa
+dtrace
 Dudka
 Dymond
 dynbuf
@@ -823,12 +824,14 @@ subdirectory
 submitters
 substring
 substrings
+sudo
 SunOS
 SunSSH
 superset
 svc
 svcb
 SVCB
+SVG
 Svyatoslav
 Swisscom
 sws

docs/Makefile.am

@@ -65,6 +65,7 @@ INTERNALDOCS =                                  \
  internals/NEW-PROTOCOL.md                      \
  internals/PORTING.md                           \
  internals/README.md                            \
+ internals/SCORECARD.md                         \
  internals/SPLAY.md                             \
  internals/STRPARSE.md                          \
  internals/TLS-SESSIONS.md                      \

docs/internals/SCORECARD.md

@@ -0,0 +1,71 @@
+<!--
+Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
+
+SPDX-License-Identifier: curl
+-->
+
+# scorecard.py
+
+This is an internal script in `tests/http/scorecard.py` used for testing
+curl's performance in a set of cases. These are for exercising parts of
+curl/libcurl in a reproducible fashion to judge improvements or detect
+regressions. They are not intended to represent real world scenarios
+as such.
+
+This script is not part of any official interface and we may
+change it in the future according to the project's needs.
+
+## setup
+
+When you are able to run curl's `pytest` suite, scorecard should work
+for you as well. They start a local Apache httpd or Caddy server and
+invoke the locally build `src/curl` (by default).
+
+## invocation
+
+A typical invocation for measuring performance of HTTP/2 downloads would be:
+
+```
+curl> python3 tests/http/scorecard.py -d h2
+```
+
+and this prints a table with the results. The last argument is the protocol to test and
+it can be `h1`, `h2` or `h3`. You can add `--json` to get results in JSON instead of text.
+
+Help for all command line options are available via:
+
+```
+curl> python3 tests/http/scorecard.py -h
+```
+
+## scenarios
+
+Apart from `-d/--downloads` there is `-u/--uploads` and `-r/--requests`. These are run with
+a variation of resource sizes and parallelism by default. You can specify these in some way
+if you are just interested in a particular case.
+
+For example, to run downloads of a 1 MB resource only, 100 times with at max 6 parallel transfers, use:
+
+```
+curl> python3 tests/http/scorecard.py -d --download-sizes=1mb --download-count=100 --download-parallel=6 h2
+```
+
+Similar options are available for uploads and requests scenarios.
+
+## dtrace
+
+With the `--dtrace` option, scorecard produces a dtrace sample of the user stacks in `tests/http/gen/curl/curl.user_stacks`. On many platforms, `dtrace` requires **special permissions**. It is therefore invoked via `sudo` and you should make sure that sudo works for the run without prompting for a password.
+
+Note: the file is the trace of the last curl invocation by scorecard. Use the parameters to narrow down the runs to the particular case you are interested in.
+
+## flame graphs
+
+With the excellent [Flame Graph](https://github.com/brendangregg/FlameGraph) by Brendan Gregg, scorecard can turn the `dtrace` samples into an interactive SVG. Set the environment variable `FLAMEGRAPH` to the location of your clone of that project and invoked scorecard with the `--flame` option. Like
+
+```
+curl> FLAMEGRAPH=/Users/sei/projects/FlameGraph python3 tests/http/scorecard.py \
+   -r --request-count=50000 --request-parallels=100 --samples=1 --flame h2
+```
+and the SVG of the run is in `tests/http/gen/curl/curl.flamegraph.svg`. You can open that in Firefox and zoom in/out of stacks of interest.
+
+Note: as with `dtrace`, the flame graph is for the last invocation of curl done by scorecard.

tests/http/scorecard.py

@@ -186,7 +186,8 @@ class ScoreRunner:
                  curl_verbose: int,
                  download_parallel: int = 0,
                  server_addr: Optional[str] = None,
-                 with_dtrace: bool = False):
+                 with_dtrace: bool = False,
+                 with_flame: bool = False):
         self.verbose = verbose
         self.env = env
         self.protocol = protocol
@@ -196,6 +197,7 @@ class ScoreRunner:
         self._silent_curl = not curl_verbose
         self._download_parallel = download_parallel
         self._with_dtrace = with_dtrace
+        self._with_flame = with_flame
 
     def info(self, msg):
         if self.verbose > 0:
@@ -205,7 +207,8 @@ class ScoreRunner:
     def mk_curl_client(self):
         return CurlClient(env=self.env, silent=self._silent_curl,
                           server_addr=self.server_addr,
-                          with_dtrace=self._with_dtrace)
+                          with_dtrace=self._with_dtrace,
+                          with_flame=self._with_flame)
 
     def handshakes(self) -> Dict[str, Any]:
         props = {}
@@ -679,7 +682,8 @@ def run_score(args, protocol):
                                verbose=args.verbose,
                                curl_verbose=args.curl_verbose,
                                download_parallel=args.download_parallel,
-                               with_dtrace=args.dtrace)
+                               with_dtrace=args.dtrace,
+                               with_flame=args.flame)
             cards.append(card)
 
         if test_httpd:
@@ -704,7 +708,8 @@ def run_score(args, protocol):
                                server_port=server_port,
                                verbose=args.verbose, curl_verbose=args.curl_verbose,
                                download_parallel=args.download_parallel,
-                               with_dtrace=args.dtrace)
+                               with_dtrace=args.dtrace,
+                               with_flame=args.flame)
             card.setup_resources(server_docs, downloads)
             cards.append(card)
 
@@ -808,7 +813,9 @@ def main():
     parser.add_argument("--remote", action='store', type=str,
                         default=None, help="score against the remote server at <ip>:<port>")
     parser.add_argument("--dtrace", action='store_true',
-                        default = False, help = "produce dtrace of curl")
+                        default = False, help="produce dtrace of curl")
+    parser.add_argument("--flame", action='store_true',
+                        default = False, help="produce a flame graph on curl, implies --dtrace")
 
     parser.add_argument("-H", "--handshakes", action='store_true',
                         default=False, help="evaluate handshakes only")

tests/http/testenv/curl.py

@@ -114,13 +114,17 @@ class DTraceProfile:
         self._pid = pid
         self._run_dir = run_dir
         self._proc = None
+        self._rc = 0
+        self._file = os.path.join(self._run_dir, 'curl.user_stacks')
 
     def start(self):
+        if os.path.exists(self._file):
+            os.remove(self._file)
         args = [
             'sudo', 'dtrace',
             '-x', 'ustackframes=100',
             '-n', f'profile-97 /pid == {self._pid}/ {{ @[ustack()] = count(); }} tick-60s {{ exit(0); }}',
-            '-o', f'{self._run_dir}/curl.user_stacks'
+            '-o', f'{self._file}'
         ]
         self._proc = subprocess.Popen(args, text=True, cwd=self._run_dir, shell=False)
         assert self._proc
@@ -128,6 +132,11 @@ class DTraceProfile:
     def finish(self):
         if self._proc:
             self._proc.terminate()
+            self._rc = self._proc.returncode
+
+    @property
+    def file(self):
+        return self._file
 
 
 class RunTcpDump:
@@ -490,7 +499,8 @@ class CurlClient:
                  silent: bool = False,
                  run_env: Optional[Dict[str, str]] = None,
                  server_addr: Optional[str] = None,
-                 with_dtrace: bool = False):
+                 with_dtrace: bool = False,
+                 with_flame: bool = False):
         self.env = env
         self._timeout = timeout if timeout else env.test_timeout
         self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl
@@ -500,6 +510,9 @@ class CurlClient:
         self._headerfile = f'{self._run_dir}/curl.headers'
         self._log_path = f'{self._run_dir}/curl.log'
         self._with_dtrace = with_dtrace
+        self._with_flame = with_flame
+        if self._with_flame:
+            self._with_dtrace = True
         self._silent = silent
         self._run_env = run_env
         self._server_addr = server_addr if server_addr else '127.0.0.1'
@@ -792,10 +805,11 @@ class CurlClient:
         exception = None
         profile = None
         tcpdump = None
-        started_at = datetime.now()
+        dtrace = None
         if with_tcpdump:
             tcpdump = RunTcpDump(self.env, self._run_dir)
             tcpdump.start()
+        started_at = datetime.now()
         try:
             with open(self._stdoutfile, 'w') as cout, open(self._stderrfile, 'w') as cerr:
                 if with_profile:
@@ -824,8 +838,6 @@ class CurlClient:
                             ptimeout = 0.01
                     exitcode = p.returncode
                     profile.finish()
-                    if self._with_dtrace:
-                        dtrace.finish()
                     log.info(f'done: exit={exitcode}, profile={profile}')
                 else:
                     p = subprocess.run(args, stderr=cerr, stdout=cout,
@@ -841,13 +853,18 @@ class CurlClient:
                         f'(configured {self._timeout}s): {args}')
             exitcode = -1
             exception = 'TimeoutExpired'
+        ended_at = datetime.now()
         if tcpdump:
             tcpdump.finish()
+        if dtrace:
+            dtrace.finish()
+        if self._with_flame and dtrace:
+            self._generate_flame(dtrace, args)
         coutput = open(self._stdoutfile).readlines()
         cerrput = open(self._stderrfile).readlines()
         return ExecResult(args=args, exit_code=exitcode, exception=exception,
                           stdout=coutput, stderr=cerrput,
-                          duration=datetime.now() - started_at,
+                          duration=ended_at - started_at,
                           with_stats=with_stats,
                           profile=profile, tcpdump=tcpdump)
 
@@ -976,3 +993,52 @@ class CurlClient:
 
         fin_response(response)
         return r
+
+    def _generate_flame(self, dtrace: DTraceProfile, curl_args: List[str]):
+        log.info('generating flame graph from dtrace for this run')
+        if not os.path.exists(dtrace.file):
+            raise Exception(f'dtrace output file does not exist: {dtrace.file}')
+        if 'FLAMEGRAPH' not in os.environ:
+            raise Exception('Env variable FLAMEGRAPH not set')
+        fg_dir = os.environ['FLAMEGRAPH']
+        if not os.path.exists(fg_dir):
+            raise Exception(f'FlameGraph directory not found: {fg_dir}')
+
+        fg_collapse = os.path.join(fg_dir, 'stackcollapse.pl')
+        if not os.path.exists(fg_collapse):
+            raise Exception(f'FlameGraph script not found: {fg_collapse}')
+
+        fg_gen_flame = os.path.join(fg_dir, 'flamegraph.pl')
+        if not os.path.exists(fg_gen_flame):
+            raise Exception(f'FlameGraph script not found: {fg_gen_flame}')
+
+        file_collapsed = f'{dtrace.file}.collapsed'
+        file_svg = os.path.join(self._run_dir, 'curl.flamegraph.svg')
+        file_err = os.path.join(self._run_dir, 'curl.flamegraph.stderr')
+        log.info('waiting a sec for dtrace to finish flusheing its buffers')
+        time.sleep(1)
+        log.info(f'collapsing stacks into {file_collapsed}')
+        with open(file_collapsed, 'w') as cout, open(file_err, 'w') as cerr:
+            p = subprocess.run([
+                fg_collapse, dtrace.file
+            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+            rc = p.returncode
+            if rc != 0:
+                raise Exception(f'{fg_collapse} returned error {rc}')
+        log.info(f'generating graph into {file_svg}')
+        cmdline = ' '.join(curl_args)
+        if len(cmdline) > 80:
+            title = f'{cmdline[:80]}...'
+            subtitle = f'...{cmdline[-80:]}'
+        else:
+            title = cmdline
+            subtitle = ''
+        with open(file_svg, 'w') as cout, open(file_err, 'w') as cerr:
+            p = subprocess.run([
+                fg_gen_flame, '--colors', 'green',
+                '--title', title, '--subtitle', subtitle,
+                file_collapsed
+            ], stdout=cout, stderr=cerr, cwd=self._run_dir, shell=False)
+            rc = p.returncode
+            if rc != 0:
+                raise Exception(f'{fg_gen_flame} returned error {rc}')