ci(util): fetch with requests and poll memory with psutil in the scripts (#1894)

.github/workflows/ci-timing.yml

@@ -54,11 +54,15 @@ jobs:
         id: python-versions
         run: python3 util/python_versions.py --github-output
 
-      # Nothing is installed: the script is standard library only.
       - uses: actions/setup-python@v7
         with:
           python-version: ${{ steps.python-versions.outputs.latest }}
 
+      # The script fetches with requests and parses the JUnit artifacts with lxml
+      # (#1894), and needs nothing else outside the standard library.
+      - name: Install requests and lxml
+        run: python -m pip install requests lxml
+
       # The newest earlier run that uploaded a report, so a week whose run failed
       # on a HARD threshold is still next week's baseline. A first run, or one
       # after 90 days without a report, has none and says so.

.github/workflows/cron-python-versions.yml

@@ -57,9 +57,11 @@ jobs:
           python-version: ${{ steps.python-versions.outputs.latest }}
 
       # PyYAML, from the `test` extra, for the workflow readers and the audit;
-      # neither imports pcapkit.
-      - name: Install PyYAML
-        run: python -m pip install PyYAML
+      # neither imports pcapkit. requests fetches the sources and markdown-it-py
+      # reads the runner-images table (#1894); only `update` imports either, so
+      # `pr`'s `publish` runs without them.
+      - name: Install PyYAML, requests and markdown-it-py
+        run: python -m pip install PyYAML requests markdown-it-py
 
       # Fetches the sources, rewrites the files, and writes the proposal to
       # $RUNNER_TEMP/proposal. `publish` is false when nothing changed, or when

.github/workflows/cron-vendor.yml

@@ -264,8 +264,9 @@ jobs:
       # zero once it has decided; if this step fails or times out instead, the
       # tree may hold modules no run passed with, and the next step discards
       # every regenerated module. The test extra is installed here because only
-      # this step needs it. The script's budget bounds its runs together, and
-      # `timeout-minutes` sits above it as the backstop. Run serially, pytest
+      # this step needs it; it carries psutil too, which the script samples a
+      # run's memory with (#1894). The script's budget bounds its runs together,
+      # and `timeout-minutes` sits above it as the backstop. Run serially, pytest
       # peaked at 0.7 GB resident over tests/const, measured locally, so the
       # memory cap leaves headroom on this runner; files are capped at 2 GiB, as
       # in experimental.yml.

.github/workflows/unit-tests.yml

@@ -472,8 +472,8 @@ jobs:
 
       # The Scapy extra is there for examples/generators/pcap.py and legacy.py,
       # which build their captures with scapy; examples/generators/pcapng.py needs
-      # nothing beyond the standard library and pcapkit itself, which it uses to
-      # parse each fixture back. DPKT joins it so the dpkt-gated methods
+      # requests (#1894), from the test extra, and pcapkit itself, which it uses
+      # to parse each fixture back. DPKT joins it so the dpkt-gated methods
       # living under tests/integration/ (test_engine_parity.py,
       # test_engine_runtime.py) actually run here instead of skipping -- see
       # #729; same reasoning as the `test` job above.

Pipfile

@@ -44,6 +44,10 @@ libpcap = {version = ">=1.11.0b29",markers = "python_version >= '3.10'"}
 pypcapfile = "*"
 beautifulsoup4 = {extras = ["html5lib"],version = "*"}
 requests = {extras = ["socks"],version = "*"}
+# The markers match the test extra in pyproject.toml (#1894).
+psutil = {version = "*",markers = "platform_python_implementation == 'CPython'"}
+lxml = {version = "*",markers = "platform_python_implementation != 'GraalVM'"}
+markdown-it-py = "*"
 autopep8 = "*"
 bandit = "*"
 isort = "*"

examples/generators/pcapng.py

@@ -6,13 +6,14 @@
 out of the repository. This module puts them back on any machine, without a
 checkout of anything private, by one of these routes:
 
-* **Downloaded** -- fetched from a public source, with the source URL and the
-  SHA-256 of the exact bytes pinned below. The digest is checked after every
-  download and a mismatch is a hard error, never a warning. An HTTP 429 or
-  5xx, or a timeout, is retried a few times with backoff (see
-  :func:`_download`). If the network is still unavailable the download
-  degrades to a synthesised stand-in, and both the console output and the
-  return value say so rather than pretending otherwise.
+* **Downloaded** -- fetched from a public source with :mod:`requests`, with the
+  source URL and the SHA-256 of the exact bytes pinned below. The digest is
+  checked after every download and a mismatch is a hard error, never a warning.
+  An HTTP 429 or 5xx, or a timeout, is retried a few times with backoff (see
+  :func:`_download`). If the network is still unavailable, or :mod:`requests`
+  is not installed, the download degrades to a synthesised stand-in, and both
+  the console output and the return value say so rather than pretending
+  otherwise.
 * **Synthesised** -- written byte by byte by :class:`_Blocks` below, with no
   network access whatsoever, from packet payloads carried in the committed
   ``examples/captures/dhcp.pcapng`` fixture. Generation is deterministic: the same input
@@ -162,8 +163,6 @@
 import struct
 import sys
 import time
-import urllib.error
-import urllib.request
 from typing import TYPE_CHECKING, NamedTuple
 
 if TYPE_CHECKING:
@@ -836,11 +835,22 @@ def _transient(exc: 'OSError') -> 'bool':
     server's final answer, and an unreachable host fails at once rather than
     after a wait, which keeps an offline run as quick as it was.
 
+    :mod:`requests` raises a :exc:`~requests.Timeout` for a timeout while it
+    connects or waits for the headers, but a timeout while the body is read
+    reaches here as a :exc:`~requests.ConnectionError` wrapping urllib3's
+    :exc:`~urllib3.exceptions.ReadTimeoutError`.
+
     """
-    if isinstance(exc, urllib.error.HTTPError):
-        return exc.code == 429 or 500 <= exc.code <= 599
-    reason = exc.reason if isinstance(exc, urllib.error.URLError) else exc
-    return isinstance(reason, (socket.timeout, TimeoutError))
+    import requests  # pylint: disable=import-outside-toplevel
+    import urllib3  # pylint: disable=import-outside-toplevel
+
+    if isinstance(exc, requests.HTTPError):
+        code = getattr(exc.response, 'status_code', None)
+        return code is not None and (code == 429 or 500 <= code <= 599)
+    if isinstance(exc, (requests.Timeout, socket.timeout, TimeoutError)):
+        return True
+    reason = exc.args[0] if isinstance(exc, requests.ConnectionError) and exc.args else None
+    return isinstance(reason, urllib3.exceptions.ReadTimeoutError)
 
 
 def _retry_after(exc: 'OSError', now: 'float') -> 'Optional[float]':
@@ -857,8 +867,8 @@ def _retry_after(exc: 'OSError', now: 'float') -> 'Optional[float]':
         The wait, or :data:`None` if there is no header or it does not parse.
 
     """
-    headers = getattr(exc, 'headers', None)
-    value = None if headers is None else headers.get('Retry-After')
+    response = getattr(exc, 'response', None)
+    value = None if response is None else response.headers.get('Retry-After')
     if value is None:
         return None
     value = value.strip()
@@ -893,14 +903,19 @@ def _download(url: 'str') -> 'bytes':
     Raises:
         OSError: The last failure, once the retries are over or for a failure
             that is not retried -- :func:`_materialise` then falls back to a
-            stand-in (#1658).
+            stand-in (#1658). Every :exc:`requests.RequestException` is one.
+        ImportError: :mod:`requests` is not installed, which
+            :func:`_materialise` treats as it does an unreachable host.
 
     """
+    import requests  # pylint: disable=import-outside-toplevel
+
     attempt, waited = 1, 0.0
     while True:
         try:
-            with urllib.request.urlopen(url, timeout=TIMEOUT) as response:  # nosec: B310
-                return response.read()
+            response = requests.get(url, timeout=TIMEOUT)
+            response.raise_for_status()
+            return response.content
         except OSError as exc:
             if attempt >= DOWNLOAD_ATTEMPTS or not _transient(exc):
                 raise
@@ -946,8 +961,10 @@ def _materialise(fixture: '_Fixture', path: 'pathlib.Path') -> 'tuple[bytes, str
 
     try:
         data = _download(fixture.url)
-    except (urllib.error.URLError, OSError, ValueError) as exc:
-        print(f'    ! {fixture.url} is unreachable ({exc});')
+    except (ImportError, OSError, ValueError) as exc:
+        why = ('cannot be fetched without requests' if isinstance(exc, ImportError)
+               else 'is unreachable')
+        print(f'    ! {fixture.url} {why} ({exc});')
         print(f'    ! synthesising a stand-in for {fixture.name} instead -- the '
               'upstream capture was NOT used')
         return fixture.build(), 'synthesised (download unavailable)'

pyproject.toml

@@ -363,6 +363,22 @@ test = [
     # Test-only, as ``pcapkit`` never imports ``yaml``. Unpinned, since
     # ``safe_load`` is all they use.
     "PyYAML",
+    # What util/vendor_test_gate.py samples a test run's memory with (#1894),
+    # here because cron-vendor.yml's gate step installs this extra and nothing
+    # else. CPython only: PyPI has wheels for every CPython leg, free-threaded
+    # 3.14t included, and none for PyPy or GraalPy, where the gate falls back to
+    # ``ps``. Unpinned, like the entries above.
+    "psutil; platform_python_implementation == 'CPython'",
+    # What util/update_python_versions.py reads the runner-images README's table
+    # with (#1894), for the tests under ``tests/project/`` that run it on the
+    # saved README. A pure-Python wheel; unpinned, like the entries above.
+    "markdown-it-py",
+    # The strict XML parser util/ci_timing.py reads JUnit with and
+    # util/pyshark_encap_map.py reads PDML with (#1894). PyPI has wheels for
+    # CPython and PyPy but none for GraalPy, which reports its implementation as
+    # ``GraalVM``; the tests that parse skip there, and the scripts do not fall
+    # back to a laxer parser. Unpinned, like the entries above.
+    "lxml; platform_python_implementation != 'GraalVM'",
 ]
 
 # setuptools specific

tests/_dependency_gates.py

@@ -250,6 +250,15 @@
     # The theme ``HAS_FURO`` asks for, whose table wrapper the #1861 transform
     # test runs. Only the ``docs`` extra carries it.
     'furo': ('furo',),
+    # The strict XML parser ``HAS_LXML`` asks for (#1894): util/ci_timing.py
+    # reads JUnit and util/pyshark_encap_map.py reads PDML with it, and neither
+    # falls back to a laxer parser. The ``test`` extra carries it with a
+    # ``platform_python_implementation != 'GraalVM'`` marker, because PyPI has
+    # no wheel for GraalPy, so the gate skips on experimental.yml's GraalPy leg.
+    # That is no :data:`DEPENDENCY_GATE_EXCLUSIONS` entry: this module does not
+    # model markers and does not read experimental.yml, so every job it reads
+    # satisfies the gate, and an entry would describe a gap that is not there.
+    'lxml': ('lxml',),
 }
 
 #: Flags whose condition is not a distribution, with what it is instead. A flag