ci(conda): move platform pins up and give them a review-by date (#1851)

.github/workflows/cron-conda.yml

@@ -161,14 +161,30 @@ jobs:
       # extra, walks what they require through the installed metadata, and
       # pins each package at the version installed here, but for its
       # `PLATFORM_PINS` (#1827). So this install is the step that resolves the
-      # newest releases, and the script itself reads nothing from the network.
+      # newest releases, and of the script's runs only the next one reads the
+      # network, through `requests`, which `.[all]` does not bring.
       - name: Install and Setup
         run: |
           set -x
 
-          python -m pip install -U pip setuptools wheel packaging
+          python -m pip install -U pip setuptools wheel packaging requests
           python -m pip install -e .[all]
 
+      # #1851: asks PyPI for each `PLATFORM_PINS` package's releases and moves
+      # its pin in conda/platform-pins.toml up to the newest release, up to the
+      # one installed above, with a wheel for the pinned platform, renewing its
+      # review-by date; a pin that reaches the installed release, or already
+      # pins it, goes.
+      # The step below then writes the moved pin into conda/requirements.txt,
+      # and Commit Changes commits both. The table is not one of
+      # util/verdict_covers_checkout.py's bot-managed paths, which says why:
+      # the workflows chained off Unit Tests wait for such a commit's own run
+      # (#1643). A package PyPI cannot be read for keeps its pin, with a
+      # warning, and a pin past its review-by date gets one too; either way the
+      # step passes.
+      - name: Refresh the platform pins
+        run: python util/conda_vendor.py --refresh-pins
+
       - name: Setup Conda Distribution
         run: |
           export PCAPKIT_DEVMODE=1

conda/platform-pins.toml

@@ -0,0 +1,18 @@
+# The packages conda/requirements.txt pins apart on some conda platforms, as a
+# table per package and conda subdir (#1827): the version that platform vendors
+# in place of the installed release, the date the pin is due for review by, and
+# why the platform cannot take the installed release.
+#
+# util/conda_vendor.py reads this file. Conda Update's `conda-update` job runs
+# its `--refresh-pins` first, which asks PyPI for the newest release, up to the
+# installed one, with a wheel each conda Python installs on the pinned platform.
+# It moves the pin up to that release with a fresh review-by date, and drops a
+# pin that reaches the installed release or already pins it (#1851). `--check`
+# fails on a pin past its review-by date: check by hand that the pin still
+# holds, then set a new date. The job writes this whole file, this comment
+# included, so anything a pin needs said goes in its `reason`.
+
+[cryptography.osx-64]
+version = "48.0.1"
+review-by = 2027-01-15
+reason = "Its newest releases ship no Intel or universal2 macOS wheel, so the osx-64 build would compile them from source; 49.0.0 was the first (#1827)."

tests/project/test_conda_cryptography_pin_1827.py

@@ -1,19 +1,23 @@
 # -*- coding: utf-8 -*-
 """cryptography is pinned apart on Intel macOS, which has no wheel of its newest releases (#1827).
 
-* :file:`conda/requirements.txt` carries a cryptography line per platform pin,
-  and on each conda build exactly one of them applies: 48.0.1 on ``osx-64``,
-  the installed release elsewhere.
+* :file:`conda/requirements.txt` carries a line per platform pin of each package
+  :file:`conda/platform-pins.toml` pins, and of cryptography whether or not it
+  is, and on each conda build exactly one of a package's lines applies: the
+  table's pin on its platform, the installed release elsewhere. Read from the
+  table rather than spelt here, since the weekly Conda Update moves and drops
+  pins (#1851).
 * :file:`util/conda_vendor.py` writes those lines from ``PLATFORM_PINS``, and
   ``--check`` passes on them and fails when the table entry or the file's pin
   goes, when the pin moves, and when the table names a package or platform no
-  build has. All offline, on stand-in metadata.
+  build has. All offline, on stand-in metadata and a stand-in 48.0.1 pin.
 
 """
 
 from __future__ import annotations
 
 import contextlib
+import datetime
 import importlib.util
 import io
 import pathlib
@@ -33,6 +37,7 @@
 
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 REQUIREMENTS = ROOT / 'conda' / 'requirements.txt'
+#: The Intel macOS line :func:`stand_in`'s pin writes.
 INTEL = 'cryptography==48.0.1; sys_platform == "darwin" and platform_machine == "x86_64"'
 #: The other line, whose pin the weekly Conda Update moves.
 REST = re.compile(r'cryptography==(\S+); sys_platform != "darwin" or platform_machine != "x86_64"')
@@ -59,45 +64,74 @@ def committed() -> 'list[str]':
             if line.startswith('cryptography')]
 
 
+def pinned(*pins: 'tuple[str, str, str]') -> 'dict[str, dict[str, Any]]':
+    """A ``PLATFORM_PINS`` of (package, conda subdir, version), with a stand-in date and reason."""
+    module, table = vendor(), {}  # type: Any, dict[str, dict[str, Any]]
+    for name, subdir, version in pins:
+        table.setdefault(name, {})[subdir] = module.Pin(version, datetime.date(2027, 1, 15), 'a stand-in')
+    return table
+
+
+def stand_in() -> 'Any':
+    """``PLATFORM_PINS`` patched to cryptography 48.0.1 on osx-64, as #1827 committed it."""
+    return mock.patch.object(vendor(), 'PLATFORM_PINS', pinned(('cryptography', 'osx-64', '48.0.1')))
+
+
+def written() -> 'list[str]':
+    """The cryptography lines the writer gives :func:`stand_in`'s pin, with 50.0.2 installed."""
+    module = vendor()
+    envs = module.environments(module.conda_pythons())
+    with stand_in():
+        return module.lines(cryptography(module, envs), envs, lambda name: '50.0.2')
+
+
 def cryptography(module: 'Any', envs: 'list[dict[str, str]]') -> 'dict[str, Any]':
     """What the conda builds vendor for ``pyproject.toml``'s cryptography line, on stand-in metadata."""
     return module.derive([req for req in module.direct() if req.name == 'cryptography'], envs, lambda name: [])
 
 
 class CommittedLinesTests(unittest.TestCase):
-    """The committed cryptography lines, evaluated for each conda build."""
+    """The committed lines of each pinned package, and of cryptography, evaluated for each conda build."""
 
-    def test_each_build_reads_one_cryptography_line(self) -> None:
+    def test_each_build_reads_one_line_per_pinned_package(self) -> None:
         from packaging.requirements import Requirement
         module = vendor()
-        lines = committed()
-        self.assertIn(INTEL, lines)
-        self.assertEqual(len([line for line in lines if REST.fullmatch(line)]), 1, lines)
-        self.assertEqual(module.PLATFORM_PINS, {'cryptography': {'osx-64': '48.0.1'}})
-        for env in module.environments(module.conda_pythons()):
-            subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items()
-                          if (sys_platform, machine) == (env['sys_platform'], env['platform_machine']))
-            with self.subTest(subdir=subdir, python=env['python_version']):
-                live = [line for line in lines if Requirement(line).marker.evaluate(env)]
-                self.assertEqual(len(live), 1, live)
-                self.assertEqual(live[0] == INTEL, subdir == 'osx-64', live)
+        lines = [Requirement(line) for line in REQUIREMENTS.read_text(encoding='utf-8').splitlines()
+                 if line.split('#', 1)[0].strip()]
+        for name in sorted({'cryptography', *module.PLATFORM_PINS}):
+            pins = module.PLATFORM_PINS.get(name, {})
+            reads = {}  # type: dict[tuple[str, str], str]
+            for env in module.environments(module.conda_pythons()):
+                subdir = next(subdir for subdir, (sys_platform, _, machine) in module.PLATFORMS.items()
+                              if (sys_platform, machine) == (env['sys_platform'], env['platform_machine']))
+                with self.subTest(package=name, subdir=subdir, python=env['python_version']):
+                    live = [str(req) for req in lines
+                            if req.name == name and (req.marker is None or req.marker.evaluate(env))]
+                    self.assertEqual(len(live), 1, live)
+                    reads[subdir, env['python_version']] = live[0]
+                    if subdir in pins:
+                        self.assertEqual(str(Requirement(live[0]).specifier), f'=={pins[subdir].version}')
+            # The builds no pin names read none of the pinned lines.
+            on_pinned = {line for (subdir, _), line in reads.items() if subdir in pins}
+            self.assertFalse({line for (subdir, _), line in reads.items() if subdir not in pins} & on_pinned,
+                             reads)
 
 
 class WriterTests(unittest.TestCase):
     """The writer emits a line per platform pin."""
 
     def test_the_writer_emits_the_platform_lines(self) -> None:
-        module = vendor()
-        envs = module.environments(module.conda_pythons())
-        self.assertEqual(module.lines(cryptography(module, envs), envs, lambda name: '50.0.2'),
+        self.assertEqual(written(),
                          [INTEL, 'cryptography==50.0.2; sys_platform != "darwin" or platform_machine != "x86_64"'])
+        self.assertIsNotNone(REST.fullmatch(written()[1]))
 
     def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None:
         from packaging.requirements import Requirement
         module = vendor()
         envs = module.environments(['3.11', '3.12'])
         found = module.derive([Requirement('cryptography; python_version >= "3.12"')], envs, lambda name: [])
-        lines = module.lines(found, envs, lambda name: '50.0.2')
+        with stand_in():
+            lines = module.lines(found, envs, lambda name: '50.0.2')
         self.assertEqual(lines, [
             'cryptography==48.0.1; (python_version == "3.12") and '
             '(sys_platform == "darwin" and platform_machine == "x86_64")',
@@ -111,38 +145,45 @@ def test_a_python_marker_is_kept_beside_the_platform_one(self) -> None:
 
 
 class CheckTests(unittest.TestCase):
-    """``--check`` against the committed cryptography lines."""
+    """``--check`` against the committed cryptography lines, and against :func:`stand_in`'s."""
 
     def problems(self, text: 'str') -> 'list[str]':
         module = vendor()
         envs = module.environments(module.conda_pythons())
         return module.compare(text, cryptography(module, envs), envs)
 
     def test_the_committed_lines_pass(self) -> None:
-        lines = committed()
-        self.assertIn(INTEL, lines)
-        self.assertEqual(self.problems('\n'.join(lines)), [])
+        self.assertEqual(self.problems('\n'.join(committed())), [])
+
+    def test_the_stand_in_lines_pass(self) -> None:
+        lines = written()
+        with stand_in():
+            self.assertEqual(self.problems('\n'.join(lines)), [])
 
     def test_without_the_override_the_platform_lines_fail(self) -> None:
+        lines = written()
         with mock.patch.object(vendor(), 'PLATFORM_PINS', {}):
-            problems = self.problems('\n'.join(committed()))
+            problems = self.problems('\n'.join(lines))
         self.assertEqual(len(problems), 1, problems)
         self.assertTrue(problems[0].startswith('marker: cryptography has ['), problems)
 
     def test_a_dropped_or_moved_platform_pin_fails(self) -> None:
-        rest = [line for line in committed() if line != INTEL]
-        dropped = self.problems('\n'.join(rest))
+        rest = [line for line in written() if line != INTEL]
+        with stand_in():
+            dropped = self.problems('\n'.join(rest))
+            unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1']))
+            moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')]))
         self.assertEqual([problem.split(':')[0] for problem in dropped], ['marker'], dropped)
-        unmarked = self.problems('\n'.join(rest + ['cryptography==48.0.1']))
         self.assertEqual([problem.split(':')[0] for problem in unmarked], ['marker'], unmarked)
-        moved = self.problems('\n'.join(rest + [INTEL.replace('48.0.1', '49.0.0')]))
         self.assertEqual(moved, ["pin: cryptography pins '49.0.0' where sys_platform == \"darwin\" and "
                                  "platform_machine == \"x86_64\", and PLATFORM_PINS '48.0.1'"])
 
     def test_an_override_no_build_can_use_fails(self) -> None:
-        pins = {'cryptography': {'osx-64': '48.0.1', 'win-64': '1.0'}, 'gone': {'osx-64': '1.0'}}
+        lines = written()
+        pins = pinned(('cryptography', 'osx-64', '48.0.1'), ('cryptography', 'win-64', '1.0'),
+                      ('gone', 'osx-64', '1.0'))
         with mock.patch.object(vendor(), 'PLATFORM_PINS', pins):
-            problems = self.problems('\n'.join(committed()))
+            problems = self.problems('\n'.join(lines))
         self.assertEqual(problems, ['override: cryptography is pinned for win-64, which no conda leg builds',
                                     'override: gone is pinned for osx-64, and no conda build vendors it'])
 
@@ -155,7 +196,7 @@ def test_the_writer_refuses_an_override_no_build_can_use(self) -> None:
             module._requires = lambda name: []
             module._version = lambda name: '1.0'
             try:
-                with mock.patch.object(module, 'PLATFORM_PINS', {'gone': {'osx-64': '1.0'}}), \
+                with mock.patch.object(module, 'PLATFORM_PINS', pinned(('gone', 'osx-64', '1.0'))), \
                         contextlib.redirect_stderr(io.StringIO()) as errors:
                     self.assertEqual(module.main([]), 1)
                 self.assertFalse(target.exists())