diff --git a/.github/actions/build_pypi_dist/action.yml b/.github/actions/build_pypi_dist/action.yml index 87f25b1..43b416f 100644 --- a/.github/actions/build_pypi_dist/action.yml +++ b/.github/actions/build_pypi_dist/action.yml @@ -207,6 +207,11 @@ runs: before_macos=() env_linux=() env_macos=("MACOSX_DEPLOYMENT_TARGET=11.0") + # Extra --exclude arguments for the repair step (see "Repair command" + # below); the needs_* branches append system libraries the wheel + # must not vendor. + repair_exclude_linux=() + repair_exclude_macos=() # gcc-toolset-14, unconditionally. The pinned manylinux_2_28 image # ships gcc-toolset-13 (13.3.1), which has a constraint-satisfaction @@ -289,9 +294,7 @@ runs: # *through* libmpi in this tree, excluding libmpi.so.40 alone is # sufficient -- auditwheel doesn't recurse into an excluded # library's own dependencies. - { - echo "CIBW_REPAIR_WHEEL_COMMAND_LINUX=auditwheel repair --exclude 'libmpi.so.40' --exclude 'libopen-rte.so.40' --exclude 'libopen-pal.so.40' -w {dest_dir} {wheel}" - } >> "$GITHUB_ENV" + repair_exclude_linux+=(libmpi.so.40 libopen-rte.so.40 libopen-pal.so.40) before_macos+=("brew install open-mpi") # macOS needs the same MPI exclusion as Linux above, for a second, # macOS-specific reason on top of the bin/orted one. @@ -321,9 +324,7 @@ runs: # one OpenMPI in the process. Substring matching (delocate matches # --exclude against the library name), unlike auditwheel's exact # sonames above. - { - echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=delocate-wheel --require-archs {delocate_archs} --exclude libmpi --exclude libopen-pal --exclude libopen-rte --exclude libpmix --exclude libevent --exclude libhwloc -w {dest_dir} -v {wheel}" - } >> "$GITHUB_ENV" + repair_exclude_macos+=(libmpi libopen-pal libopen-rte libpmix libevent libhwloc) # Homebrew's open-mpi (and its own deps: libpmix, libhwloc, # libevent) are built against the actual runner OS, so their # dylibs report a minimum target matching it (14.0 on macos-14) -- @@ -424,7 +425,34 @@ runs: echo "$out" } + # Repair command. Every leg goes through repair_wheel.py, not only + # needs_mpi ones: any package with ecosystem dependencies links + # libraries (libparallelzone, libspdlog, ...) that its dependency + # wheels already install, and auditwheel/delocate would otherwise + # vendor a second copy of each -- or, since the build env holding + # those wheels is gone by repair time, fail to find them at all. + # See repair_wheel.py. cibuildwheel has no {project} placeholder for + # this command, so the script is copied into the checkout root, the + # repair step's working directory on both platforms (/project inside + # the Linux container). The sdist was already built above, so the + # copy can't leak into it. + cp "${GITHUB_ACTION_PATH}/repair_wheel.py" .nwx_repair_wheel.py + repair_args=() + if [ -n "${{ inputs.extra_index_url }}" ]; then + repair_args+=(--extra-index-url "${{ inputs.extra_index_url }}") + fi + repair_linux="python .nwx_repair_wheel.py --tool auditwheel ${repair_args[*]}" + for lib in "${repair_exclude_linux[@]}"; do + repair_linux+=" --exclude '${lib}'" + done + repair_macos="python .nwx_repair_wheel.py --tool delocate --archs {delocate_archs} ${repair_args[*]}" + for lib in "${repair_exclude_macos[@]}"; do + repair_macos+=" --exclude '${lib}'" + done + { + echo "CIBW_REPAIR_WHEEL_COMMAND_LINUX=${repair_linux} {wheel} {dest_dir}" + echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=${repair_macos} {wheel} {dest_dir}" echo "CIBW_BEFORE_ALL_LINUX=$(join_and "${before_linux[@]}")" echo "CIBW_BEFORE_ALL_MACOS=$(join_and "${before_macos[@]}")" echo "CIBW_ENVIRONMENT_LINUX=${env_linux[*]}" diff --git a/.github/actions/build_pypi_dist/repair_wheel.py b/.github/actions/build_pypi_dist/repair_wheel.py new file mode 100644 index 0000000..344488a --- /dev/null +++ b/.github/actions/build_pypi_dist/repair_wheel.py @@ -0,0 +1,175 @@ +# Copyright 2026 NWChemEx-Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Repair a wheel without re-vendoring its ecosystem dependencies. + +cibuildwheel's repair step (auditwheel on Linux, delocate on macOS) copies +every shared library a wheel links against into the wheel. For an ecosystem +package that is wrong: e.g. libpluginplay links libparallelzone, which the +nwchemex-parallelzone wheel already installs into the same site-packages/lib. +Vendoring a second copy brings back the problem the per-project install +components fixed: two copies of one library, possibly different versions, +both loaded into one process. + +Excluding those libraries is not enough on its own. Both tools refuse to +repair a wheel when a dependency can't be found at all ("Could not find all +dependencies"), whether or not it is excluded, and the build env holding the +dependency wheels is gone by the time the repair runs. So this script: + +1. reads the wheel's own Requires-Dist for nwchemex-* packages, +2. installs them (newest versions, with their dependencies) into a throwaway + directory, +3. puts that directory's lib/ on the loader's search path, so the repair tool + can resolve the libraries, and +4. excludes every shared library those wheels ship, so none is copied. + +cibuildwheel offers no {project} placeholder for the repair command, so the +action copies this file into the checkout root, which is the repair step's +working directory on both Linux (/project in the container) and macOS. + +Usage: + python repair_wheel.py --tool {auditwheel,delocate} [--archs ARCHS] + [--exclude NAME ...] [--extra-index-url URL] WHEEL DEST_DIR +""" + +import argparse +import os +import re +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + +# Requirement names this script treats as ecosystem packages. nwxcmake is +# pure Python and ships no libraries, so including it is harmless. +_ECOSYSTEM_PREFIX = "nwchemex-" + +# A shared library's file name: libfoo.so, libfoo.so.1.2, libfoo.dylib, +# libfoo.1.2.dylib. Wheels don't keep symlinks, so every soname variant is +# its own file and all of them get excluded. +_SHARED_LIB_RE = re.compile(r"^lib.+\.(so(\.[0-9.]+)?|([0-9.]+\.)?dylib)$") + + +def _ecosystem_requirements(wheel): + """The nwchemex-* requirements in the wheel's METADATA, unparsed.""" + with zipfile.ZipFile(wheel) as zf: + metadata = next( + n for n in zf.namelist() if n.endswith(".dist-info/METADATA") + ) + text = zf.read(metadata).decode() + reqs = [] + for line in text.splitlines(): + if not line.startswith("Requires-Dist:"): + continue + req = line.split(":", 1)[1].strip() + # Skip optional extras (e.g. a [dev] dependency): they are not + # runtime dependencies of the libraries being repaired. + if "extra ==" in req: + continue + if req.lower().startswith(_ECOSYSTEM_PREFIX): + reqs.append(req) + return reqs + + +def _install_dependencies(reqs, target, extra_index_url): + """Install the requirements (and their dependencies) into target.""" + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--quiet", + "--no-compile", + "--only-binary", + ":all:", + "--target", + str(target), + ] + if extra_index_url: + cmd += ["--extra-index-url", extra_index_url] + subprocess.run(cmd + reqs, check=True) + + +def _shared_libs(root): + """File names of every shared library under root.""" + return sorted( + { + p.name + for p in Path(root).rglob("*") + if p.is_file() and _SHARED_LIB_RE.match(p.name) + } + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--tool", choices=("auditwheel", "delocate"), required=True + ) + parser.add_argument( + "--archs", default="", help="delocate --require-archs value" + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + help="extra library to exclude (e.g. MPI)", + ) + parser.add_argument("--extra-index-url", default="") + parser.add_argument("wheel") + parser.add_argument("dest_dir") + args = parser.parse_args() + + excludes = list(args.exclude) + env = dict(os.environ) + with tempfile.TemporaryDirectory() as deps_dir: + reqs = _ecosystem_requirements(args.wheel) + if reqs: + print(f"repair_wheel: ecosystem dependencies: {reqs}", flush=True) + _install_dependencies(reqs, deps_dir, args.extra_index_url) + provided = _shared_libs(deps_dir) + print( + f"repair_wheel: provided by dependency wheels, not " + f"vendored: {provided}", + flush=True, + ) + excludes += provided + # The dependency wheels' libraries live in /lib, + # a few also flat at next to their extension. + search = [str(Path(deps_dir) / "lib"), deps_dir] + var = ( + "LD_LIBRARY_PATH" + if args.tool == "auditwheel" + else "DYLD_LIBRARY_PATH" + ) + env[var] = os.pathsep.join( + search + ([env[var]] if env.get(var) else []) + ) + + if args.tool == "auditwheel": + cmd = ["auditwheel", "repair"] + else: + cmd = ["delocate-wheel", "-v"] + if args.archs: + cmd += ["--require-archs", args.archs] + for name in excludes: + cmd += ["--exclude", name] + cmd += ["-w", args.dest_dir, args.wheel] + print(f"repair_wheel: {' '.join(cmd)}", flush=True) + return subprocess.run(cmd, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main())