diff --git a/build/fbcode_builder/getdeps/buildopts.py b/build/fbcode_builder/getdeps/buildopts.py index d7086af61..a0342f5db 100644 --- a/build/fbcode_builder/getdeps/buildopts.py +++ b/build/fbcode_builder/getdeps/buildopts.py @@ -266,6 +266,7 @@ def get_context_generator( "os": host_type.ostype, "distro": host_type.distro, "distro_vers": host_type.distrovers, + "distro_family": host_type.distro_family, "fb": "on" if self.facebook_internal else "off", "fbsource": "on" if self.fbsource_dir else "off", "test": "off", diff --git a/build/fbcode_builder/getdeps/cli.py b/build/fbcode_builder/getdeps/cli.py index b719cc1e4..d85d84ff1 100644 --- a/build/fbcode_builder/getdeps/cli.py +++ b/build/fbcode_builder/getdeps/cli.py @@ -314,7 +314,7 @@ def setup_project_cmd_parser(self, parser): parser.add_argument( "--distro", help="Filter to just this distro to run", - choices=["ubuntu", "centos_stream", "fedora"], + choices=["ubuntu", "centos_stream", "fedora", "rhel"], action="store", dest="distro", default=None, diff --git a/build/fbcode_builder/getdeps/getdeps_platform.py b/build/fbcode_builder/getdeps/getdeps_platform.py index b4ba52341..6f009043a 100644 --- a/build/fbcode_builder/getdeps/getdeps_platform.py +++ b/build/fbcode_builder/getdeps/getdeps_platform.py @@ -18,13 +18,37 @@ def is_windows() -> bool: return sys.platform.startswith("win") +# Distribution families, after Chef's platform_family: distros that share a +# package namespace and release cadence for the purposes of manifests. +# Fedora is deliberately its own family (unlike Ansible's os_family, which +# folds it into RedHat): its base repos carry far more of our dependencies +# than the EL family's do, and manifests already select on that difference. +DISTRO_FAMILIES: dict[str, tuple[str, ...]] = { + "rhel": ("rhel", "centos", "centos_stream", "alma", "rocky"), + "fedora": ("fedora",), + "debian": ("debian", "ubuntu", "pop!_os", "mint"), + "arch": ("arch",), +} + + +def distro_family(distro: str | None) -> str | None: + for family, members in DISTRO_FAMILIES.items(): + if distro in members: + return family + return None + + def get_linux_type() -> tuple[str | None, str | None, str | None]: try: with open("/etc/os-release") as f: data = f.read() except EnvironmentError: return (None, None, None) + return parse_os_release(data) + +def parse_os_release(data: str) -> tuple[str, str | None, str | None]: + """Derive (ostype, distro, distrovers) from the contents of /etc/os-release.""" os_vars: dict[str, str] = {} for line in data.splitlines(): parts = line.split("=", 1) @@ -48,6 +72,17 @@ def get_linux_type() -> tuple[str | None, str | None, str | None]: if version_id: version_id = version_id.lower() + # Red Hat Enterprise Linux's NAME would otherwise become + # "red_hat_enterprise"; use the same short name as its os-release ID. + if name == "red_hat_enterprise": + name = "rhel" + + # The EL family is versioned by major release for packaging purposes, + # and manifests select on distro_vers=9; RHEL, Alma and Rocky report + # a minor too ("9.8") which would never match. + if version_id and distro_family(name) == "rhel": + version_id = version_id.split(".")[0] + return "linux", name, version_id @@ -260,6 +295,10 @@ def is_linux(self) -> bool: def is_freebsd(self) -> bool: return self.ostype == "freebsd" + @property + def distro_family(self) -> str | None: + return distro_family(self.distro) + def as_tuple_string(self) -> str: return "%s-%s-%s" % ( self.ostype, @@ -272,13 +311,12 @@ def get_package_manager(self) -> str | None: return None if self.is_darwin(): return "homebrew" - if self.distro in ("fedora", "centos", "centos_stream", "rocky", "alma"): + family = self.distro_family + if family in ("fedora", "rhel"): return "rpm" - if self.distro is not None and self.distro.startswith( - ("debian", "ubuntu", "pop!_os", "mint") - ): + if family == "debian": return "deb" - if self.distro == "arch": + if family == "arch": return "pacman-package" return None diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 43ece9fa2..23859efea 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -963,6 +963,7 @@ class ManifestContext: "os", "distro", "distro_vers", + "distro_family", "fb", "fbsource", "test", diff --git a/build/fbcode_builder/getdeps/test/builder_test.py b/build/fbcode_builder/getdeps/test/builder_test.py index 7b5fba8cb..f74e7200a 100644 --- a/build/fbcode_builder/getdeps/test/builder_test.py +++ b/build/fbcode_builder/getdeps/test/builder_test.py @@ -13,7 +13,6 @@ from ..envfuncs import Env from ..manifest import ManifestContext, ManifestParser - MINIMAL_MANIFEST = """ [manifest] name = test @@ -41,6 +40,7 @@ def make_cmake_builder() -> CMakeBuilder: "os": None, "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off", diff --git a/build/fbcode_builder/getdeps/test/features_test.py b/build/fbcode_builder/getdeps/test/features_test.py index 01072b6b0..b994dcce0 100644 --- a/build/fbcode_builder/getdeps/test/features_test.py +++ b/build/fbcode_builder/getdeps/test/features_test.py @@ -21,6 +21,7 @@ def _ctx_with_features(features: set[str] | None = None) -> ManifestContext: "os": "linux", "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off", @@ -64,6 +65,7 @@ def _make_loader(manifests: dict[str, str]) -> ManifestLoader: "os": "linux", "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off", diff --git a/build/fbcode_builder/getdeps/test/platform_test.py b/build/fbcode_builder/getdeps/test/platform_test.py index 52572bb47..858e2f8dd 100644 --- a/build/fbcode_builder/getdeps/test/platform_test.py +++ b/build/fbcode_builder/getdeps/test/platform_test.py @@ -6,7 +6,7 @@ import unittest -from ..getdeps_platform import HostType +from ..getdeps_platform import HostType, parse_os_release class PlatformTest(unittest.TestCase): @@ -37,3 +37,42 @@ def test_is_methods(self) -> None: self.assertFalse(p.is_windows()) self.assertFalse(p.is_darwin()) self.assertTrue(p.is_linux()) + + +class OsReleaseTest(unittest.TestCase): + def parse(self, name: str, version_id: str) -> tuple[str, str | None, str | None]: + return parse_os_release(f'NAME="{name}"\nVERSION_ID="{version_id}"\n') + + def test_fedora(self) -> None: + self.assertEqual(self.parse("Fedora Linux", "44"), ("linux", "fedora", "44")) + self.assertEqual(HostType("linux", "fedora", "44").distro_family, "fedora") + + def test_centos_stream(self) -> None: + self.assertEqual( + self.parse("CentOS Stream", "9"), ("linux", "centos_stream", "9") + ) + + def test_rhel_gets_short_name_and_major_version(self) -> None: + # EPEL buildroots run real RHEL, whose NAME would otherwise become + # "red_hat_enterprise" and whose VERSION_ID carries a minor. + ostype, distro, vers = self.parse("Red Hat Enterprise Linux", "9.8") + self.assertEqual((ostype, distro, vers), ("linux", "rhel", "9")) + host = HostType(ostype, distro, vers) + self.assertEqual(host.distro_family, "rhel") + self.assertEqual(host.get_package_manager(), "rpm") + + def test_el_rebuilds_share_family_and_major_version(self) -> None: + self.assertEqual(self.parse("AlmaLinux", "9.6"), ("linux", "alma", "9")) + self.assertEqual(self.parse("Rocky Linux", "9.6"), ("linux", "rocky", "9")) + self.assertEqual(HostType("linux", "alma", "9").distro_family, "rhel") + + def test_debian_family_keeps_full_version(self) -> None: + self.assertEqual(self.parse("Ubuntu", "22.04"), ("linux", "ubuntu", "22.04")) + host = HostType("linux", "ubuntu", "22.04") + self.assertEqual(host.distro_family, "debian") + self.assertEqual(host.get_package_manager(), "deb") + + def test_unknown_distro_has_no_family(self) -> None: + host = HostType("linux", "gentoo", None) + self.assertIsNone(host.distro_family) + self.assertIsNone(host.get_package_manager()) diff --git a/build/fbcode_builder/getdeps/test/vendor_test.py b/build/fbcode_builder/getdeps/test/vendor_test.py index ed571cb6d..1d96fea61 100644 --- a/build/fbcode_builder/getdeps/test/vendor_test.py +++ b/build/fbcode_builder/getdeps/test/vendor_test.py @@ -35,6 +35,7 @@ def make_ctx() -> ManifestContext: "os": "linux", "distro": None, "distro_vers": None, + "distro_family": None, "fb": "off", "fbsource": "off", "test": "off",