Handle known ARM64 libc6 usr-merge diversion warnings

packaging_automation/packaging_warning_handler.py

@@ -2,7 +2,8 @@
 import re
 import sys
 from enum import Enum
-from typing import List, Tuple
+from itertools import groupby
+from typing import List, Set, Tuple
 
 import yaml
 
@@ -62,6 +63,43 @@ def validate_output(output: str, ignore_file_path: str, package_type: PackageTyp
         print("Build output check completed succesfully. No warnings")
 
 
+def _known_arm64_diversion_warnings(output_lines: List[str]) -> Set[int]:
+    warning = "dpkg-shlibdeps: warning: diversions involved - output may be incorrect"
+    source = " diversion by libc6 from: /lib/ld-linux-aarch64.so.1"
+    target = " diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged"
+    known_warnings = set()
+
+    # Keep unknown/malformed diversion records in the block so they veto it.
+    for is_diagnostic, block in groupby(
+        enumerate(output_lines),
+        lambda item: item[1]
+        .lstrip()
+        .startswith(("dpkg-shlibdeps:", "diversion", "local diversion")),
+    ):
+        if not is_diagnostic:
+            continue
+        records = list(block)
+        lines = [line for _, line in records]
+        if (
+            set(lines) != {warning, source, target}
+            or lines.count(source) != lines.count(target)
+            or lines.count(warning) != 2 * lines.count(source)
+        ):
+            continue
+
+        # Parallel dh_shlibdeps may interleave records, but every detail must
+        # follow a warning. Only complete libc6 ARM64 usr-merge blocks qualify.
+        pending_warnings = 0
+        for line in lines:
+            pending_warnings += 1 if line == warning else -1
+            if pending_warnings < 0:
+                break
+        else:
+            known_warnings.update(index for index, line in records if line == warning)
+
+    return known_warnings
+
+
 def filter_warning_lines(
     output_lines: List[str], package_type: PackageType
 ) -> Tuple[List[str], List[str]]:
@@ -76,7 +114,10 @@ def filter_warning_lines(
     package_specific_warning_lines = []
     is_deb_warning_line = False
     is_rpm_warning_line = False
-    for output_line in output_lines:
+    known_diversion_warnings = set()
+    if package_type == PackageType.deb:
+        known_diversion_warnings = _known_arm64_diversion_warnings(output_lines)
+    for line_index, output_line in enumerate(output_lines):
         if package_type == PackageType.deb:
             if debian_lintian_starter in output_line:
                 is_deb_warning_line = True
@@ -87,7 +128,7 @@ def filter_warning_lines(
                         package_specific_warning_lines.append(output_line)
                     else:
                         is_deb_warning_line = False
-                else:
+                elif line_index not in known_diversion_warnings:
                     base_warning_lines.append(output_line)
         else:
             if rpm_lintian_starter in output_line:

packaging_automation/tests/files/packaging_warning/sample_warning_build_output_deb_arm64_diversions.txt

@@ -0,0 +1,26 @@
+   dh_shlibdeps -a
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 from: /lib/ld-linux-aarch64.so.1
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 from: /lib/ld-linux-aarch64.so.1
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged
+ diversion by libc6 from: /lib/ld-linux-aarch64.so.1
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 from: /lib/ld-linux-aarch64.so.1
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 from: /lib/ld-linux-aarch64.so.1
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 from: /lib/ld-linux-aarch64.so.1
+dpkg-shlibdeps: warning: diversions involved - output may be incorrect
+ diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged
+   dh_installdeb -a

packaging_automation/tests/test_packaging_warning_handler.py

@@ -15,6 +15,13 @@
 )
 
 TEST_BASE_PATH = pathlib2.Path(__file__).parent
+IGNORE_FILE = f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml"
+DIVERSION_WARNING = (
+    "dpkg-shlibdeps: warning: diversions involved - output may be incorrect"
+)
+DIVERSION_FROM = " diversion by libc6 from: /lib/ld-linux-aarch64.so.1"
+DIVERSION_TO = " diversion by libc6 to: /lib/ld-linux-aarch64.so.1.usr-is-merged"
+KNOWN_DIVERSION = [DIVERSION_WARNING, DIVERSION_FROM, DIVERSION_WARNING, DIVERSION_TO]
 
 
 def test_parse_ignore_lists():
@@ -205,3 +212,210 @@ def test_validate_output_rpm_success():
             f"{TEST_BASE_PATH}/files/packaging_warning/packaging_ignore.yml",
             PackageType.rpm,
         )
+
+
+@pytest.mark.parametrize(
+    "lines",
+    [
+        pytest.param(KNOWN_DIVERSION, id="serial"),
+        pytest.param(KNOWN_DIVERSION * 3, id="repeated"),
+        pytest.param(
+            [DIVERSION_WARNING, DIVERSION_WARNING, DIVERSION_TO, DIVERSION_FROM],
+            id="interleaved",
+        ),
+        pytest.param(
+            KNOWN_DIVERSION + ["   dh_gencontrol -a"] + KNOWN_DIVERSION,
+            id="separate-complete-blocks",
+        ),
+    ],
+)
+def test_known_arm64_diversion(lines):
+    original_lines = lines.copy()
+    assert filter_warning_lines(lines, PackageType.deb) == ([], [])
+    validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb)
+    assert lines == original_lines
+
+
+def test_known_arm64_diversion_topn_noble_output():
+    # Original topn/Noble output, including warning/warning/to/from interleaving.
+    output = (
+        TEST_BASE_PATH
+        / "files/packaging_warning/sample_warning_build_output_deb_arm64_diversions.txt"
+    ).read_text(
+        encoding=DEFAULT_ENCODING_FOR_FILE_HANDLING,
+        errors=DEFAULT_UNICODE_ERROR_HANDLER,
+    )
+    assert filter_warning_lines(output.splitlines(), PackageType.deb) == ([], [])
+    validate_output(output, IGNORE_FILE, PackageType.deb)
+
+
+@pytest.mark.parametrize(
+    "source,target",
+    [
+        pytest.param(
+            DIVERSION_FROM.replace("libc6", "other"),
+            DIVERSION_TO.replace("libc6", "other"),
+            id="unknown-owner",
+        ),
+        pytest.param(
+            DIVERSION_FROM.replace("libc6", "libc6:arm64"),
+            DIVERSION_TO,
+            id="different-owner-spelling",
+        ),
+        pytest.param(
+            DIVERSION_FROM.replace("diversion by libc6", "local diversion"),
+            DIVERSION_TO.replace("diversion by libc6", "local diversion"),
+            id="local-diversion",
+        ),
+        pytest.param(
+            DIVERSION_FROM.replace("aarch64", "x86-64"),
+            DIVERSION_TO.replace("aarch64", "x86-64"),
+            id="other-architecture",
+        ),
+        pytest.param(
+            DIVERSION_FROM.replace("/lib/", "/usr/lib/"),
+            DIVERSION_TO,
+            id="other-source-path",
+        ),
+        pytest.param(
+            DIVERSION_FROM,
+            DIVERSION_TO.replace(".usr-is-merged", ".distrib"),
+            id="other-target-path",
+        ),
+        pytest.param("", DIVERSION_TO, id="missing-source"),
+        pytest.param(DIVERSION_FROM, "", id="missing-target"),
+        pytest.param(DIVERSION_FROM, DIVERSION_TO + ".extra", id="target-suffix"),
+        pytest.param(DIVERSION_FROM + " extra", DIVERSION_TO, id="extra-text"),
+        pytest.param(DIVERSION_FROM.lstrip(), DIVERSION_TO, id="missing-indent"),
+        pytest.param("\t" + DIVERSION_FROM, DIVERSION_TO, id="extra-indent"),
+        pytest.param(DIVERSION_FROM, DIVERSION_TO + " ", id="trailing-space"),
+        pytest.param(
+            DIVERSION_FROM.replace("from:", "from"),
+            DIVERSION_TO,
+            id="malformed-record",
+        ),
+        pytest.param(
+            DIVERSION_FROM,
+            DIVERSION_TO.replace("to: /lib/ld-linux-aarch64.so.1.usr-is-merged", "to:"),
+            id="incomplete-record",
+        ),
+    ],
+)
+@pytest.mark.parametrize("interleaved", [False, True])
+def test_unknown_diversion_context(source, target, interleaved):
+    if interleaved:
+        lines = [DIVERSION_WARNING, DIVERSION_WARNING, target, source]
+    else:
+        lines = [DIVERSION_WARNING, source, DIVERSION_WARNING, target]
+    assert filter_warning_lines(lines, PackageType.deb) == (
+        [DIVERSION_WARNING, DIVERSION_WARNING],
+        [],
+    )
+    # Valid records elsewhere must not rescue missing or unknown context.
+    output = "\n".join(KNOWN_DIVERSION + lines + KNOWN_DIVERSION)
+    with pytest.raises(SystemExit) as error:
+        validate_output(output, IGNORE_FILE, PackageType.deb)
+    assert error.value.code == 1
+
+
+@pytest.mark.parametrize(
+    "lines",
+    [
+        pytest.param([DIVERSION_WARNING], id="no-context"),
+        pytest.param([DIVERSION_WARNING, DIVERSION_FROM], id="source-only"),
+        pytest.param([DIVERSION_WARNING, DIVERSION_TO], id="target-only"),
+        pytest.param(
+            [DIVERSION_WARNING, DIVERSION_WARNING, DIVERSION_FROM],
+            id="truncated-interleaving",
+        ),
+        pytest.param([DIVERSION_WARNING, DIVERSION_FROM] * 2, id="unbalanced-source"),
+        pytest.param([DIVERSION_WARNING, DIVERSION_TO] * 2, id="unbalanced-target"),
+        pytest.param(KNOWN_DIVERSION + [DIVERSION_WARNING], id="extra-warning"),
+        pytest.param(
+            KNOWN_DIVERSION + [DIVERSION_FROM, DIVERSION_TO],
+            id="extra-details",
+        ),
+        pytest.param(
+            [DIVERSION_FROM, DIVERSION_WARNING, DIVERSION_WARNING, DIVERSION_TO],
+            id="detail-before-warning",
+        ),
+        pytest.param(
+            [DIVERSION_WARNING, DIVERSION_FROM, DIVERSION_TO, DIVERSION_WARNING],
+            id="detail-overtakes-warning",
+        ),
+        pytest.param(
+            KNOWN_DIVERSION
+            + ["dpkg-shlibdeps: warning: cannot find library libunknown.so"],
+            id="other-shlibdeps-warning",
+        ),
+        pytest.param(
+            KNOWN_DIVERSION + ["dpkg-shlibdeps: error: unknown diagnostic"],
+            id="extra-diagnostic-type",
+        ),
+        pytest.param(
+            KNOWN_DIVERSION + [" local diversion from: /unknown"],
+            id="extra-local-diversion",
+        ),
+        pytest.param(
+            KNOWN_DIVERSION + [DIVERSION_WARNING + " extra"],
+            id="nonliteral-warning",
+        ),
+    ],
+)
+def test_incomplete_or_ambiguous_diversion_block(lines):
+    assert filter_warning_lines(lines, PackageType.deb) == (
+        [line for line in lines if "warning" in line.lower()],
+        [],
+    )
+    with pytest.raises(SystemExit) as error:
+        validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb)
+    assert error.value.code == 1
+
+
+@pytest.mark.parametrize(
+    "boundary",
+    ["", "unrelated output", "   dh_gencontrol -a", "compiler: warning: unrelated"],
+)
+def test_diversion_context_does_not_cross_boundaries(boundary):
+    lines = [
+        DIVERSION_WARNING,
+        DIVERSION_FROM,
+        boundary,
+        DIVERSION_WARNING,
+        DIVERSION_TO,
+    ]
+    base_warnings, _ = filter_warning_lines(lines, PackageType.deb)
+    assert base_warnings.count(DIVERSION_WARNING) == 2
+    with pytest.raises(SystemExit):
+        validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb)
+
+
+@pytest.mark.parametrize("before", [False, True])
+def test_unrelated_diversion_warning_remains_fatal(before):
+    missing_context = [DIVERSION_WARNING]
+    blocks = (
+        [missing_context, KNOWN_DIVERSION]
+        if before
+        else [KNOWN_DIVERSION, missing_context]
+    )
+    lines = blocks[0] + ["   dh_gencontrol -a"] + blocks[1]
+    assert filter_warning_lines(lines, PackageType.deb) == ([DIVERSION_WARNING], [])
+    with pytest.raises(SystemExit):
+        validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb)
+
+
+def test_known_diversion_does_not_ignore_lintian():
+    lintian_lines = ["package: W: unknown-warning", "package: E: unknown-error"]
+    lines = KNOWN_DIVERSION + ["Now running lintian"] + lintian_lines
+    assert filter_warning_lines(lines, PackageType.deb) == ([], lintian_lines)
+    with pytest.raises(SystemExit):
+        validate_output("\n".join(lines), IGNORE_FILE, PackageType.deb)
+
+
+def test_known_diversion_not_ignored_for_rpm():
+    assert filter_warning_lines(KNOWN_DIVERSION, PackageType.rpm) == (
+        [DIVERSION_WARNING, DIVERSION_WARNING],
+        [],
+    )
+    with pytest.raises(SystemExit):
+        validate_output("\n".join(KNOWN_DIVERSION), IGNORE_FILE, PackageType.rpm)