openstack_test: serial coda for Machine leak, suite flakes, and topology

collection/stages/roles/openstack_test/defaults/main.yml

@@ -14,3 +14,26 @@ openstack_test_suite: openstack-test/all
 openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
 # Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
 openstack_test_ote_run_serial: false
+# Serial coda: after the main filtered batch suite, run selected tests
+# serially in fixed order. Starts with the bogus Machine leak cluster
+# (bz_2073398 cleanup, then Machine trio), then known suite-load flakes
+# that pass alone but time out in the randomized batch (egressIP FIP
+# failover AAP wait, prometheus PVC resize pod-ready wait), then topology
+# pair ordered so the AZ check runs while enable_topology is still true
+# before the mutator ([Serial] set enable_topology=false) can race it in
+# batch. Keeps the bulk suite in batch to limit wall time. No-ops when
+# the matchers are absent (e.g. lb_tests allowlists). Requires OTE
+# filtered run path.
+openstack_test_machines_leak_cluster_serial: true
+# Ordered substrings matched against list_of_tests_to_run.txt (one phrase
+# each; written to a matchers file so shell never splits them). For the
+# MachineSet replica matcher, ControlPlane variants are skipped.
+openstack_test_machines_leak_cluster_matchers:
+  - "Bugfix bz_2073398: [Serial] MachineSet scale-in does not leak OpenStack ports"
+  - "Machine are in phase Running"
+  - "MachineSet replica number corresponds to the number of Machines"
+  - "Machine ProviderSpec is correctly applied to OpenStack instances"
+  - "egressIP attached to a floating IP should be kept after EgressIP node failover"
+  - "should follow PVC specs during resizing for prometheus"
+  - "enables topology aware scheduling when compute and volume AZs are identical"
+  - "should allow the manual setting of enable_topology to false"

collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Split serial-coda tests out of an OTE test list.
+
+Reads the filtered list_of_tests_to_run.txt and an ordered matchers file
+(one substring per line), writes:
+  - leak_cluster_serial.txt: ordered matches for serial coda
+    (Machine leak cluster, suite-load flakes, topology pair)
+  - list_of_tests_to_run.txt: remaining tests for the batch suite
+
+Matchers are plain substrings (one full phrase per line). The MachineSet
+replica matcher skips any line that also contains "ControlPlane".
+
+When any matcher hits, every matcher must resolve to exactly one test;
+missing or non-unique matches fail hard. Zero hits keep the allowlist
+no-op (empty serial file, batch unchanged).
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+
+def load_matchers(path: str) -> list[str]:
+    with open(path, "r", encoding="utf-8") as f:
+        return [ln.strip() for ln in f if ln.strip() and not ln.strip().startswith("#")]
+
+
+def pick(remaining: list[str], matcher: str) -> list[str]:
+    """Return all candidate lines for matcher (ControlPlane skipped for MachineSet replica)."""
+    candidates: list[str] = []
+    for ln in remaining:
+        if not ln.strip():
+            continue
+        if matcher not in ln:
+            continue
+        # Avoid ControlPlane MachineSet when matching worker MachineSet replica.
+        if "MachineSet replica number corresponds to the number of Machines" in matcher:
+            if "ControlPlane" in ln:
+                continue
+        candidates.append(ln)
+    return candidates
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tests_to_run_path")
+    parser.add_argument("leak_cluster_path")
+    parser.add_argument("matchers_file")
+    args = parser.parse_args()
+
+    matchers = load_matchers(args.matchers_file)
+    if not matchers:
+        print("leak_cluster=0 batch_remaining=unchanged (empty matchers file)")
+        Path(args.leak_cluster_path).write_text("", encoding="utf-8")
+        return 0
+
+    with open(args.tests_to_run_path, "r", encoding="utf-8") as f:
+        lines = [ln.rstrip("\n") for ln in f]
+
+    remaining = list(lines)
+    resolved: list[tuple[str, list[str]]] = []
+    any_hit = False
+
+    for matcher in matchers:
+        candidates = pick(remaining, matcher)
+        if candidates:
+            any_hit = True
+        resolved.append((matcher, candidates))
+
+    if not any_hit:
+        # Allowlist no-op: none of the coda tests are in this filtered list.
+        print("leak_cluster=0 batch_remaining=unchanged (no matchers hit)")
+        Path(args.leak_cluster_path).write_text("", encoding="utf-8")
+        return 0
+
+    selected: list[str] = []
+    for matcher, candidates in resolved:
+        if len(candidates) == 0:
+            print(
+                f"error: matcher matched no tests: {matcher!r}",
+                file=sys.stderr,
+            )
+            return 1
+        if len(candidates) > 1:
+            print(
+                f"error: matcher matched {len(candidates)} tests (expected 1): {matcher!r}",
+                file=sys.stderr,
+            )
+            for c in candidates:
+                print(f"  candidate: {c}", file=sys.stderr)
+            return 1
+        selected.append(candidates[0])
+        remaining = [ln for ln in remaining if ln != candidates[0]]
+
+    with open(args.leak_cluster_path, "w", encoding="utf-8") as f:
+        for ln in selected:
+            f.write(ln + "\n")
+
+    with open(args.tests_to_run_path, "w", encoding="utf-8") as f:
+        for ln in remaining:
+            f.write(ln + "\n")
+
+    print(
+        f"leak_cluster={len(selected)} batch_remaining={len([x for x in remaining if x.strip()])}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml

@@ -0,0 +1,101 @@
+---
+# Serial coda: serially run the ordered coda list after the main batch suite
+# (Machine leak cluster, egressIP / prometheus, then topology AZ check
+# before enable_topology=false mutator).
+# Expects openstack_test_leak_cluster_path to already list the ordered tests.
+- name: Stat machines leak-cluster serial list
+  ansible.builtin.stat:
+    path: "{{ openstack_test_leak_cluster_path }}"
+  register: openstack_test_leak_cluster_stat
+
+- name: Count machines leak-cluster serial tests
+  ansible.builtin.command:
+    argv:
+      - bash
+      - -c
+      - "grep -cve '^[[:space:]]*$' '{{ openstack_test_leak_cluster_path }}' || true"
+  register: openstack_test_leak_cluster_count
+  changed_when: false
+  when: openstack_test_leak_cluster_stat.stat.exists | default(false)
+
+- name: Skip machines leak-cluster serial coda when empty
+  ansible.builtin.debug:
+    msg: >-
+      No machines leak-cluster tests matched the filtered list; skipping serial coda.
+  when: >
+    (not (openstack_test_leak_cluster_stat.stat.exists | default(false)))
+    or ((openstack_test_leak_cluster_count.stdout | default('0') | trim | int) == 0)
+
+- name: Run machines leak-cluster tests serially and merge into suite log/junit
+  when:
+    - openstack_test_leak_cluster_stat.stat.exists | default(false)
+    - (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0
+  block:
+    - name: Execute machines leak-cluster run-test loop
+      ansible.builtin.shell: |
+        set -o pipefail
+        suite_exit=0
+        py_exit=0
+        resolve="{{ openstack_test_ote_resolve_script }}"
+        leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log"
+        leak_list="{{ openstack_test_leak_cluster_path }}"
+        # Pre-count from the list file (do not increment in-loop; avoids
+        # false UNSTABLE when resolvable!=expected after a successful coda).
+        expected="$(grep -cve '^[[:space:]]*$' "$leak_list" || true)"
+        : > "$leak_log"
+        while IFS= read -r tname || [ -n "${tname:-}" ]; do
+          [ -z "${tname// }" ] && continue
+          tmp=$(mktemp)
+          rc=0
+          {{ openstack_test_executable }} run-test --output=json -c 1 "$tname" \
+            > "$tmp" 2>&1 || rc=$?
+          cat "$tmp" >> "$leak_log"
+          cat "$tmp" >> {{ openstack_test_log_path }}
+          if [ "$rc" -ne 0 ]; then
+            failed=$(python3 "$resolve" count "$tmp" failed || echo 1)
+            skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0)
+            passed=$(python3 "$resolve" count "$tmp" passed || echo 0)
+            unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0)
+            if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then
+              suite_exit=1
+            elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then
+              if grep -q 'FAIL!' "$tmp" 2>/dev/null; then
+                suite_exit=1
+              elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then
+                suite_exit=1
+              fi
+            fi
+          fi
+          rm -f "$tmp"
+        done < "$leak_list"
+        python3 "$resolve" junit \
+          "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
+        failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0)
+        skipped_count=$(python3 "$resolve" count "$leak_log" skipped || echo 0)
+        passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0)
+        unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0)
+        resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} ))
+        echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected} resolvable=${resolvable}"
+        if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \
+          || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \
+          || [ "$resolvable" -ne "$expected" ]; then
+          exit 1
+        fi
+        exit 0
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+        KUBECONFIG: "{{ kubeconfig }}"
+        RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}"
+        SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}"
+      changed_when: true
+      register: openstack_test_leak_cluster_run
+
+  rescue:
+    - name: Mark openstack-test UNSTABLE after machines leak-cluster serial failure
+      ansible.builtin.include_role:
+        name: tools_stage_results
+        tasks_from: mark_test_stage_unstable.yml
+      vars:
+        unstable_msg: >-
+          openstack_test serial coda failed (Machine leak cluster, egressIP /
+          prometheus resize, and/or topology enable_topology pair).

collection/stages/roles/openstack_test/tasks/run_openstack_test.yml

@@ -141,6 +141,53 @@
   ansible.builtin.set_fact:
     openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}"
 
+# Serial coda: pull matchers (Machine leak cluster + suite-load flakes)
+# out of the batch list for an ordered serial run after the batch.
+- name: Set machines leak-cluster serial list path
+  ansible.builtin.set_fact:
+    openstack_test_leak_cluster_path: >-
+      {{ openstack_test_results_dir }}/machines_leak_cluster_serial.txt
+    openstack_test_leak_cluster_matchers_path: >-
+      {{ openstack_test_results_dir }}/machines_leak_cluster_matchers.txt
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
+# Write matchers one-per-line so multi-word phrases are never shell-split.
+- name: Write machines leak-cluster matchers file
+  ansible.builtin.copy:
+    dest: "{{ openstack_test_leak_cluster_matchers_path }}"
+    mode: "0644"
+    content: "{{ openstack_test_machines_leak_cluster_matchers | map('trim') | join('\n') }}\n"
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
+- name: Split machines leak-cluster tests from batch list for serial coda
+  ansible.builtin.command:
+    argv:
+      - python3
+      - "{{ role_path }}/files/split_machines_leak_cluster.py"
+      - "{{ tests_to_run_path }}"
+      - "{{ openstack_test_leak_cluster_path }}"
+      - "{{ openstack_test_leak_cluster_matchers_path }}"
+  register: openstack_test_leak_cluster_split
+  changed_when: true
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
+- name: Print machines leak-cluster split summary
+  ansible.builtin.debug:
+    msg: "{{ openstack_test_leak_cluster_split.stdout | default('leak cluster split skipped') }}"
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
 # OTE run-suite often marks every outer JSON result as failed when stderr has
 # klog lines (Deserializaion Error: invalid character 'I'...). True outcomes
 # live in nested STDOUT JSON / ginkgo SUCCESS!|FAIL! lines — see
@@ -385,3 +432,12 @@
         name: tools_must-gather
       vars:
         must_gather_suffix: "openstack-tests"
+
+# Serial coda after the main batch (or filtered serial) suite.
+- name: Run serial-coda tests after main suite
+  ansible.builtin.include_tasks: run_machines_leak_cluster_serial.yml
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+    - openstack_test_leak_cluster_path is defined