From 2fdd30c5d8a88de8ff6022491efcaa6809925ef0 Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Sat, 26 Sep 2026 21:20:51 +0530
Subject: [PATCH 1/7] osp_verification: serial coda for bogus Machine leak
 cluster

Keep the main openstack_test OTE suite in batch for wall time, but split
bz_2073398 and the Machine trio into an ordered serial coda so DeferCleanup
from openstack-test#321 can finish before Machine Running / MachineSet
replica / ProviderSpec. No-ops when those tests are absent (lb allowlists).

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../roles/openstack_test/defaults/main.yml    | 12 +++
 .../files/split_machines_leak_cluster.py      | 68 +++++++++++++
 .../run_machines_leak_cluster_serial.yml      | 96 +++++++++++++++++++
 .../tasks/run_openstack_test.yml              | 43 +++++++++
 4 files changed, 219 insertions(+)
 create mode 100644 collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
 create mode 100644 collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml

diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml
index 2ec13f5d..61b50392 100644
--- a/collection/stages/roles/openstack_test/defaults/main.yml
+++ b/collection/stages/roles/openstack_test/defaults/main.yml
@@ -14,3 +14,15 @@ openstack_test_suite: openstack-test/all
 openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
 # Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
 openstack_test_ote_run_serial: false
+# Option B: after the main filtered batch suite, run the bogus Machine leak
+# cluster serially in fixed order (bz_2073398 cleanup, then Machine trio).
+# Keeps the bulk suite in batch to limit wall time. No-ops when the matchers
+# are absent (e.g. lb_tests allowlists). Requires OTE filtered run path.
+openstack_test_machines_leak_cluster_serial: true
+# Ordered substrings matched against list_of_tests_to_run.txt. For the
+# MachineSet replica matcher, ControlPlane variants are skipped.
+openstack_test_machines_leak_cluster_matchers:
+  - "Bugfix bz_2073398: [Serial] MachineSet scale-in does not leak OpenStack ports"
+  - "Machine are in phase Running"
+  - "MachineSet replica number corresponds to the number of Machines"
+  - "Machine ProviderSpec is correctly applied to OpenStack instances"
diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
new file mode 100644
index 00000000..b3a33bd8
--- /dev/null
+++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Split machines leak-cluster tests out of an OTE test list for serial coda.
+
+Reads the filtered list_of_tests_to_run.txt, writes:
+  - leak_cluster_serial.txt: ordered matches for Option B serial run
+  - list_of_tests_to_run.txt: remaining tests for the batch suite
+
+Matchers are plain substrings. The MachineSet replica matcher skips any line
+that also contains "ControlPlane".
+"""
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("tests_to_run_path")
+    parser.add_argument("leak_cluster_path")
+    parser.add_argument(
+        "matchers",
+        nargs="+",
+        help="Ordered substrings to pull into the serial leak-cluster list",
+    )
+    args = parser.parse_args()
+
+    with open(args.tests_to_run_path, "r", encoding="utf-8") as f:
+        lines = [ln.rstrip("\n") for ln in f]
+
+    remaining = list(lines)
+    selected: list[str] = []
+
+    for matcher in args.matchers:
+        found = None
+        for ln in remaining:
+            if not ln.strip():
+                continue
+            if matcher not in ln:
+                continue
+            if matcher.startswith("MachineSet replica") and "ControlPlane" in ln:
+                continue
+            if "MachineSet replica number corresponds to the number of Machines" in matcher:
+                if "ControlPlane" in ln:
+                    continue
+            found = ln
+            break
+        if found is None:
+            continue
+        selected.append(found)
+        remaining = [ln for ln in remaining if ln != found]
+
+    with open(args.leak_cluster_path, "w", encoding="utf-8") as f:
+        for ln in selected:
+            f.write(ln + "\n")
+
+    with open(args.tests_to_run_path, "w", encoding="utf-8") as f:
+        for ln in remaining:
+            f.write(ln + "\n")
+
+    print(
+        f"leak_cluster={len(selected)} batch_remaining={len([x for x in remaining if x.strip()])}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
new file mode 100644
index 00000000..1004cf34
--- /dev/null
+++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
@@ -0,0 +1,96 @@
+---
+# Option B: serially run bz_2073398 then Machine trio after the main batch suite.
+# Expects openstack_test_leak_cluster_path to already list the ordered tests.
+- name: Stat machines leak-cluster serial list
+  ansible.builtin.stat:
+    path: "{{ openstack_test_leak_cluster_path }}"
+  register: openstack_test_leak_cluster_stat
+
+- name: Count machines leak-cluster serial tests
+  ansible.builtin.command:
+    argv:
+      - bash
+      - -c
+      - "grep -cve '^[[:space:]]*$' {{ openstack_test_leak_cluster_path }} || true"
+  register: openstack_test_leak_cluster_count
+  changed_when: false
+  when: openstack_test_leak_cluster_stat.stat.exists | default(false)
+
+- name: Skip machines leak-cluster serial coda when empty
+  ansible.builtin.debug:
+    msg: >-
+      No machines leak-cluster tests matched the filtered list; skipping serial coda.
+  when: >
+    (not (openstack_test_leak_cluster_stat.stat.exists | default(false)))
+    or ((openstack_test_leak_cluster_count.stdout | default('0') | trim | int) == 0)
+
+- name: Run machines leak-cluster tests serially and merge into suite log/junit
+  when:
+    - openstack_test_leak_cluster_stat.stat.exists | default(false)
+    - (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0
+  block:
+    - name: Execute machines leak-cluster run-test loop
+      ansible.builtin.shell: |
+        set -o pipefail
+        suite_exit=0
+        py_exit=0
+        expected=0
+        resolve="{{ openstack_test_ote_resolve_script }}"
+        leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log"
+        : > "$leak_log"
+        while IFS= read -r test || [[ -n "$test" ]]; do
+          [[ -z "${test// }" ]] && continue
+          expected=$((expected + 1))
+          tmp=$(mktemp)
+          rc=0
+          {{ openstack_test_executable }} run-test --output=json -c 1 "$test" \
+            > "$tmp" 2>&1 || rc=$?
+          cat "$tmp" >> "$leak_log"
+          cat "$tmp" >> {{ openstack_test_log_path }}
+          if [ "$rc" -ne 0 ]; then
+            failed=$(python3 "$resolve" count "$tmp" failed || echo 1)
+            skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0)
+            passed=$(python3 "$resolve" count "$tmp" passed || echo 0)
+            unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0)
+            if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then
+              suite_exit=1
+            elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then
+              if grep -q 'FAIL!' "$tmp" 2>/dev/null; then
+                suite_exit=1
+              elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then
+                suite_exit=1
+              fi
+            fi
+          fi
+          rm -f "$tmp"
+        done < {{ openstack_test_leak_cluster_path }}
+        python3 "$resolve" junit \
+          "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
+        failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0)
+        skipped_count=$(python3 "$resolve" count "$leak_log" skipped || echo 0)
+        passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0)
+        unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0)
+        resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} ))
+        echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected}"
+        if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \
+          || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \
+          || [ "$resolvable" -ne "$expected" ]; then
+          exit 1
+        fi
+        exit 0
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+        KUBECONFIG: "{{ kubeconfig }}"
+        RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}"
+        SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}"
+      changed_when: true
+      register: openstack_test_leak_cluster_run
+
+  rescue:
+    - name: Mark openstack-test UNSTABLE after machines leak-cluster serial failure
+      ansible.builtin.include_role:
+        name: tools_stage_results
+        tasks_from: mark_test_stage_unstable.yml
+      vars:
+        unstable_msg: >-
+          Machines leak-cluster serial coda failed (bz_2073398 / Machine trio).
diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
index 179da042..fa94ef4a 100644
--- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
@@ -141,6 +141,40 @@
   ansible.builtin.set_fact:
     openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}"
 
+# Option B: pull bz_2073398 + Machine trio out of the batch list for a serial coda.
+- name: Set machines leak-cluster serial list path
+  ansible.builtin.set_fact:
+    openstack_test_leak_cluster_path: >-
+      {{ openstack_test_results_dir }}/machines_leak_cluster_serial.txt
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
+- name: Split machines leak-cluster tests from batch list for serial coda
+  ansible.builtin.shell: |
+    set -o pipefail
+    python3 "{{ role_path }}/files/split_machines_leak_cluster.py" \
+      "{{ tests_to_run_path }}" \
+      "{{ openstack_test_leak_cluster_path }}" \
+    {% for m in openstack_test_machines_leak_cluster_matchers %}
+      {{ m | quote }}{% if not loop.last %} \{% endif %}
+    {% endfor %}
+  register: openstack_test_leak_cluster_split
+  changed_when: true
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
+- name: Print machines leak-cluster split summary
+  ansible.builtin.debug:
+    msg: "{{ openstack_test_leak_cluster_split.stdout | default('leak cluster split skipped') }}"
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
 # OTE run-suite often marks every outer JSON result as failed when stderr has
 # klog lines (Deserializaion Error: invalid character 'I'...). True outcomes
 # live in nested STDOUT JSON / ginkgo SUCCESS!|FAIL! lines — see
@@ -385,3 +419,12 @@
         name: tools_must-gather
       vars:
         must_gather_suffix: "openstack-tests"
+
+# Option B: serial coda after the main batch (or filtered serial) suite.
+- name: Run machines leak-cluster tests serially after main suite
+  ansible.builtin.include_tasks: run_machines_leak_cluster_serial.yml
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+    - openstack_test_leak_cluster_path is defined

From b167626461c4232a8380f6b276580f2d1d37af91 Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Sat, 26 Sep 2026 21:33:04 +0530
Subject: [PATCH 2/7] openstack_test: pass leak-cluster matchers via file

Shell-splitting multi-word matcher argv pulled accidental tests into the
serial coda (leak_cluster=7). Write one matcher per line and invoke the
splitter with a matchers file path instead.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../roles/openstack_test/defaults/main.yml    |  3 +-
 .../files/split_machines_leak_cluster.py      | 53 ++++++++++++-------
 .../tasks/run_openstack_test.yml              | 28 +++++++---
 3 files changed, 55 insertions(+), 29 deletions(-)

diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml
index 61b50392..bccccb6a 100644
--- a/collection/stages/roles/openstack_test/defaults/main.yml
+++ b/collection/stages/roles/openstack_test/defaults/main.yml
@@ -19,7 +19,8 @@ openstack_test_ote_run_serial: false
 # Keeps the bulk suite in batch to limit wall time. No-ops when the matchers
 # are absent (e.g. lb_tests allowlists). Requires OTE filtered run path.
 openstack_test_machines_leak_cluster_serial: true
-# Ordered substrings matched against list_of_tests_to_run.txt. For the
+# Ordered substrings matched against list_of_tests_to_run.txt (one phrase
+# each; written to a matchers file so shell never splits them). For the
 # MachineSet replica matcher, ControlPlane variants are skipped.
 openstack_test_machines_leak_cluster_matchers:
   - "Bugfix bz_2073398: [Serial] MachineSet scale-in does not leak OpenStack ports"
diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
index b3a33bd8..3cd5aff8 100644
--- a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
+++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
 """Split machines leak-cluster tests out of an OTE test list for serial coda.
 
-Reads the filtered list_of_tests_to_run.txt, writes:
+Reads the filtered list_of_tests_to_run.txt and an ordered matchers file
+(one substring per line), writes:
   - leak_cluster_serial.txt: ordered matches for Option B serial run
   - list_of_tests_to_run.txt: remaining tests for the batch suite
 
-Matchers are plain substrings. The MachineSet replica matcher skips any line
-that also contains "ControlPlane".
+Matchers are plain substrings (one full phrase per line). The MachineSet
+replica matcher skips any line that also contains "ControlPlane".
 """
 from __future__ import annotations
 
@@ -14,37 +15,49 @@
 import sys
 
 
+def load_matchers(path: str) -> list[str]:
+    with open(path, "r", encoding="utf-8") as f:
+        return [ln.strip() for ln in f if ln.strip() and not ln.strip().startswith("#")]
+
+
+def pick(remaining: list[str], matcher: str) -> str | None:
+    for ln in remaining:
+        if not ln.strip():
+            continue
+        if matcher not in ln:
+            continue
+        # Avoid ControlPlane MachineSet when matching worker MachineSet replica.
+        if "MachineSet replica number corresponds to the number of Machines" in matcher:
+            if "ControlPlane" in ln:
+                continue
+        return ln
+    return None
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("tests_to_run_path")
     parser.add_argument("leak_cluster_path")
     parser.add_argument(
-        "matchers",
-        nargs="+",
-        help="Ordered substrings to pull into the serial leak-cluster list",
+        "matchers_file",
+        help="File with one matcher substring per line (ordered)",
     )
     args = parser.parse_args()
 
+    matchers = load_matchers(args.matchers_file)
+    if not matchers:
+        print("leak_cluster=0 batch_remaining=unchanged (empty matchers file)")
+        open(args.leak_cluster_path, "w", encoding="utf-8").close()
+        return 0
+
     with open(args.tests_to_run_path, "r", encoding="utf-8") as f:
         lines = [ln.rstrip("\n") for ln in f]
 
     remaining = list(lines)
     selected: list[str] = []
 
-    for matcher in args.matchers:
-        found = None
-        for ln in remaining:
-            if not ln.strip():
-                continue
-            if matcher not in ln:
-                continue
-            if matcher.startswith("MachineSet replica") and "ControlPlane" in ln:
-                continue
-            if "MachineSet replica number corresponds to the number of Machines" in matcher:
-                if "ControlPlane" in ln:
-                    continue
-            found = ln
-            break
+    for matcher in matchers:
+        found = pick(remaining, matcher)
         if found is None:
             continue
         selected.append(found)
diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
index fa94ef4a..1a0d45f9 100644
--- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
@@ -146,20 +146,32 @@
   ansible.builtin.set_fact:
     openstack_test_leak_cluster_path: >-
       {{ openstack_test_results_dir }}/machines_leak_cluster_serial.txt
+    openstack_test_leak_cluster_matchers_path: >-
+      {{ openstack_test_results_dir }}/machines_leak_cluster_matchers.txt
+  when:
+    - openstack_test_use_ote | bool
+    - openstack_test_machines_leak_cluster_serial | bool
+    - openstack_test_filtering | bool
+
+# Write matchers one-per-line so multi-word phrases are never shell-split.
+- name: Write machines leak-cluster matchers file
+  ansible.builtin.copy:
+    dest: "{{ openstack_test_leak_cluster_matchers_path }}"
+    mode: "0644"
+    content: "{{ openstack_test_machines_leak_cluster_matchers | map('trim') | join('\n') }}\n"
   when:
     - openstack_test_use_ote | bool
     - openstack_test_machines_leak_cluster_serial | bool
     - openstack_test_filtering | bool
 
 - name: Split machines leak-cluster tests from batch list for serial coda
-  ansible.builtin.shell: |
-    set -o pipefail
-    python3 "{{ role_path }}/files/split_machines_leak_cluster.py" \
-      "{{ tests_to_run_path }}" \
-      "{{ openstack_test_leak_cluster_path }}" \
-    {% for m in openstack_test_machines_leak_cluster_matchers %}
-      {{ m | quote }}{% if not loop.last %} \{% endif %}
-    {% endfor %}
+  ansible.builtin.command:
+    argv:
+      - python3
+      - "{{ role_path }}/files/split_machines_leak_cluster.py"
+      - "{{ tests_to_run_path }}"
+      - "{{ openstack_test_leak_cluster_path }}"
+      - "{{ openstack_test_leak_cluster_matchers_path }}"
   register: openstack_test_leak_cluster_split
   changed_when: true
   when:

From dd9b6302847718517258c8bc211838e83b2f2759 Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Sat, 26 Sep 2026 21:53:12 +0530
Subject: [PATCH 3/7] openstack_test: fix leak-cluster serial expected count

Pre-count expected tests from the serial list file instead of incrementing
in the while loop. The in-loop counter stayed at 1 after four successful
runs (passed=4 expected=1), falsely marking the coda UNSTABLE.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../tasks/run_machines_leak_cluster_serial.yml | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
index 1004cf34..eaebe104 100644
--- a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
@@ -11,7 +11,7 @@
     argv:
       - bash
       - -c
-      - "grep -cve '^[[:space:]]*$' {{ openstack_test_leak_cluster_path }} || true"
+      - "grep -cve '^[[:space:]]*$' '{{ openstack_test_leak_cluster_path }}' || true"
   register: openstack_test_leak_cluster_count
   changed_when: false
   when: openstack_test_leak_cluster_stat.stat.exists | default(false)
@@ -34,16 +34,18 @@
         set -o pipefail
         suite_exit=0
         py_exit=0
-        expected=0
         resolve="{{ openstack_test_ote_resolve_script }}"
         leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log"
+        leak_list="{{ openstack_test_leak_cluster_path }}"
+        # Pre-count from the list file (do not increment in-loop; avoids
+        # false UNSTABLE when resolvable!=expected after a successful coda).
+        expected="$(grep -cve '^[[:space:]]*$' "$leak_list" || true)"
         : > "$leak_log"
-        while IFS= read -r test || [[ -n "$test" ]]; do
-          [[ -z "${test// }" ]] && continue
-          expected=$((expected + 1))
+        while IFS= read -r tname || [ -n "${tname:-}" ]; do
+          [ -z "${tname// }" ] && continue
           tmp=$(mktemp)
           rc=0
-          {{ openstack_test_executable }} run-test --output=json -c 1 "$test" \
+          {{ openstack_test_executable }} run-test --output=json -c 1 "$tname" \
             > "$tmp" 2>&1 || rc=$?
           cat "$tmp" >> "$leak_log"
           cat "$tmp" >> {{ openstack_test_log_path }}
@@ -63,7 +65,7 @@
             fi
           fi
           rm -f "$tmp"
-        done < {{ openstack_test_leak_cluster_path }}
+        done < "$leak_list"
         python3 "$resolve" junit \
           "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
         failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0)
@@ -71,7 +73,7 @@
         passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0)
         unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0)
         resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} ))
-        echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected}"
+        echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected} resolvable=${resolvable}"
         if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \
           || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \
           || [ "$resolvable" -ne "$expected" ]; then

From ddab31b8e02653bea5ba94af0b73e2510e2a7dc3 Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Mon, 28 Sep 2026 10:31:15 +0530
Subject: [PATCH 4/7] openstack_test: tidy serial-coda split script style

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../openstack_test/files/split_machines_leak_cluster.py   | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
index 3cd5aff8..63c8fcb2 100644
--- a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
+++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
@@ -13,6 +13,7 @@
 
 import argparse
 import sys
+from pathlib import Path
 
 
 def load_matchers(path: str) -> list[str]:
@@ -38,16 +39,13 @@ def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("tests_to_run_path")
     parser.add_argument("leak_cluster_path")
-    parser.add_argument(
-        "matchers_file",
-        help="File with one matcher substring per line (ordered)",
-    )
+    parser.add_argument("matchers_file")
     args = parser.parse_args()
 
     matchers = load_matchers(args.matchers_file)
     if not matchers:
         print("leak_cluster=0 batch_remaining=unchanged (empty matchers file)")
-        open(args.leak_cluster_path, "w", encoding="utf-8").close()
+        Path(args.leak_cluster_path).write_text("", encoding="utf-8")
         return 0
 
     with open(args.tests_to_run_path, "r", encoding="utf-8") as f:

From 456acdd281d87ce9f22ca54440c50e6e98ae3c88 Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Sun, 27 Sep 2026 16:25:08 +0530
Subject: [PATCH 5/7] openstack_test: serial-coda egressIP FIP and prometheus
 resize

Extend Option B (PR #44) so egressIP FIP failover and prometheus PVC
resize leave the randomized batch and run one-by-one after the Machine
leak cluster. Both passed alone on warm serval71 but timed out under
suite load (10s AAP wait / 3m pod-ready wait).

Related: https://github.com/shiftstack/shiftstack-qa/pull/44
Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../stages/roles/openstack_test/defaults/main.yml   | 13 +++++++++----
 .../files/split_machines_leak_cluster.py            |  3 ++-
 .../tasks/run_machines_leak_cluster_serial.yml      |  6 ++++--
 .../openstack_test/tasks/run_openstack_test.yml     |  5 +++--
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml
index bccccb6a..c1a0871a 100644
--- a/collection/stages/roles/openstack_test/defaults/main.yml
+++ b/collection/stages/roles/openstack_test/defaults/main.yml
@@ -14,10 +14,13 @@ openstack_test_suite: openstack-test/all
 openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
 # Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
 openstack_test_ote_run_serial: false
-# Option B: after the main filtered batch suite, run the bogus Machine leak
-# cluster serially in fixed order (bz_2073398 cleanup, then Machine trio).
-# Keeps the bulk suite in batch to limit wall time. No-ops when the matchers
-# are absent (e.g. lb_tests allowlists). Requires OTE filtered run path.
+# Option B: after the main filtered batch suite, run selected tests serially
+# in fixed order. Starts with the bogus Machine leak cluster (bz_2073398
+# cleanup, then Machine trio), then known suite-load flakes that pass alone
+# but time out in the randomized batch (egressIP FIP failover AAP wait,
+# prometheus PVC resize pod-ready wait). Keeps the bulk suite in batch to
+# limit wall time. No-ops when the matchers are absent (e.g. lb_tests
+# allowlists). Requires OTE filtered run path.
 openstack_test_machines_leak_cluster_serial: true
 # Ordered substrings matched against list_of_tests_to_run.txt (one phrase
 # each; written to a matchers file so shell never splits them). For the
@@ -27,3 +30,5 @@ openstack_test_machines_leak_cluster_matchers:
   - "Machine are in phase Running"
   - "MachineSet replica number corresponds to the number of Machines"
   - "Machine ProviderSpec is correctly applied to OpenStack instances"
+  - "egressIP attached to a floating IP should be kept after EgressIP node failover"
+  - "should follow PVC specs during resizing for prometheus"
diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
index 63c8fcb2..6cad9833 100644
--- a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
+++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
-"""Split machines leak-cluster tests out of an OTE test list for serial coda.
+"""Split Option B serial-coda tests out of an OTE test list.
 
 Reads the filtered list_of_tests_to_run.txt and an ordered matchers file
 (one substring per line), writes:
   - leak_cluster_serial.txt: ordered matches for Option B serial run
+    (Machine leak cluster, plus optional suite-load flakes)
   - list_of_tests_to_run.txt: remaining tests for the batch suite
 
 Matchers are plain substrings (one full phrase per line). The MachineSet
diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
index eaebe104..0e21923a 100644
--- a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
@@ -1,5 +1,6 @@
 ---
-# Option B: serially run bz_2073398 then Machine trio after the main batch suite.
+# Option B: serially run the ordered coda list after the main batch suite
+# (Machine leak cluster, then egressIP FIP failover / prometheus resize).
 # Expects openstack_test_leak_cluster_path to already list the ordered tests.
 - name: Stat machines leak-cluster serial list
   ansible.builtin.stat:
@@ -95,4 +96,5 @@
         tasks_from: mark_test_stage_unstable.yml
       vars:
         unstable_msg: >-
-          Machines leak-cluster serial coda failed (bz_2073398 / Machine trio).
+          openstack_test serial coda failed (Machine leak cluster and/or
+          egressIP FIP failover / prometheus PVC resize).
diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
index 1a0d45f9..06693dfc 100644
--- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
@@ -141,7 +141,8 @@
   ansible.builtin.set_fact:
     openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}"
 
-# Option B: pull bz_2073398 + Machine trio out of the batch list for a serial coda.
+# Option B: pull serial-coda matchers (Machine leak cluster + suite-load
+# flakes) out of the batch list for an ordered serial run after the batch.
 - name: Set machines leak-cluster serial list path
   ansible.builtin.set_fact:
     openstack_test_leak_cluster_path: >-
@@ -433,7 +434,7 @@
         must_gather_suffix: "openstack-tests"
 
 # Option B: serial coda after the main batch (or filtered serial) suite.
-- name: Run machines leak-cluster tests serially after main suite
+- name: Run Option B serial-coda tests after main suite
   ansible.builtin.include_tasks: run_machines_leak_cluster_serial.yml
   when:
     - openstack_test_use_ote | bool

From 51c979b4e8fd6d107dfa07c467c8a96a145ae798 Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Sun, 27 Sep 2026 18:01:10 +0530
Subject: [PATCH 6/7] openstack_test: serial-coda topology AZ and
 enable_topology=false

Pull the topology AZ check and the enable_topology=false mutator out of
the randomized batch into the Option B serial coda. Order AZ check first
so it still sees enable_topology=true, then the mutator. Avoids the race
seen on warm serval71 TP (Expected false to equal true at topology.go:89).

Related: https://github.com/shiftstack/shiftstack-qa/pull/45
Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../stages/roles/openstack_test/defaults/main.yml      | 10 +++++++---
 .../files/split_machines_leak_cluster.py               |  2 +-
 .../tasks/run_machines_leak_cluster_serial.yml         |  7 ++++---
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml
index c1a0871a..7f1ac127 100644
--- a/collection/stages/roles/openstack_test/defaults/main.yml
+++ b/collection/stages/roles/openstack_test/defaults/main.yml
@@ -18,9 +18,11 @@ openstack_test_ote_run_serial: false
 # in fixed order. Starts with the bogus Machine leak cluster (bz_2073398
 # cleanup, then Machine trio), then known suite-load flakes that pass alone
 # but time out in the randomized batch (egressIP FIP failover AAP wait,
-# prometheus PVC resize pod-ready wait). Keeps the bulk suite in batch to
-# limit wall time. No-ops when the matchers are absent (e.g. lb_tests
-# allowlists). Requires OTE filtered run path.
+# prometheus PVC resize pod-ready wait), then topology pair ordered so the
+# AZ check runs while enable_topology is still true before the mutator
+# ([Serial] set enable_topology=false) can race it in batch. Keeps the bulk
+# suite in batch to limit wall time. No-ops when the matchers are absent
+# (e.g. lb_tests allowlists). Requires OTE filtered run path.
 openstack_test_machines_leak_cluster_serial: true
 # Ordered substrings matched against list_of_tests_to_run.txt (one phrase
 # each; written to a matchers file so shell never splits them). For the
@@ -32,3 +34,5 @@ openstack_test_machines_leak_cluster_matchers:
   - "Machine ProviderSpec is correctly applied to OpenStack instances"
   - "egressIP attached to a floating IP should be kept after EgressIP node failover"
   - "should follow PVC specs during resizing for prometheus"
+  - "enables topology aware scheduling when compute and volume AZs are identical"
+  - "should allow the manual setting of enable_topology to false"
diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
index 6cad9833..5f0f6562 100644
--- a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
+++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
@@ -4,7 +4,7 @@
 Reads the filtered list_of_tests_to_run.txt and an ordered matchers file
 (one substring per line), writes:
   - leak_cluster_serial.txt: ordered matches for Option B serial run
-    (Machine leak cluster, plus optional suite-load flakes)
+    (Machine leak cluster, suite-load flakes, topology pair)
   - list_of_tests_to_run.txt: remaining tests for the batch suite
 
 Matchers are plain substrings (one full phrase per line). The MachineSet
diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
index 0e21923a..589d8297 100644
--- a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
@@ -1,6 +1,7 @@
 ---
 # Option B: serially run the ordered coda list after the main batch suite
-# (Machine leak cluster, then egressIP FIP failover / prometheus resize).
+# (Machine leak cluster, egressIP / prometheus, then topology AZ check
+# before enable_topology=false mutator).
 # Expects openstack_test_leak_cluster_path to already list the ordered tests.
 - name: Stat machines leak-cluster serial list
   ansible.builtin.stat:
@@ -96,5 +97,5 @@
         tasks_from: mark_test_stage_unstable.yml
       vars:
         unstable_msg: >-
-          openstack_test serial coda failed (Machine leak cluster and/or
-          egressIP FIP failover / prometheus PVC resize).
+          openstack_test serial coda failed (Machine leak cluster, egressIP /
+          prometheus resize, and/or topology enable_topology pair).

From d9002682370257904b1872374759e4424912ecbb Mon Sep 17 00:00:00 2001
From: tusharjadhav3302 <tjadhav@redhat.com>
Date: Mon, 28 Sep 2026 10:33:46 +0530
Subject: [PATCH 7/7] openstack_test: require unique serial-coda matcher hits

Fail hard when any coda matcher is present but missing or ambiguous,
and drop leftover Option B wording in favor of serial coda.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../roles/openstack_test/defaults/main.yml    | 19 +++----
 .../files/split_machines_leak_cluster.py      | 53 +++++++++++++++----
 .../run_machines_leak_cluster_serial.yml      |  2 +-
 .../tasks/run_openstack_test.yml              |  8 +--
 4 files changed, 57 insertions(+), 25 deletions(-)

diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml
index 7f1ac127..5e5386cf 100644
--- a/collection/stages/roles/openstack_test/defaults/main.yml
+++ b/collection/stages/roles/openstack_test/defaults/main.yml
@@ -14,15 +14,16 @@ openstack_test_suite: openstack-test/all
 openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
 # Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
 openstack_test_ote_run_serial: false
-# Option B: after the main filtered batch suite, run selected tests serially
-# in fixed order. Starts with the bogus Machine leak cluster (bz_2073398
-# cleanup, then Machine trio), then known suite-load flakes that pass alone
-# but time out in the randomized batch (egressIP FIP failover AAP wait,
-# prometheus PVC resize pod-ready wait), then topology pair ordered so the
-# AZ check runs while enable_topology is still true before the mutator
-# ([Serial] set enable_topology=false) can race it in batch. Keeps the bulk
-# suite in batch to limit wall time. No-ops when the matchers are absent
-# (e.g. lb_tests allowlists). Requires OTE filtered run path.
+# Serial coda: after the main filtered batch suite, run selected tests
+# serially in fixed order. Starts with the bogus Machine leak cluster
+# (bz_2073398 cleanup, then Machine trio), then known suite-load flakes
+# that pass alone but time out in the randomized batch (egressIP FIP
+# failover AAP wait, prometheus PVC resize pod-ready wait), then topology
+# pair ordered so the AZ check runs while enable_topology is still true
+# before the mutator ([Serial] set enable_topology=false) can race it in
+# batch. Keeps the bulk suite in batch to limit wall time. No-ops when
+# the matchers are absent (e.g. lb_tests allowlists). Requires OTE
+# filtered run path.
 openstack_test_machines_leak_cluster_serial: true
 # Ordered substrings matched against list_of_tests_to_run.txt (one phrase
 # each; written to a matchers file so shell never splits them). For the
diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
index 5f0f6562..ccf137f6 100644
--- a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
+++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py
@@ -1,14 +1,18 @@
 #!/usr/bin/env python3
-"""Split Option B serial-coda tests out of an OTE test list.
+"""Split serial-coda tests out of an OTE test list.
 
 Reads the filtered list_of_tests_to_run.txt and an ordered matchers file
 (one substring per line), writes:
-  - leak_cluster_serial.txt: ordered matches for Option B serial run
+  - leak_cluster_serial.txt: ordered matches for serial coda
     (Machine leak cluster, suite-load flakes, topology pair)
   - list_of_tests_to_run.txt: remaining tests for the batch suite
 
 Matchers are plain substrings (one full phrase per line). The MachineSet
 replica matcher skips any line that also contains "ControlPlane".
+
+When any matcher hits, every matcher must resolve to exactly one test;
+missing or non-unique matches fail hard. Zero hits keep the allowlist
+no-op (empty serial file, batch unchanged).
 """
 from __future__ import annotations
 
@@ -22,7 +26,9 @@ def load_matchers(path: str) -> list[str]:
         return [ln.strip() for ln in f if ln.strip() and not ln.strip().startswith("#")]
 
 
-def pick(remaining: list[str], matcher: str) -> str | None:
+def pick(remaining: list[str], matcher: str) -> list[str]:
+    """Return all candidate lines for matcher (ControlPlane skipped for MachineSet replica)."""
+    candidates: list[str] = []
     for ln in remaining:
         if not ln.strip():
             continue
@@ -32,8 +38,8 @@ def pick(remaining: list[str], matcher: str) -> str | None:
         if "MachineSet replica number corresponds to the number of Machines" in matcher:
             if "ControlPlane" in ln:
                 continue
-        return ln
-    return None
+        candidates.append(ln)
+    return candidates
 
 
 def main() -> int:
@@ -53,14 +59,39 @@ def main() -> int:
         lines = [ln.rstrip("\n") for ln in f]
 
     remaining = list(lines)
-    selected: list[str] = []
+    resolved: list[tuple[str, list[str]]] = []
+    any_hit = False
 
     for matcher in matchers:
-        found = pick(remaining, matcher)
-        if found is None:
-            continue
-        selected.append(found)
-        remaining = [ln for ln in remaining if ln != found]
+        candidates = pick(remaining, matcher)
+        if candidates:
+            any_hit = True
+        resolved.append((matcher, candidates))
+
+    if not any_hit:
+        # Allowlist no-op: none of the coda tests are in this filtered list.
+        print("leak_cluster=0 batch_remaining=unchanged (no matchers hit)")
+        Path(args.leak_cluster_path).write_text("", encoding="utf-8")
+        return 0
+
+    selected: list[str] = []
+    for matcher, candidates in resolved:
+        if len(candidates) == 0:
+            print(
+                f"error: matcher matched no tests: {matcher!r}",
+                file=sys.stderr,
+            )
+            return 1
+        if len(candidates) > 1:
+            print(
+                f"error: matcher matched {len(candidates)} tests (expected 1): {matcher!r}",
+                file=sys.stderr,
+            )
+            for c in candidates:
+                print(f"  candidate: {c}", file=sys.stderr)
+            return 1
+        selected.append(candidates[0])
+        remaining = [ln for ln in remaining if ln != candidates[0]]
 
     with open(args.leak_cluster_path, "w", encoding="utf-8") as f:
         for ln in selected:
diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
index 589d8297..4af26764 100644
--- a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml
@@ -1,5 +1,5 @@
 ---
-# Option B: serially run the ordered coda list after the main batch suite
+# Serial coda: serially run the ordered coda list after the main batch suite
 # (Machine leak cluster, egressIP / prometheus, then topology AZ check
 # before enable_topology=false mutator).
 # Expects openstack_test_leak_cluster_path to already list the ordered tests.
diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
index 06693dfc..70d5bdb8 100644
--- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
+++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml
@@ -141,8 +141,8 @@
   ansible.builtin.set_fact:
     openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}"
 
-# Option B: pull serial-coda matchers (Machine leak cluster + suite-load
-# flakes) out of the batch list for an ordered serial run after the batch.
+# Serial coda: pull matchers (Machine leak cluster + suite-load flakes)
+# out of the batch list for an ordered serial run after the batch.
 - name: Set machines leak-cluster serial list path
   ansible.builtin.set_fact:
     openstack_test_leak_cluster_path: >-
@@ -433,8 +433,8 @@
       vars:
         must_gather_suffix: "openstack-tests"
 
-# Option B: serial coda after the main batch (or filtered serial) suite.
-- name: Run Option B serial-coda tests after main suite
+# Serial coda after the main batch (or filtered serial) suite.
+- name: Run serial-coda tests after main suite
   ansible.builtin.include_tasks: run_machines_leak_cluster_serial.yml
   when:
     - openstack_test_use_ote | bool