Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 49 additions & 10 deletions dev/breeze/src/airflow_breeze/utils/fork_ci_github.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@
# Network failures before the request reached GitHub, so repeating it is safe for any call.
CONNECTION_ERROR = re.compile(r"TLS handshake timeout|dial tcp")
SERVER_ERROR_WAIT_SECONDS = 10
# GitHub answers HTTP 502 on every attempt when it lists 100 jobs per page of some runs with a few
# hundred jobs, re-run more than once, while it lists smaller pages of the same run. A smaller size is
# tried once the previous one keeps failing with server errors.
JOB_PAGE_SIZES = (100, 30, 15, 5)
JOB_PAGE_SIZE_SERVER_ERROR_ATTEMPTS = 2
# A hung `gh` would otherwise hold the job until its timeout-minutes.
GH_TIMEOUT_SECONDS = 120

Expand All @@ -69,6 +74,7 @@ def _gh(
*,
allow_error_output: bool = False,
retry_server_errors: bool = True,
max_server_error_attempts: int = MAX_ATTEMPTS,
) -> Any:
"""
:param allow_error_output: return the JSON output of a failed call when it has any. ``gh api
Expand All @@ -77,6 +83,8 @@ def _gh(
:param retry_server_errors: repeat a call that failed with a server error. Off for a call that
may have taken effect anyway and must not be repeated, such as creating a comment. A call
that failed to connect is repeated either way.
:param max_server_error_attempts: how many times the call is made before giving up on server
errors; rate limits and connection errors are retried up to ``MAX_ATTEMPTS`` times either way.
"""
payload = json.dumps(body) if body is not None else None
rate_limited = False
Expand All @@ -98,12 +106,10 @@ def _gh(
rate_limited = any(marker in result.stderr.lower() for marker in RATE_LIMIT_MARKERS)
self.rate_limited += rate_limited
connection_error = bool(CONNECTION_ERROR.search(result.stderr))
if (
rate_limited
or connection_error
or (retry_server_errors and SERVER_ERROR.search(result.stderr))
):
if attempt == MAX_ATTEMPTS - 1:
server_error = not rate_limited and not connection_error and SERVER_ERROR.search(result.stderr)
if rate_limited or connection_error or (retry_server_errors and server_error):
last_attempt = max_server_error_attempts if server_error else MAX_ATTEMPTS
if attempt >= last_attempt - 1:
break
reason = (
"rate limit"
Expand All @@ -123,9 +129,9 @@ def _gh(
pass
raise GitHubError(f"gh {' '.join(args)} failed: {result.stderr.strip()}")
if rate_limited:
raise GitHubError(f"gh {' '.join(args)} still rate limited after {MAX_ATTEMPTS} attempts")
raise GitHubError(f"gh {' '.join(args)} still rate limited after {attempt + 1} attempts")
raise GitHubError(
f"gh {' '.join(args)} still failing after {MAX_ATTEMPTS} attempts: {result.stderr.strip()}"
f"gh {' '.join(args)} still failing after {attempt + 1} attempts: {result.stderr.strip()}"
)

def graphql(
Expand All @@ -149,20 +155,53 @@ def graphql(
console_print(f"[warning]Partial GraphQL result: {response['errors']}[/]")
return response["data"]

def rest_get(self, path: str, *, paginate: bool = False, items_key: str | None = None) -> Any:
def rest_get(
self,
path: str,
*,
paginate: bool = False,
items_key: str | None = None,
max_server_error_attempts: int = MAX_ATTEMPTS,
) -> Any:
"""
GET a REST path.

:param paginate: follow every page and return the concatenated items.
:param items_key: for endpoints whose pages are objects, the key holding the items
(``workflow_runs``, ``jobs``, ``files``); pages are plain lists otherwise.
:param max_server_error_attempts: how many times the call is made before giving up on server
errors.
"""
args = ["api", path, *(["--paginate", "--slurp"] if paginate else [])]
result = self._gh(args, None, "rest")
result = self._gh(args, None, "rest", max_server_error_attempts=max_server_error_attempts)
if paginate:
return [item for page in result for item in (page[items_key] if items_key else page)]
return result

def get_run_jobs(self, repo: str, run_id: int) -> list[dict[str, Any]]:
"""The jobs of a workflow run's latest attempt, in smaller pages while GitHub fails to list them."""

def get_jobs(page_size: int, max_server_error_attempts: int) -> list[dict[str, Any]]:
return self.rest_get(
f"repos/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page={page_size}",
paginate=True,
items_key="jobs",
max_server_error_attempts=max_server_error_attempts,
)

*larger, smallest = JOB_PAGE_SIZES
for page_size in larger:
try:
return get_jobs(page_size, JOB_PAGE_SIZE_SERVER_ERROR_ATTEMPTS)
except GitHubError as e:
if not SERVER_ERROR.search(str(e)):
raise
console_print(
f"[warning]Listing the jobs of {repo} run {run_id} {page_size} per page keeps "
"failing, trying smaller pages[/]"
)
return get_jobs(smallest, MAX_ATTEMPTS)

def rest_get_pages(self, path: str) -> list[Any]:
"""GET every page of a paginated REST path, keeping page-level fields such as ``total_count``."""
return self._gh(["api", path, "--paginate", "--slurp"], None, "rest")
Expand Down
12 changes: 2 additions & 10 deletions dev/breeze/src/airflow_breeze/utils/fork_ci_reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -1418,11 +1418,7 @@ def report(self, pr: PullRequestSnapshot, fork: ForkObservation) -> str | None:
if self.remaining == 0:
self._defer(pr, fork)
return None
jobs = self._client.rest_get(
f"repos/{pr.head_repo}/actions/runs/{fork.run_id}/jobs?filter=latest&per_page=100",
paginate=True,
items_key="jobs",
)
jobs = self._client.get_run_jobs(pr.head_repo, fork.run_id)
# By the case-folded context, as GitHub compares them.
current = {
status["context"].casefold(): (
Expand Down Expand Up @@ -1575,11 +1571,7 @@ def __init__(

def _get_jobs(self, run: WorkflowRun) -> list[dict[str, Any]]:
if run.id not in self._jobs:
self._jobs[run.id] = self._client.rest_get(
f"repos/{self._repo}/actions/runs/{run.id}/jobs?filter=latest&per_page=100",
paginate=True,
items_key="jobs",
)
self._jobs[run.id] = self._client.get_run_jobs(self._repo, run.id)
return self._jobs[run.id]

def _did_run_project_ci(self, run: WorkflowRun) -> bool:
Expand Down
69 changes: 69 additions & 0 deletions dev/breeze/tests/test_fork_ci_github.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,3 +238,72 @@ def test_set_status_context(mock_run, kwargs, context):
GitHubClient().set_status(1, "abc", "success", "https://x", "desc", **kwargs)

assert json.loads(mock_run.call_args.kwargs["input"])["context"] == context


def _jobs_page(*names: str) -> str:
return json.dumps([{"total_count": len(names), "jobs": [{"name": name} for name in names]}])


def _jobs_path(page_size: int) -> str:
return f"repos/someone/airflow/actions/runs/7/jobs?filter=latest&per_page={page_size}"


BAD_GATEWAY = "gh: Server Error (HTTP 502)"


@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True)
@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True)
def test_lists_run_jobs_in_smaller_pages_while_github_fails_to_list_them(mock_run, mock_sleep):
mock_run.side_effect = [
*[_completed(1, stderr=BAD_GATEWAY)] * 4,
_completed(0, stdout=_jobs_page("Tests", "Docs")),
]

jobs = GitHubClient().get_run_jobs("someone/airflow", 7)

assert jobs == [{"name": "Tests"}, {"name": "Docs"}]
assert [c.args[0][2] for c in mock_run.call_args_list] == [
_jobs_path(100),
_jobs_path(100),
_jobs_path(30),
_jobs_path(30),
_jobs_path(15),
]


@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True)
@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True)
def test_gives_up_listing_run_jobs_when_the_smallest_pages_keep_failing(mock_run, mock_sleep):
mock_run.return_value = _completed(1, stderr=BAD_GATEWAY)

with pytest.raises(GitHubError, match=r"per_page=5 --paginate --slurp still failing after 4 attempts"):
GitHubClient().get_run_jobs("someone/airflow", 7)
assert [c.args[0][2] for c in mock_run.call_args_list] == [
*[_jobs_path(100)] * 2,
*[_jobs_path(30)] * 2,
*[_jobs_path(15)] * 2,
*[_jobs_path(5)] * 4,
]


@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True)
@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True)
def test_listing_run_jobs_retries_a_rate_limit_as_often_as_any_call(mock_run, mock_sleep):
mock_run.side_effect = [
*[_completed(1, stderr="API rate limit exceeded")] * 3,
_completed(0, stdout=_jobs_page("Tests")),
]

assert GitHubClient().get_run_jobs("someone/airflow", 7) == [{"name": "Tests"}]
assert [c.args[0][2] for c in mock_run.call_args_list] == [_jobs_path(100)] * 4
assert [c.args[0] for c in mock_sleep.call_args_list] == [30, 60, 120]


@mock.patch("airflow_breeze.utils.fork_ci_github.time.sleep", autospec=True)
@mock.patch("airflow_breeze.utils.fork_ci_github.subprocess.run", autospec=True)
def test_does_not_shrink_run_job_pages_for_errors_other_than_server_errors(mock_run, mock_sleep):
mock_run.return_value = _completed(1, stderr="gh: Not Found (HTTP 404)")

with pytest.raises(GitHubError, match="HTTP 404"):
GitHubClient().get_run_jobs("someone/airflow", 7)
mock_run.assert_called_once()
15 changes: 6 additions & 9 deletions dev/breeze/tests/test_fork_ci_reconcile.py
Original file line number Diff line number Diff line change
Expand Up @@ -957,8 +957,6 @@ def rest_get_pages(path):
return [{"total_count": count, "workflow_runs": event_runs}]

def rest_get(path, paginate=False, items_key=None):
if path.endswith("jobs?filter=latest&per_page=100"):
return list(jobs)
if "/compare/" in path:
assert not paginate, "the compare API returns files only on its first page"
return {"files": [{"filename": name} for name in compare_files]}
Expand All @@ -968,6 +966,7 @@ def rest_get(path, paginate=False, items_key=None):

client.rest_get_pages.side_effect = rest_get_pages
client.rest_get.side_effect = rest_get
client.get_run_jobs.return_value = list(jobs)
return BypassFinder(client, repo=REPO, now=NOW, base_triggers=BASE_TRIGGERS, is_gated_base=is_gated_base)


Expand Down Expand Up @@ -1154,9 +1153,7 @@ def test_finder_reports_a_run_another_pull_request_may_own(client, other_pulls):
def test_finder_catches_a_workflow_moved_away(client):
finder = _finder(client, [_run(7)], jobs=TAMPERED_JOBS)
client.rest_get.side_effect = lambda path, paginate=False, items_key=None: (
TAMPERED_JOBS
if path.endswith("jobs?filter=latest&per_page=100")
else [{"number": 42, "closed_at": None}]
[{"number": 42, "closed_at": None}]
if "/pulls?head=" in path
else {"files": [{"filename": "docs/x.yml", "previous_filename": ".github/workflows/x.yml"}]}
)
Expand Down Expand Up @@ -2111,13 +2108,12 @@ def test_run_reconcile_deletes_gated_runs(mock_classify, client, tmp_path, state
def rest_get(path, paginate=False, items_key=None):
if path.endswith("/actions/workflows/ci-amd.yml"):
return {"state": "active"}
if path.endswith("jobs?filter=latest&per_page=100"):
return GATED_JOBS
if "/compare/" in path:
return {"files": [{"filename": "README.md"}]}
raise AssertionError(path)

client.rest_get.side_effect = rest_get
client.get_run_jobs.return_value = GATED_JOBS
(tmp_path / "ci-amd.yml").write_text("on: [pull_request]\n")

run_reconcile(
Expand Down Expand Up @@ -2438,13 +2434,14 @@ def test_build_job_statuses_cannot_post_the_reconciler_contexts(name):

def _reporter_client(client, jobs, current=()):
def rest_get(path, paginate=False, items_key=None):
if path.startswith("repos/someone/airflow/actions/runs/1/jobs"):
return jobs
if path.startswith(f"repos/{REPO}/commits/abc123/status"):
return list(current)
raise AssertionError(path)

client.rest_get.side_effect = rest_get
client.get_run_jobs.side_effect = lambda repo, run_id: (
jobs if (repo, run_id) == ("someone/airflow", 1) else pytest.fail(f"{repo} run {run_id}")
)
return client


Expand Down
Loading