From 81845b28aea953a0304f06f33f0aa31419f3bb5b Mon Sep 17 00:00:00 2001 From: ilkedemir Date: Wed, 16 Sep 2026 22:44:04 -0700 Subject: [PATCH 1/6] Experimenting with managing the list --- .github/workflows/check-links.yml | 83 ++++++++++++++++ .github/workflows/check_links.py | 154 ++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 .github/workflows/check-links.yml create mode 100644 .github/workflows/check_links.py diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml new file mode 100644 index 0000000..dd10a68 --- /dev/null +++ b/.github/workflows/check-links.yml @@ -0,0 +1,83 @@ +name: Check Links Quarterly + +on: + schedule: + - cron: '0 0 1 */3 *' # 00:00 UTC on the 1st of Jan, Apr, Jul, Oct + workflow_dispatch: # lets you trigger a run manually from the Actions tab + +env: + FILE_TO_CHECK: softbinding-algorithm-list.json + +jobs: + check-links: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: pip install requests + + - name: Check links in file + id: check + run: python scripts/check_links.py --file "$FILE_TO_CHECK" + continue-on-error: true + + - name: Upload report artifact + uses: actions/upload-artifact@v4 + with: + name: link-check-report + path: link_check_report.json + + - name: Create or update issue on broken links + if: steps.check.outcome == 'failure' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const report = JSON.parse(fs.readFileSync('link_check_report.json', 'utf8')); + const broken = report.results.filter(r => !r.ok); + + const title = `Broken links found in ${report.file}`; + const body = [ + `The quarterly link check found ${broken.length} broken link(s) in \`${report.file}\`:`, + '', + ...broken.map(b => `- [ ] ${b.url} — ${b.error}`), + '', + `_Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}_` + ].join('\n'); + + const { data: issues } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'broken-links' + }); + + const existing = issues.find(i => i.title === title); + + if (existing) { + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: ['broken-links'] + }); + } + + - name: Fail workflow if broken links were found + if: steps.check.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/check_links.py b/.github/workflows/check_links.py new file mode 100644 index 0000000..bf8e08a --- /dev/null +++ b/.github/workflows/check_links.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +check_links.py + +Scans the softbinging-algorithm-list json and validates "informationalUrl" links. +Exports summary to GITHUB_STEP_SUMMARY and a JSON report to link_check_report.json. + +Links that respond with a status commonly used by bot-protection services +(403, 429, 999) are reported separately as "blocked" rather than "broken". + +Usage: + python check_links.py --file path/to/file.md [--timeout 15] [--retries 2] +""" +import argparse +import json +import os +import re +import sys +import time + +import requests + +# Matches the value of an "informationalUrl" key, e.g.: +# "informationalUrl": "https://example.com/page" +INFORMATIONAL_URL_RE = re.compile(r'"informationalUrl"\s*:\s*"(https?://[^"]+)"') + +# Status codes that typically indicate bot/WAF blocking rather than a dead link. +SOFT_FAIL_CODES = {403, 429, 999} + +# Headers that mimic a real browser request. +HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Connection": "keep-alive", +} + + +def extract_urls(text): + urls = INFORMATIONAL_URL_RE.findall(text) + seen = set() + result = [] + for u in urls: + if u not in seen: + seen.add(u) + result.append(u) + return result + + +def check_url(url, timeout, retries): + """ + Returns (state, status, error) where state is one of: + "ok" - link is reachable + "blocked" - site responded with a bot-protection style status code + "broken" - link genuinely appears dead + """ + last_status = None + last_error = None + + for attempt in range(retries + 1): + try: + # Use GET (not HEAD) since many bot-protection layers block or + # mishandle HEAD requests outright. + resp = requests.get( + url, allow_redirects=True, timeout=timeout, headers=HEADERS, stream=True + ) + status = resp.status_code + if status < 400: + return "ok", status, None + + last_status = status + if status in SOFT_FAIL_CODES: + last_error = f"HTTP {status} (site may be blocking automated requests)" + else: + last_error = f"HTTP {status}" + except requests.RequestException as e: + last_error = str(e) + + if attempt < retries: + time.sleep(3 * (attempt + 1)) # back off a bit longer each retry + + if last_status in SOFT_FAIL_CODES: + return "blocked", last_status, last_error + return "broken", last_status, last_error + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--file", required=True, help="Path to the file containing links to check") + parser.add_argument("--timeout", type=int, default=15) + parser.add_argument("--retries", type=int, default=2) + args = parser.parse_args() + + if not os.path.isfile(args.file): + print(f"::error::File not found: {args.file}") + sys.exit(1) + + with open(args.file, "r", encoding="utf-8", errors="ignore") as f: + text = f.read() + + urls = extract_urls(text) + print(f"Found {len(urls)} unique URL(s) in {args.file}") + + results = [] + broken = [] + blocked = [] + + for url in urls: + state, status, error = check_url(url, args.timeout, args.retries) + results.append({"url": url, "state": state, "status": status, "error": error}) + print(f"[{state.upper()}] {url} ({status or error})") + if state == "broken": + broken.append({"url": url, "error": error}) + elif state == "blocked": + blocked.append({"url": url, "error": error}) + + with open("link_check_report.json", "w") as f: + json.dump({"file": args.file, "results": results}, f, indent=2) + + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with open(summary_path, "a") as f: + f.write(f"## Link check results for `{args.file}`\n\n") + f.write( + f"Checked {len(urls)} link(s). {len(broken)} broken, " + f"{len(blocked)} possibly blocked by bot protection.\n\n" + ) + if broken: + f.write("### Broken\n\n| URL | Error |\n|---|---|\n") + for b in broken: + f.write(f"| {b['url']} | {b['error']} |\n") + f.write("\n") + if blocked: + f.write("### Blocked (please verify manually)\n\n| URL | Error |\n|---|---|\n") + for b in blocked: + f.write(f"| {b['url']} | {b['error']} |\n") + + if blocked: + print(f"::warning::{len(blocked)} link(s) returned bot-protection style responses; verify manually.") + + if broken: + print(f"::error::{len(broken)} broken link(s) found in {args.file}") + sys.exit(1) + + print("All links OK (some may be unverifiable due to bot protection; see summary).") + sys.exit(0) + + +if __name__ == "__main__": + main() From e3f70056d341017bf2f07c807d559cb8a0a4a831 Mon Sep 17 00:00:00 2001 From: ilkedemir Date: Wed, 16 Sep 2026 22:47:09 -0700 Subject: [PATCH 2/6] Update check-links.yml --- .github/workflows/check-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index dd10a68..c209dda 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -25,7 +25,7 @@ jobs: - name: Check links in file id: check - run: python scripts/check_links.py --file "$FILE_TO_CHECK" + run: python check_links.py --file "$FILE_TO_CHECK" continue-on-error: true - name: Upload report artifact From 6870ea26418f98925f6716f535365cc337201d79 Mon Sep 17 00:00:00 2001 From: ilkedemir Date: Wed, 16 Sep 2026 22:49:18 -0700 Subject: [PATCH 3/6] Update check-links.yml --- .github/workflows/check-links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index c209dda..4718f5e 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -25,7 +25,7 @@ jobs: - name: Check links in file id: check - run: python check_links.py --file "$FILE_TO_CHECK" + run: python .github/workflows/check_links.py --file "$FILE_TO_CHECK" continue-on-error: true - name: Upload report artifact From 6add1a945c1b557e1936badf462e17d790992ef5 Mon Sep 17 00:00:00 2001 From: ilkedemir Date: Wed, 16 Sep 2026 23:01:51 -0700 Subject: [PATCH 4/6] Update check-links.yml --- .github/workflows/check-links.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index 4718f5e..8821a55 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -5,6 +5,10 @@ on: - cron: '0 0 1 */3 *' # 00:00 UTC on the 1st of Jan, Apr, Jul, Oct workflow_dispatch: # lets you trigger a run manually from the Actions tab +permissions: + contents: read + issues: write # required for actions/github-script to create/update issues + env: FILE_TO_CHECK: softbinding-algorithm-list.json @@ -41,7 +45,7 @@ jobs: script: | const fs = require('fs'); const report = JSON.parse(fs.readFileSync('link_check_report.json', 'utf8')); - const broken = report.results.filter(r => !r.ok); + const broken = report.results.filter(r => r.state === 'broken'); const title = `Broken links found in ${report.file}`; const body = [ From ec8156bf93cb124d16a3adeaa873f9e04cf14917 Mon Sep 17 00:00:00 2001 From: ilkedemir Date: Wed, 30 Sep 2026 07:53:11 -0700 Subject: [PATCH 5/6] Adding Dom's changes to test --- .github/workflows/check-links.yml | 84 +++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index 8821a55..cde4143 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -1,8 +1,8 @@ -name: Check Links Quarterly +name: Check Links Monthly on: schedule: - - cron: '0 0 1 */3 *' # 00:00 UTC on the 1st of Jan, Apr, Jul, Oct + - cron: '17 3 1 * *' # 03:17 UTC on the first day of each month workflow_dispatch: # lets you trigger a run manually from the Actions tab permissions: @@ -33,37 +33,85 @@ jobs: continue-on-error: true - name: Upload report artifact + if: always() && hashFiles('link_check_report.json') != '' uses: actions/upload-artifact@v4 with: name: link-check-report path: link_check_report.json - - name: Create or update issue on broken links - if: steps.check.outcome == 'failure' + - name: Reconcile broken-links issue + if: steps.check.outputs.result != '' uses: actions/github-script@v7 with: script: | const fs = require('fs'); const report = JSON.parse(fs.readFileSync('link_check_report.json', 'utf8')); const broken = report.results.filter(r => r.state === 'broken'); + const blocked = report.results.filter(r => r.state === 'blocked'); + const unreachable = report.results.filter(r => r.state === 'unreachable'); const title = `Broken links found in ${report.file}`; - const body = [ - `The quarterly link check found ${broken.length} broken link(s) in \`${report.file}\`:`, - '', - ...broken.map(b => `- [ ] ${b.url} — ${b.error}`), - '', - `_Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}_` - ].join('\n'); - - const { data: issues } = await github.rest.issues.listForRepo({ + const issues = await github.paginate(github.rest.issues.listForRepo, { owner: context.repo.owner, repo: context.repo.repo, state: 'open', - labels: 'broken-links' + per_page: 100 }); + const existing = issues.find(i => !i.pull_request && i.title === title); - const existing = issues.find(i => i.title === title); + if (!broken.length) { + if (existing && !blocked.length && !unreachable.length) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: `All links passed the latest check. Closing this issue.\n\n_Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}_` + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed' + }); + } else if (existing) { + const unverified = [...blocked, ...unreachable]; + const table = items => [ + '| Algorithm | Field | URL | Result |', + '|---|---|---|---|', + ...items.map(item => `| ${item.algorithm} | ${item.field} | ${item.url} | ${item.error} |`) + ].join('\n'); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: [ + 'The latest check found no confirmed broken links, but some URLs could not be verified.', + '', + table(unverified), + '', + `_Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}_` + ].join('\n') + }); + } + return; + } + + const table = items => [ + '| Algorithm | Field | URL | Result |', + '|---|---|---|---|', + ...items.map(item => `| ${item.algorithm} | ${item.field} | ${item.url} | ${item.error} |`) + ].join('\n'); + const sections = [ + `The monthly link check found ${broken.length} confirmed broken link(s) in \`${report.file}\`.`, + '', + '## Broken', + '', + table(broken) + ]; + if (blocked.length) sections.push('', '## Blocked', '', table(blocked)); + if (unreachable.length) sections.push('', '## Unreachable', '', table(unreachable)); + sections.push('', `_Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}_`); + const body = sections.join('\n'); if (existing) { await github.rest.issues.update({ @@ -83,5 +131,9 @@ jobs: } - name: Fail workflow if broken links were found - if: steps.check.outcome == 'failure' + if: steps.check.outputs.result == 'broken' + run: exit 1 + + - name: Fail workflow if the checker failed + if: steps.check.outcome == 'failure' && steps.check.outputs.result == '' run: exit 1 From dbd84b6ca7e6d94d37d321fe6c944ddf4bf74aab Mon Sep 17 00:00:00 2001 From: ilkedemir Date: Wed, 30 Sep 2026 07:55:44 -0700 Subject: [PATCH 6/6] Adding Dom's changes to test --- .github/workflows/check_links.py | 156 ++++++++++++++++++++----------- 1 file changed, 100 insertions(+), 56 deletions(-) diff --git a/.github/workflows/check_links.py b/.github/workflows/check_links.py index bf8e08a..e6c9b0e 100644 --- a/.github/workflows/check_links.py +++ b/.github/workflows/check_links.py @@ -2,30 +2,26 @@ """ check_links.py -Scans the softbinging-algorithm-list json and validates "informationalUrl" links. +Scans the softbinding algorithm list and validates its external links. Exports summary to GITHUB_STEP_SUMMARY and a JSON report to link_check_report.json. -Links that respond with a status commonly used by bot-protection services -(403, 429, 999) are reported separately as "blocked" rather than "broken". +Links are classified as ok, broken, blocked, or unreachable. Only HTTP 404 +and 410 responses are treated as confirmed broken links. Usage: - python check_links.py --file path/to/file.md [--timeout 15] [--retries 2] + python check_links.py --file path/to/file.json [--timeout 15] [--retries 2] """ import argparse import json import os -import re import sys import time import requests -# Matches the value of an "informationalUrl" key, e.g.: -# "informationalUrl": "https://example.com/page" -INFORMATIONAL_URL_RE = re.compile(r'"informationalUrl"\s*:\s*"(https?://[^"]+)"') - # Status codes that typically indicate bot/WAF blocking rather than a dead link. -SOFT_FAIL_CODES = {403, 429, 999} +BLOCKED_CODES = {401, 403, 429, 999} +BROKEN_CODES = {404, 410} # Headers that mimic a real browser request. HEADERS = { @@ -40,14 +36,22 @@ } -def extract_urls(text): - urls = INFORMATIONAL_URL_RE.findall(text) - seen = set() +def extract_urls(entries): + """Return URLs together with their algorithm and source field.""" result = [] - for u in urls: - if u not in seen: - seen.add(u) - result.append(u) + for entry in entries: + algorithm = entry.get("alg", "") + metadata = entry.get("entryMetadata", {}) + candidates = [ + ("informationalUrl", metadata.get("informationalUrl")), + *( + ("softBindingResolutionApis", url) + for url in entry.get("softBindingResolutionApis", []) + ), + ] + for field, url in candidates: + if url: + result.append({"algorithm": algorithm, "field": field, "url": url}) return result @@ -56,36 +60,51 @@ def check_url(url, timeout, retries): Returns (state, status, error) where state is one of: "ok" - link is reachable "blocked" - site responded with a bot-protection style status code - "broken" - link genuinely appears dead + "broken" - resource is confirmed absent (HTTP 404 or 410) + "unreachable" - server or network failure prevents verification """ - last_status = None - last_error = None - for attempt in range(retries + 1): try: # Use GET (not HEAD) since many bot-protection layers block or # mishandle HEAD requests outright. - resp = requests.get( + with requests.get( url, allow_redirects=True, timeout=timeout, headers=HEADERS, stream=True - ) - status = resp.status_code + ) as response: + status = response.status_code if status < 400: return "ok", status, None - - last_status = status - if status in SOFT_FAIL_CODES: - last_error = f"HTTP {status} (site may be blocking automated requests)" + if status in BROKEN_CODES: + state = "broken" + error = f"HTTP {status}" + elif status in BLOCKED_CODES: + state = "blocked" + error = f"HTTP {status} (site may be blocking automated requests)" else: - last_error = f"HTTP {status}" + state = "unreachable" + error = f"HTTP {status}" except requests.RequestException as e: - last_error = str(e) + state = "unreachable" + status = None + error = str(e) if attempt < retries: time.sleep(3 * (attempt + 1)) # back off a bit longer each retry - if last_status in SOFT_FAIL_CODES: - return "blocked", last_status, last_error - return "broken", last_status, last_error + return state, status, error + + +def markdown_table(items): + lines = ["| Algorithm | Field | URL | Result |", "|---|---|---|---|"] + for item in items: + values = ( + item["algorithm"], + item["field"], + item["url"], + item["error"], + ) + escaped = [str(value).replace("|", "\\|").replace("\n", " ") for value in values] + lines.append("| " + " | ".join(escaped) + " |") + return "\n".join(lines) def main(): @@ -97,26 +116,45 @@ def main(): if not os.path.isfile(args.file): print(f"::error::File not found: {args.file}") - sys.exit(1) + sys.exit(2) + + try: + with open(args.file, "r", encoding="utf-8") as f: + entries = json.load(f) + except (OSError, json.JSONDecodeError) as error: + print(f"::error::Could not read algorithm list: {error}") + sys.exit(2) - with open(args.file, "r", encoding="utf-8", errors="ignore") as f: - text = f.read() + if not isinstance(entries, list): + print("::error::Algorithm list must be a JSON array") + sys.exit(2) - urls = extract_urls(text) - print(f"Found {len(urls)} unique URL(s) in {args.file}") + links = extract_urls(entries) + if not links: + print("::error::No URLs found in the algorithm list") + sys.exit(2) + print(f"Found {len(links)} URL reference(s) in {args.file}") results = [] broken = [] blocked = [] - - for url in urls: - state, status, error = check_url(url, args.timeout, args.retries) - results.append({"url": url, "state": state, "status": status, "error": error}) + unreachable = [] + + checked_urls = {} + for link in links: + url = link["url"] + if url not in checked_urls: + checked_urls[url] = check_url(url, args.timeout, args.retries) + state, status, error = checked_urls[url] + result = {**link, "state": state, "status": status, "error": error} + results.append(result) print(f"[{state.upper()}] {url} ({status or error})") if state == "broken": - broken.append({"url": url, "error": error}) + broken.append(result) elif state == "blocked": - blocked.append({"url": url, "error": error}) + blocked.append(result) + elif state == "unreachable": + unreachable.append(result) with open("link_check_report.json", "w") as f: json.dump({"file": args.file, "results": results}, f, indent=2) @@ -126,27 +164,33 @@ def main(): with open(summary_path, "a") as f: f.write(f"## Link check results for `{args.file}`\n\n") f.write( - f"Checked {len(urls)} link(s). {len(broken)} broken, " - f"{len(blocked)} possibly blocked by bot protection.\n\n" + f"Checked {len(links)} link(s): {len(broken)} broken, " + f"{len(blocked)} blocked, and {len(unreachable)} unreachable.\n\n" ) - if broken: - f.write("### Broken\n\n| URL | Error |\n|---|---|\n") - for b in broken: - f.write(f"| {b['url']} | {b['error']} |\n") - f.write("\n") - if blocked: - f.write("### Blocked (please verify manually)\n\n| URL | Error |\n|---|---|\n") - for b in blocked: - f.write(f"| {b['url']} | {b['error']} |\n") + for title, items in ( + ("Broken", broken), + ("Blocked (please verify manually)", blocked), + ("Unreachable (please retry or verify manually)", unreachable), + ): + if items: + f.write(f"### {title}\n\n{markdown_table(items)}\n\n") if blocked: print(f"::warning::{len(blocked)} link(s) returned bot-protection style responses; verify manually.") + if unreachable: + print(f"::warning::{len(unreachable)} link(s) could not be verified; retry or verify manually.") + + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as f: + f.write(f"result={'broken' if broken else 'healthy'}\n") + f.write(f"unverified={'true' if blocked or unreachable else 'false'}\n") if broken: print(f"::error::{len(broken)} broken link(s) found in {args.file}") sys.exit(1) - print("All links OK (some may be unverifiable due to bot protection; see summary).") + print("No confirmed broken links found; see the summary for unverifiable links.") sys.exit(0)