Fix token-usage report aggregation at production scale

.github/workflows/performance-report.yml

@@ -0,0 +1,158 @@
+name: Update PR performance comment
+on:
+  workflow_call:
+    inputs:
+      section:
+        required: true
+        type: string
+      artifact-name:
+        required: true
+        type: string
+      measurement-result:
+        required: true
+        type: string
+permissions:
+  contents: read
+  pull-requests: write
+jobs:
+  publish:
+    name: Update existing performance comment
+    if: >-
+      github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    # Both performance workflows read and update the same comment. Serialize the
+    # entire job so each merge sees the other publisher's completed update.
+    concurrency:
+      group: pr-performance-comment-${{ github.repository }}-${{ github.event.pull_request.number }}
+      cancel-in-progress: false
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        id: download
+        continue-on-error: true
+        with:
+          name: ${{ inputs.artifact-name }}
+          path: report
+      # This write-enabled job never checks out the PR or executes artifact code.
+      - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          REPORT_SECTION: ${{ inputs.section }}
+          MEASUREMENT_RESULT: ${{ inputs.measurement-result }}
+          DOWNLOAD_OUTCOME: ${{ steps.download.outcome }}
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- devfeed-pr-performance -->';
+            const legacyToken = '<!-- devfeed-token-usage-performance -->';
+            const sections = ['token-usage', 'reader-api'];
+            const titles = {'token-usage': 'Token usage benchmark', 'reader-api': 'Reader API performance'};
+            const section = process.env.REPORT_SECTION;
+            if (!sections.includes(section)) throw new Error('Unknown performance report section');
+            const pr = context.payload.pull_request;
+            const sha = pr.head.sha;
+            const run = String(context.runId);
+            const attempt = process.env.GITHUB_RUN_ATTEMPT;
+            if (!/^[a-f0-9]{40}$/.test(sha) || !/^[1-9]\d*$/.test(run) || !/^[1-9]\d*$/.test(attempt)) {
+              throw new Error('Invalid performance report provenance');
+            }
+            const current = await github.rest.pulls.get({...context.repo, pull_number: pr.number});
+            if (current.data.head.sha !== sha) {
+              core.info('Skipping results for a superseded PR head.');
+              return;
+            }
+            const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${run}`;
+            let summary;
+            let missing = false;
+            try {
+              if (process.env.DOWNLOAD_OUTCOME !== 'success') throw new Error('Artifact unavailable');
+              const stat = fs.lstatSync('report/summary.md');
+              if (!stat.isFile() || stat.size > 25000) throw new Error('Invalid summary file');
+              summary = fs.readFileSync('report/summary.md', 'utf8');
+              const valid = section === 'token-usage'
+                ? /^\*\*(PASS|FAIL|INCOMPLETE) — Token usage benchmark:\*\*/.test(summary)
+                : /^(# PR reader API performance|\*\*ERROR\*\*)/.test(summary);
+              if (!valid) throw new Error('Invalid summary verdict');
+            } catch {
+              missing = true;
+              summary = `**INCOMPLETE — ${titles[section]}:** no readable results artifact was produced. Inspect the job logs; no performance pass can be claimed.`;
+            }
+            const outcome = process.env.MEASUREMENT_RESULT;
+            if (outcome !== 'success') {
+              const verdict = outcome === 'failure' ? 'FAIL' : 'INCOMPLETE';
+              if (section === 'token-usage') {
+                summary = summary.replace(/^\*\*(PASS|FAIL|INCOMPLETE)/, `**${verdict}`);
+              } else {
+                summary = `**${verdict} — Reader API performance:** the comparison job did not complete successfully.\n\n${summary}`;
+              }
+              summary += `\n\nMeasurement job outcome: **${['failure', 'cancelled', 'skipped'].includes(outcome) ? outcome : 'unknown'}**. The run did not complete successfully.`;
+            }
+            // Artifact text is inert Markdown. Neutralize HTML comment delimiters
+            // so it cannot forge the publisher's section/provenance boundaries.
+            summary = summary.replace(/<!--/g, '&lt;!--');
+            const link = section === 'token-usage' ? 'Full JSON artifact and job logs' : 'Full HTML/CSV reports and logs';
+            const incoming = {head: sha, run, attempt, text: `${summary}\n\n[${link}](${url}) · attempt ${attempt}`};
+            const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: pr.number});
+            const previous = comments.filter(comment =>
+              comment.user?.login === 'github-actions[bot]' && comment.user.type === 'Bot' &&
+              (comment.body?.startsWith(marker) || comment.body?.startsWith(legacyToken))
+            ).sort((a, b) => a.id - b.id);
+            const reports = {};
+            const newer = (left, right) => {
+              if ((left.head === sha) !== (right.head === sha)) return left.head === sha;
+              return BigInt(left.run) > BigInt(right.run) ||
+                (left.run === right.run && BigInt(left.attempt) > BigInt(right.attempt));
+            };
+            const preserve = (key, report) => {
+              if (!reports[key] || newer(report, reports[key])) reports[key] = report;
+            };
+            for (const comment of previous) {
+              const expression = /^<!-- devfeed-performance-section:(token-usage|reader-api) head:([a-f0-9]{40}|unknown) run:([1-9]\d*|0) attempt:([1-9]\d*) -->\n([\s\S]*?)\n<!-- \/devfeed-performance-section:\1 -->$/gm;
+              const matches = [...comment.body.matchAll(expression)];
+              if (matches.length) {
+                for (const match of matches) {
+                  preserve(match[1], {head: match[2], run: match[3], attempt: match[4], text: match[5]});
+                }
+                continue;
+              }
+              // Adopt both older standalone performance comment formats once,
+              // keeping their original content and provenance during migration.
+              const key = comment.body.startsWith(legacyToken) ? 'token-usage' : 'reader-api';
+              const oldHead = comment.body.match(/(?:Compared head:|→ PR)\s*`?([a-f0-9]{40})`?/i)?.[1] ?? 'unknown';
+              const tokenRun = comment.body.match(/<!-- devfeed-token-usage-run:([1-9]\d*) attempt:([1-9]\d*) -->/);
+              const oldRun = tokenRun?.[1] ?? [...comment.body.matchAll(/\/actions\/runs\/([1-9]\d*)/g)].at(-1)?.[1] ?? '0';
+              const oldAttempt = tokenRun?.[2] ?? [...comment.body.matchAll(/\battempt\s+([1-9]\d*)\b/gi)].at(-1)?.[1] ?? '1';
+              const text = comment.body.replace(/^<!--[^\n]*-->\n/gm, '').trim().replace(/<!--/g, '&lt;!--');
+              preserve(key, {head: oldHead, run: oldRun, attempt: oldAttempt, text});
+            }
+            if (reports[section] && newer(reports[section], incoming)) {
+              core.info('Keeping the newer results for this performance section.');
+              return;
+            }
+            reports[section] = incoming;
+            const rendered = sections.map(key => {
+              const report = reports[key];
+              if (!report) return `## ${titles[key]}\n\n**Pending:** no results have been published.`;
+              const stale = report.head !== sha
+                ? `**Previous head:** these results are from \`${report.head}\`; awaiting results for the current PR head.\n\n`
+                : '';
+              return `## ${titles[key]}\n\n${stale}Source head: \`${report.head}\` · run ${report.run} · attempt ${report.attempt}\n\n<!-- devfeed-performance-section:${key} head:${report.head} run:${report.run} attempt:${report.attempt} -->\n${report.text}\n<!-- /devfeed-performance-section:${key} -->`;
+            });
+            const body = `${marker}\n# PR performance results\n\nCurrent PR head: \`${sha}\`\n\n${rendered.join('\n\n')}`;
+            if (Buffer.byteLength(body, 'utf8') > 60000) {
+              core.setFailed('Combined performance report exceeds the comment limit; existing results were preserved.');
+              return;
+            }
+            const latest = await github.rest.pulls.get({...context.repo, pull_number: pr.number});
+            if (latest.data.head.sha !== sha) return;
+            if (previous.length) {
+              await github.rest.issues.updateComment({...context.repo, comment_id: previous[0].id, body});
+              // Remove only duplicate, explicitly marked Actions performance
+              // comments after their sections have been saved in the oldest one.
+              for (const duplicate of previous.slice(1)) {
+                await github.rest.issues.deleteComment({...context.repo, comment_id: duplicate.id});
+              }
+            } else {
+              await github.rest.issues.createComment({...context.repo, issue_number: pr.number, body});
+            }
+            if (missing) core.setFailed('Published incomplete results: the summary artifact was unavailable or invalid.');

.github/workflows/performance.yml

@@ -153,30 +153,11 @@ jobs:
       always() && !cancelled() && github.event_name == 'pull_request' &&
       github.event.pull_request.head.repo.full_name == github.repository &&
       github.actor != 'renovate[bot]'
-    runs-on: ubuntu-24.04
+    uses: ./.github/workflows/performance-report.yml
     permissions:
       contents: read
       pull-requests: write
-    steps:
-      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          name: pr-performance-${{ github.run_attempt }}
-          path: report
-      - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
-        with:
-          script: |
-            const fs = require('fs');
-            const marker = '<!-- devfeed-pr-performance -->';
-            const pr = context.payload.pull_request;
-            const current = await github.rest.pulls.get({...context.repo, pull_number: pr.number});
-            if (current.data.head.sha !== pr.head.sha) return;
-            const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-            const summary = fs.readFileSync('report/summary.md', 'utf8').slice(0, 50000);
-            const body = `${marker}\n${summary}\n[Full HTML/CSV reports and logs](${url}) · attempt ${process.env.GITHUB_RUN_ATTEMPT}`;
-            const comments = await github.paginate(github.rest.issues.listComments, {...context.repo, issue_number: pr.number});
-            const previous = comments.find(c => c.user.type === 'Bot' && c.body.startsWith(marker));
-            if (previous) {
-              await github.rest.issues.updateComment({...context.repo, comment_id: previous.id, body});
-            } else {
-              await github.rest.issues.createComment({...context.repo, issue_number: pr.number, body});
-            }
+    with:
+      section: reader-api
+      artifact-name: pr-performance-${{ github.run_attempt }}
+      measurement-result: ${{ needs.compare.result }}

.github/workflows/token-usage-performance.yml

@@ -0,0 +1,101 @@
+name: Token usage performance
+on:
+  pull_request:
+    paths:
+      - apps/admin-api/src/automation.py
+      - scripts/profile_token_usage.py
+      - scripts/ci/token-usage-summary.py
+      - tests/tooling/ci/test_token_usage_comment.py
+      - tests/tooling/profiling/test_token_usage_summary.py
+      - .github/workflows/performance-report.yml
+      - .github/workflows/token-usage-performance.yml
+  workflow_dispatch:
+concurrency:
+  group: token-usage-performance-${{ github.ref }}
+  cancel-in-progress: true
+permissions:
+  contents: read
+jobs:
+  benchmark:
+    name: Native ARM64 PostgreSQL 17
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 30
+    services:
+      postgres:
+        image: postgres:17.11-bookworm
+        env:
+          POSTGRES_USER: devfeed
+          POSTGRES_HOST_AUTH_METHOD: trust
+          POSTGRES_DB: devfeed_token_profile_test
+        ports:
+          - 127.0.0.1:5432:5432
+        options: >-
+          --health-cmd "pg_isready -U devfeed -d devfeed_token_profile_test"
+          --health-interval 5s --health-timeout 5s --health-retries 12
+    env:
+      DEVFEED_DATABASE_URL: postgresql+psycopg://devfeed@127.0.0.1:5432/devfeed_token_profile_test
+      DEVFEED_TEST_DATABASE_URL: postgresql+psycopg://devfeed@127.0.0.1:5432/devfeed_token_profile_test
+      DEVFEED_REDIS_URL: redis://127.0.0.1:6379/15
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          version: "0.13.5"
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+          cache-suffix: token-usage-profile
+      - name: Install locked application
+        run: sh scripts/ci/python-bootstrap.sh . --editable --all-packages
+      - name: Migrate disposable database
+        run: uv run --locked --no-sync --no-build devfeed db upgrade
+      - name: Compare production-sized and larger histories
+        id: measure
+        env:
+          POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
+        run: |
+          test "$(uname -m)" = aarch64
+          uv run --locked --no-sync --no-build python scripts/profile_token_usage.py --seed --articles 100000 300000 \
+            --postgres-container "$POSTGRES_CONTAINER"
+      - name: Summarize before and after measurements
+        if: always()
+        env:
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          MEASUREMENT_OUTCOME: ${{ steps.measure.outcome }}
+        run: |
+          set +e
+          python3 scripts/ci/token-usage-summary.py \
+            --head-sha "$HEAD_SHA" --run-url "$RUN_URL" --outcome "$MEASUREMENT_OUTCOME"
+          summary_status=$?
+          set -e
+          if [ -f reports/token-usage/summary.md ]; then
+            cat reports/token-usage/summary.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+          exit "$summary_status"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: token-usage-native-arm64-${{ github.run_attempt }}
+          path: reports/token-usage/
+          if-no-files-found: warn
+          retention-days: 30
+  report:
+    name: Publish token usage results
+    needs: benchmark
+    if: >-
+      always() && !cancelled() && github.event_name == 'pull_request' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    uses: ./.github/workflows/performance-report.yml
+    permissions:
+      contents: read
+      pull-requests: write
+    with:
+      section: token-usage
+      artifact-name: token-usage-native-arm64-${{ github.run_attempt }}
+      measurement-result: ${{ needs.benchmark.result }}