feat: add Slack notification workflow for coverage improvements (#10977)

## Summary Adds a GitHub Actions workflow + TypeScript script that posts to Slack when a merged PR improves unit or E2E test coverage. ## Changes - **What**: New `coverage-slack-notify.yaml` workflow triggered on push to main. Compares current coverage against previous baselines, generates Slack Block Kit payload with progress bars and milestone celebrations, posts to `#p-frontend-automated-testing`. - **Script**: `scripts/coverage-slack-notify.ts` — parses lcov files, computes deltas, detects milestone crossings (every 5%), builds Slack payload. Pure functions exported for testability. - **Tests**: 26 unit tests in `scripts/coverage-slack-notify.test.ts` covering all pure functions including edge cases (malformed lcov, exact boundaries, zero coverage). ### Security hardening - All `${{ }}` expressions moved from `run:` blocks to `env:` variables - `SLACK_BOT_TOKEN` passed via env var, not inline - Unique heredoc delimiter (timestamp-based) prevents payload injection - `parseInt` fallback (`|| 0`) guards against malformed lcov - PR regex anchored to first line of commit message ### Robustness - `continue-on-error: true` on Slack post step (outage does not fail the job) - Baseline save guarded by `steps.unit-tests.outcome == success` (prevents corrupt baselines on test failure) - Channel ID commented for maintainability - Top-level `text` field added for Slack mobile push notifications - Author linked to GitHub profile instead of bare `@username` ## Review Focus - Workflow step ordering and conditional logic - Security of expression handling and secret management - Slack payload structure and Block Kit formatting ┆Issue is synchronized with this [Notion page](https://www.notion.so/PR-10977-feat-add-Slack-notification-workflow-for-coverage-improvements-33d6d73d3650819c8950f483c83f297c) by [Unito](https://www.unito.io) --------- Co-authored-by: GitHub Action <action@github.com>
2026-05-26 07:57:36 +00:00 · 2026-04-14 20:58:47 -07:00
parent 66e8d570e7
commit 033b3dad3a
15 changed files with 863 additions and 117 deletions
--- a/.github/workflows/ci-tests-e2e-coverage.yaml
+++ b/.github/workflows/ci-tests-e2e-coverage.yaml
@@ -0,0 +1,73 @@
+name: 'CI: E2E Coverage'
+
+on:
+  workflow_run:
+    workflows: ['CI: Tests E2E']
+    types:
+      - completed
+
+concurrency:
+  group: e2e-coverage-${{ github.event.workflow_run.head_sha }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  merge:
+    if: >
+      github.repository == 'Comfy-Org/ComfyUI_frontend' &&
+      github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup frontend
+        uses: ./.github/actions/setup-frontend
+
+      - name: Download all shard coverage data
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          name: e2e-coverage-shard-.*
+          name_is_regexp: true
+          path: temp/coverage-shards
+          if_no_artifact_found: warn
+
+      - name: Install lcov
+        run: sudo apt-get install -y -qq lcov
+
+      - name: Merge shard coverage into single LCOV
+        run: |
+          mkdir -p coverage/playwright
+          LCOV_FILES=$(find temp/coverage-shards -name 'coverage.lcov' -type f)
+          if [ -z "$LCOV_FILES" ]; then
+            echo "No coverage.lcov files found"
+            touch coverage/playwright/coverage.lcov
+            exit 0
+          fi
+          ADD_ARGS=""
+          for f in $LCOV_FILES; do ADD_ARGS="$ADD_ARGS -a $f"; done
+          lcov $ADD_ARGS -o coverage/playwright/coverage.lcov
+          wc -l coverage/playwright/coverage.lcov
+
+      - name: Upload merged coverage data
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: e2e-coverage
+          path: coverage/playwright/
+          retention-days: 30
+          if-no-files-found: warn
+
+      - name: Upload E2E coverage to Codecov
+        if: always()
+        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
+        with:
+          files: coverage/playwright/coverage.lcov
+          flags: e2e
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false
--- a/.github/workflows/ci-tests-e2e.yaml
+++ b/.github/workflows/ci-tests-e2e.yaml
@@ -87,6 +87,7 @@ jobs:
        run: pnpm exec playwright test --project=chromium --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
        env:
          PLAYWRIGHT_BLOB_OUTPUT_DIR: ./blob-report
+          COLLECT_COVERAGE: 'true'

      - name: Upload blob report
        uses: actions/upload-artifact@v6
@@ -96,6 +97,15 @@ jobs:
          path: blob-report/
          retention-days: 1

+      - name: Upload shard coverage data
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: e2e-coverage-shard-${{ matrix.shardIndex }}
+          path: coverage/playwright/
+          retention-days: 1
+          if-no-files-found: warn
+
  playwright-tests:
    # Ideally, each shard runs test in 6 minutes, but allow up to 15 minutes
    timeout-minutes: 15
--- a/.github/workflows/ci-tests-unit.yaml
+++ b/.github/workflows/ci-tests-unit.yaml
@@ -27,10 +27,20 @@ jobs:
      - name: Run Vitest tests with coverage
        run: pnpm test:coverage

+      - name: Upload unit coverage artifact
+        if: always() && github.event_name == 'push'
+        uses: actions/upload-artifact@v6
+        with:
+          name: unit-coverage
+          path: coverage/lcov.info
+          retention-days: 30
+          if-no-files-found: warn
+
      - name: Upload coverage to Codecov
        if: always()
        uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
        with:
          files: coverage/lcov.info
+          flags: unit
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false
--- a/.github/workflows/coverage-slack-notify.yaml
+++ b/.github/workflows/coverage-slack-notify.yaml
@@ -0,0 +1,149 @@
+name: 'Coverage: Slack Notification'
+
+on:
+  workflow_run:
+    workflows: ['CI: Tests Unit']
+    branches: [main]
+    types:
+      - completed
+
+permissions:
+  contents: read
+  actions: read
+  pull-requests: read
+
+jobs:
+  notify:
+    if: >
+      github.repository == 'Comfy-Org/ComfyUI_frontend' &&
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Setup frontend
+        uses: ./.github/actions/setup-frontend
+
+      - name: Download current unit coverage
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          name: unit-coverage
+          path: coverage
+
+      - name: Download previous unit coverage baseline
+        continue-on-error: true
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          branch: main
+          workflow: coverage-slack-notify.yaml
+          name: unit-coverage-baseline
+          path: temp/coverage-baseline
+          if_no_artifact_found: warn
+
+      - name: Download latest E2E coverage
+        continue-on-error: true
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          branch: main
+          workflow: ci-tests-e2e-coverage.yaml
+          name: e2e-coverage
+          path: temp/e2e-coverage
+          if_no_artifact_found: warn
+
+      - name: Download previous E2E coverage baseline
+        continue-on-error: true
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          branch: main
+          workflow: coverage-slack-notify.yaml
+          name: e2e-coverage-baseline
+          path: temp/e2e-coverage-baseline
+          if_no_artifact_found: warn
+
+      - name: Resolve merged PR metadata
+        id: pr-meta
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const sha = context.payload.workflow_run.head_sha;
+            const { data: commit } = await github.rest.repos.getCommit({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: sha,
+            });
+            const message = commit.commit.message ?? '';
+            const firstLine = message.split('\n')[0];
+            const match = firstLine.match(/\(#(\d+)\)\s*$/);
+            if (!match) {
+              core.setOutput('skip', 'true');
+              core.info('No PR number found in commit message — skipping.');
+              return;
+            }
+            const prNumber = match[1];
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: Number(prNumber),
+            });
+            core.setOutput('skip', 'false');
+            core.setOutput('number', prNumber);
+            core.setOutput('url', pr.html_url);
+            core.setOutput('author', pr.user.login);
+
+      - name: Generate Slack notification
+        if: steps.pr-meta.outputs.skip != 'true'
+        id: slack-payload
+        env:
+          PR_URL: ${{ steps.pr-meta.outputs.url }}
+          PR_NUMBER: ${{ steps.pr-meta.outputs.number }}
+          PR_AUTHOR: ${{ steps.pr-meta.outputs.author }}
+        run: |
+          PAYLOAD=$(pnpm exec tsx scripts/coverage-slack-notify.ts \
+            --pr-url="$PR_URL" \
+            --pr-number="$PR_NUMBER" \
+            --author="$PR_AUTHOR")
+          if [ -n "$PAYLOAD" ]; then
+            echo "has_payload=true" >> "$GITHUB_OUTPUT"
+            DELIM="SLACK_PAYLOAD_$(date +%s)"
+            echo "payload<<$DELIM" >> "$GITHUB_OUTPUT"
+            printf '%s\n' "$PAYLOAD" >> "$GITHUB_OUTPUT"
+            echo "$DELIM" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_payload=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Post to Slack
+        if: steps.slack-payload.outputs.has_payload == 'true'
+        continue-on-error: true
+        env:
+          SLACK_PAYLOAD: ${{ steps.slack-payload.outputs.payload }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        run: |
+          # Channel: #p-deprecated-frontend-automated-testing
+          BODY=$(echo "$SLACK_PAYLOAD" | jq --arg ch "C0AP09LKRDZ" '. + {channel: $ch}')
+          curl -sf -X POST \
+            -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "$BODY" \
+            -o /dev/null \
+            https://slack.com/api/chat.postMessage
+
+      - name: Save unit coverage baseline
+        if: always() && hashFiles('coverage/lcov.info') != ''
+        uses: actions/upload-artifact@v6
+        with:
+          name: unit-coverage-baseline
+          path: coverage/lcov.info
+          retention-days: 90
+          if-no-files-found: warn
+
+      - name: Save E2E coverage baseline
+        if: always() && hashFiles('temp/e2e-coverage/coverage.lcov') != ''
+        uses: actions/upload-artifact@v6
+        with:
+          name: e2e-coverage-baseline
+          path: temp/e2e-coverage/coverage.lcov
+          retention-days: 90
+          if-no-files-found: warn
--- a/.github/workflows/pr-report.yaml
+++ b/.github/workflows/pr-report.yaml
@@ -2,7 +2,7 @@ name: 'PR: Unified Report'

 on:
  workflow_run:
-    workflows: ['CI: Size Data', 'CI: Performance Report']
+    workflows: ['CI: Size Data', 'CI: Performance Report', 'CI: E2E Coverage']
    types:
      - completed

@@ -67,73 +67,23 @@ jobs:
            core.setOutput('base', livePr.base.ref);
            core.setOutput('head-sha', livePr.head.sha);

-      - name: Find size workflow run for this commit
+      - name: Find size workflow run
        if: steps.pr-meta.outputs.skip != 'true'
        id: find-size
-        uses: actions/github-script@v8
+        uses: ./.github/actions/find-workflow-run
        with:
-          script: |
-            const headSha = '${{ steps.pr-meta.outputs.head-sha }}';
-            const { data: runs } = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'ci-size-data.yaml',
-              head_sha: headSha,
-              per_page: 1,
-            });
+          workflow-id: ci-size-data.yaml
+          head-sha: ${{ steps.pr-meta.outputs.head-sha }}
+          token: ${{ secrets.GITHUB_TOKEN }}

-            const run = runs.workflow_runs[0];
-            if (!run) {
-              core.setOutput('status', 'pending');
-              return;
-            }
-
-            if (run.status !== 'completed') {
-              core.setOutput('status', 'pending');
-              return;
-            }
-
-            if (run.conclusion !== 'success') {
-              core.setOutput('status', 'failed');
-              return;
-            }
-
-            core.setOutput('status', 'ready');
-            core.setOutput('run-id', String(run.id));
-
-      - name: Find perf workflow run for this commit
+      - name: Find perf workflow run
        if: steps.pr-meta.outputs.skip != 'true'
        id: find-perf
-        uses: actions/github-script@v8
+        uses: ./.github/actions/find-workflow-run
        with:
-          script: |
-            const headSha = '${{ steps.pr-meta.outputs.head-sha }}';
-            const { data: runs } = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'ci-perf-report.yaml',
-              head_sha: headSha,
-              per_page: 1,
-            });
-
-            const run = runs.workflow_runs[0];
-            if (!run) {
-              core.setOutput('status', 'pending');
-              return;
-            }
-
-            if (run.status !== 'completed') {
-              core.setOutput('status', 'pending');
-              return;
-            }
-
-            if (run.conclusion !== 'success') {
-              core.setOutput('status', 'failed');
-              return;
-            }
-
-            core.setOutput('status', 'ready');
-            core.setOutput('run-id', String(run.id));
+          workflow-id: ci-perf-report.yaml
+          head-sha: ${{ steps.pr-meta.outputs.head-sha }}
+          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Download size data (current)
        if: steps.pr-meta.outputs.skip != 'true' && steps.find-size.outputs.status == 'ready'
@@ -154,6 +104,25 @@ jobs:
          path: temp/size-prev
          if_no_artifact_found: warn

+      - name: Find coverage workflow run
+        if: steps.pr-meta.outputs.skip != 'true'
+        id: find-coverage
+        uses: ./.github/actions/find-workflow-run
+        with:
+          workflow-id: ci-tests-e2e-coverage.yaml
+          head-sha: ${{ steps.pr-meta.outputs.head-sha }}
+          not-found-status: skip
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Download coverage data
+        if: steps.pr-meta.outputs.skip != 'true' && steps.find-coverage.outputs.status == 'ready'
+        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
+        with:
+          name: e2e-coverage
+          run_id: ${{ steps.find-coverage.outputs.run-id }}
+          path: temp/coverage
+          if_no_artifact_found: warn
+
      - name: Download perf metrics (current)
        if: steps.pr-meta.outputs.skip != 'true' && steps.find-perf.outputs.status == 'ready'
        uses: dawidd6/action-download-artifact@0bd50d53a6d7fb5cb921e607957e9cc12b4ce392 # v12
@@ -189,9 +158,10 @@ jobs:
      - name: Generate unified report
        if: steps.pr-meta.outputs.skip != 'true'
        run: >
-          node scripts/unified-report.js
+          pnpm exec tsx scripts/unified-report.ts
          --size-status=${{ steps.find-size.outputs.status }}
          --perf-status=${{ steps.find-perf.outputs.status }}
+          --coverage-status=${{ steps.find-coverage.outputs.status }}
          > pr-report.md

      - name: Remove legacy separate comments