detection-proof: 90 min job budget - all-surfaces-broken run does ~7x a green run's work

.github/workflows/ci-tests-custom-nodes.yaml

@@ -54,7 +54,10 @@ jobs:
       (github.event_name != 'pull_request' ||
       github.event.pull_request.head.repo.full_name == github.repository)
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    # DETECTION PROOF (demo branch only): a green suite fits in 30, but with
+    # every surface deliberately broken the suite walks hundreds of failure
+    # paths (single re-runs, drains), so it needs far more headroom.
+    timeout-minutes: 90
     permissions:
       contents: read
     steps:
@@ -122,6 +125,38 @@ jobs:
             echo "::endgroup::"
           done
 
+      # DETECTION PROOF (rows 8-10): the pack-shipped bugs the suite is meant to
+      # catch live inside third-party pack repos, which CI clones fresh at their
+      # pins - a normal frontend commit cannot reach that code. This step, on the
+      # never-merge nathaniel/detection-proof branch ONLY, pokes one verified
+      # break into each cloned pack right after install, so the Pack-mode rows go
+      # red on real pack failures. Each break asserts it landed (grep) so a silent
+      # no-op cannot fake a pass. Do NOT port this step to any real suite branch.
+      - name: DETECTION PROOF - break packs (rows 8-10)
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Row 8 (console/pageerror ledger, s10): Custom-Scripts showText.js
+          # logs a console.error on node configure. Expected: curated run (T1)
+          # red with `console errors during curated run` + the text + script URL.
+          f=ComfyUI/custom_nodes/ComfyUI-Custom-Scripts/web/js/showText.js
+          perl -0pi -e "s/(onConfigure\?\.apply\(this, arguments\);)/\$1\n\t\t\t\tconsole.error('DETECTION PROOF (row 8): pack showText.js onConfigure failure');/" "$f"
+          grep -q "DETECTION PROOF (row 8)" "$f" || { echo "::error::row 8 break did not apply"; exit 1; }
+          # Row 9 (execution runtime, s7): WAS `return_constant_number` raises
+          # on entry. Expected: auto-run tier red `Constant Number:
+          # EXECUTION_ERROR ... not in cannotRunAlone; a regression`.
+          f=ComfyUI/custom_nodes/was-node-suite-comfyui/WAS_Node_Suite.py
+          perl -pi -e 's/(def return_constant_number\(self, number_type, number, number_as_text=None\):)/$1\n        raise ValueError("DETECTION PROOF (row 9): pack node runtime failure")/' "$f"
+          grep -q "DETECTION PROOF (row 9)" "$f" || { echo "::error::row 9 break did not apply"; exit 1; }
+          # Row 10 (registration sentinels, s5/s10): Impact renames the
+          # `ImpactInt` node-class key, so the node no longer registers under
+          # its expected class_type. Expected: the mount + curated tests that
+          # reference ImpactInt skip, and the "Forbid skipped tests" gate fails
+          # the job on `skipped != 0`.
+          f=ComfyUI/custom_nodes/ComfyUI-Impact-Pack/__init__.py
+          perl -pi -e 's/"ImpactInt": ImpactInt,/"ImpactIntDETECTIONPROOF": ImpactInt,/' "$f"
+          grep -q "ImpactIntDETECTIONPROOF" "$f" || { echo "::error::row 10 break did not apply"; exit 1; }
+
       # The VHS run-tier workflow reads input/plain_video.mp4.
       - name: Stage run-tier assets
         shell: bash

browser_tests/tests/customNodes/DETECTION_PROOF.md

@@ -44,14 +44,16 @@ The gate protects against two distinct things, and the proof covers both:
   is the primary thing the gate guards on every frontend PR. These breaks live
   in `src/`.
 - **Pack-bug** - a pack itself ships a bug (or a pinned pack is bumped to a
-  broken version). The gate catches these too. On CI these breaks are
-  delivered the only way a frontend-repo commit can deliver them: through the
-  manifest (`browser_tests/fixtures/data/customNodeManifest.json`), by
-  pointing the pack's `repo`/`pin` at a fork commit that carries the bug -
-  which is exactly the pinned-bump scenario the mode describes. CI clones
-  every pack fresh at its pin, so editing pack files in the frontend repo
-  does nothing there; direct pack-file edits only work against a local
-  backend (which is how the exact reds below were captured).
+  broken version). The gate catches these too. CI clones every pack fresh at
+  its pin, so editing pack files in the frontend repo does nothing - the clone
+  overwrites them. Two ways deliver a pack break on CI: (a) point the manifest
+  (`browser_tests/fixtures/data/customNodeManifest.json`) `repo`/`pin` at a
+  broken fork, which is exactly the pinned-bump scenario and the most
+  production-faithful; or (b) a self-contained CI step that patches each cloned
+  pack in place right after install. The proof PR uses (b) - no external repos,
+  and each patch asserts it landed (`grep`, fails the job otherwise) so a silent
+  no-op cannot fake a pass. Both reproduce the same edits captured against a
+  local backend (which is how the exact reds below were captured).
 
 Each row below is labelled with its mode.
 
@@ -65,18 +67,18 @@ change WHICH pair or node the message names without weakening the catch - the
 promise is the tier and the failure class, not byte-identical offender text
 across pin changes. Sections refer to [ARCHITECTURE.md](ARCHITECTURE.md).
 
-| #   | Surface (ARCH section)                           | Mode | Real regression it recreates                                                                                                          | The one-file break                                                                                                                                                                     | CI check that catches it                   | Exact red                                                                                                   |
-| --- | ------------------------------------------------ | ---- | ------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
-| 1   | Mount completeness, canvas / v1 (s1, s5)         | FE   | A change dropping declared parts on the canvas renderer (class; no single ticket - the v2 wave below shows how this family presents)  | `src/services/litegraphService.ts` `addInputs`: stop materializing the last declared input                                                                                             | Tests Custom Nodes / mount tier            | `BatchCount+: instance is missing declared input "batch" (litegraph)`                                       |
-| 2   | Mount completeness, DOM / v2 (s1, s5)            | FE   | Widgets missing under Nodes 2.0 (FE-627/FE-634 iTools buttons; FE-841 is the adjacent wrong-style class, present but unproven caught) | `src/renderer/extensions/vueNodes/widgets/registry/widgetRegistry.ts`: drop the `int` widget component mapping                                                                         | Tests Custom Nodes / mount tier (Vue pass) | `Ideogram4PromptBuilderKJ: Vue mounts 9 of 15 widgets`                                                      |
-| 3   | Persistence, save/reload (s1, s8)                | FE   | Widgets reverting to socket-only on reload: the defaultInput migration regression that PR #12279 (open) exists to fix                 | `src/lib/litegraph/src/LGraphNode.ts` `configure`: off-by-one drops the last `widgets_values` entry                                                                                    | Tests Custom Nodes / persistence tier      | `Seed (rgthree): widgets_values [1,"fixed"] -> [1,"randomize"] on set-values reload`                        |
-| 4   | Wiring - type compatibility (s5, s6)             | FE   | A frontend change narrowing connectable types (class; no single verified ticket)                                                      | `src/lib/litegraph/src/LiteGraphGlobal.ts` `isValidConnection`: reject IMAGE links                                                                                                     | Tests Custom Nodes / connectivity sweep    | `AddLabel.IMAGE -> FastPreviewBatch.input: CONNECT_REJECTED` (full pair list)                               |
-| 5   | Wiring - drop resolution (s5)                    | FE   | Drag/slot resolution family (nearest reported symptoms: FE-625/FE-632 EditUtils connections shift after drag)                         | `src/lib/litegraph/src/canvas/measureSlots.ts` `getNodeInputOnPos`: return undefined                                                                                                   | Tests Custom Nodes / connectivity drag     | `EmptyImage.IMAGE -> ImageBatch.image2 with VueNodes=false`                                                 |
-| 6   | Execution - frontend prompt serialization (s7)   | FE   | A prompt-serialization change corrupting inputs (class; no single verified ticket)                                                    | `src/utils/executionUtil.ts`: drop numeric widget values from the API prompt                                                                                                           | Tests Custom Nodes / curated run (T1)      | `Prompt outputs failed validation; ImpactInt: value; ImpactFloat: value`                                    |
-| 7   | Zero-visible-errors / load hook (s1)             | FE   | An extension hook crashing on graph load, the mechanism packs hook (FE-751 class; the break is in a core extension, hence FE mode)    | `src/composables/node/useNodeBadge.ts` `afterConfigureGraph`: throw                                                                                                                    | Tests Custom Nodes / curated run (T1)      | `Error calling extension 'Comfy.NodeBadge' method 'afterConfigureGraph' ...`                                |
-| 8   | Console / pageerror ledger (s10)                 | Pack | An uncaught pack-JS error during save/reload (the betterCombos.js `typeof null` bug this suite found)                                 | manifest pin swap: point ComfyUI-Custom-Scripts at a fork commit whose `showText.js` logs a `console.error` in `onConfigure` (captured locally by editing the installed pack directly) | Tests Custom Nodes / curated run (T1)      | `console errors during curated run` + the exact text + script URL                                           |
-| 9   | Execution - runtime (s7)                         | Pack | A pack node raising at execution (WAS Text Find/Replace infinite loop; KJ ImageGridtoBatch min violation)                             | manifest pin swap: point was-node-suite at a fork commit whose `return_constant_number` raises (captured locally by editing the installed pack directly)                               | Tests Custom Nodes / auto-run tier         | `Constant Number: EXECUTION_ERROR (Constant Number: ValueError) - not in cannotRunAlone; a regression, ...` |
-| 10  | Registration / expectedNodes sentinels (s5, s10) | Pack | A pinned pack bump renaming a node key                                                                                                | manifest pin swap: point ComfyUI-Impact-Pack at a fork commit that renames the `ImpactInt` mapping key (captured locally by editing the installed pack directly)                       | Tests Custom Nodes / zero-skip gate        | job goes red on `skipped != 0` (T0 + T1 skip; the workflow's "Forbid skipped tests" step fails)             |
+| #   | Surface (ARCH section)                           | Mode | Real regression it recreates                                                                                                          | The one-file break                                                                                                                                                  | CI check that catches it                   | Exact red                                                                                                   |
+| --- | ------------------------------------------------ | ---- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
+| 1   | Mount completeness, canvas / v1 (s1, s5)         | FE   | A change dropping declared parts on the canvas renderer (class; no single ticket - the v2 wave below shows how this family presents)  | `src/services/litegraphService.ts` `addInputs`: stop materializing the last declared input                                                                          | Tests Custom Nodes / mount tier            | `BatchCount+: instance is missing declared input "batch" (litegraph)`                                       |
+| 2   | Mount completeness, DOM / v2 (s1, s5)            | FE   | Widgets missing under Nodes 2.0 (FE-627/FE-634 iTools buttons; FE-841 is the adjacent wrong-style class, present but unproven caught) | `src/renderer/extensions/vueNodes/widgets/registry/widgetRegistry.ts`: drop the `int` widget component mapping                                                      | Tests Custom Nodes / mount tier (Vue pass) | `Ideogram4PromptBuilderKJ: Vue mounts 9 of 15 widgets`                                                      |
+| 3   | Persistence, save/reload (s1, s8)                | FE   | Widgets reverting to socket-only on reload: the defaultInput migration regression that PR #12279 (open) exists to fix                 | `src/lib/litegraph/src/LGraphNode.ts` `configure`: off-by-one drops the last `widgets_values` entry                                                                 | Tests Custom Nodes / persistence tier      | `Seed (rgthree): widgets_values [1,"fixed"] -> [1,"randomize"] on set-values reload`                        |
+| 4   | Wiring - type compatibility (s5, s6)             | FE   | A frontend change narrowing connectable types (class; no single verified ticket)                                                      | `src/lib/litegraph/src/LiteGraphGlobal.ts` `isValidConnection`: reject IMAGE links                                                                                  | Tests Custom Nodes / connectivity sweep    | `AddLabel.IMAGE -> FastPreviewBatch.input: CONNECT_REJECTED` (full pair list)                               |
+| 5   | Wiring - drop resolution (s5)                    | FE   | Drag/slot resolution family (nearest reported symptoms: FE-625/FE-632 EditUtils connections shift after drag)                         | `src/lib/litegraph/src/canvas/measureSlots.ts` `getNodeInputOnPos`: return undefined                                                                                | Tests Custom Nodes / connectivity drag     | `EmptyImage.IMAGE -> ImageBatch.image2 with VueNodes=false`                                                 |
+| 6   | Execution - frontend prompt serialization (s7)   | FE   | A prompt-serialization change corrupting inputs (class; no single verified ticket)                                                    | `src/utils/executionUtil.ts`: drop numeric widget values from the API prompt                                                                                        | Tests Custom Nodes / curated run (T1)      | `Prompt outputs failed validation; ImpactInt: value; ImpactFloat: value`                                    |
+| 7   | Zero-visible-errors / load hook (s1)             | FE   | An extension hook crashing on graph load, the mechanism packs hook (FE-751 class; the break is in a core extension, hence FE mode)    | `src/composables/node/useNodeBadge.ts` `afterConfigureGraph`: throw                                                                                                 | Tests Custom Nodes / curated run (T1)      | `Error calling extension 'Comfy.NodeBadge' method 'afterConfigureGraph' ...`                                |
+| 8   | Console / pageerror ledger (s10)                 | Pack | An uncaught pack-JS error during save/reload (the betterCombos.js `typeof null` bug this suite found)                                 | CI step patches the cloned ComfyUI-Custom-Scripts `showText.js` to log a `console.error` in `onConfigure` (captured locally by editing the installed pack directly) | Tests Custom Nodes / curated run (T1)      | `console errors during curated run` + the exact text + script URL                                           |
+| 9   | Execution - runtime (s7)                         | Pack | A pack node raising at execution (WAS Text Find/Replace infinite loop; KJ ImageGridtoBatch min violation)                             | CI step patches the cloned was-node-suite `return_constant_number` to raise on entry (captured locally by editing the installed pack directly)                      | Tests Custom Nodes / auto-run tier         | `Constant Number: EXECUTION_ERROR (Constant Number: ValueError) - not in cannotRunAlone; a regression, ...` |
+| 10  | Registration / expectedNodes sentinels (s5, s10) | Pack | A pinned pack bump renaming a node key                                                                                                | CI step patches the cloned ComfyUI-Impact-Pack `__init__.py` to rename the `ImpactInt` mapping key (captured locally by editing the installed pack directly)        | Tests Custom Nodes / zero-skip gate        | job goes red on `skipped != 0` (T0 + T1 skip; the workflow's "Forbid skipped tests" step fails)             |
 
 ### Links of various types (surface 4/5 expanded)
 
@@ -155,23 +157,28 @@ against an environment that changed under it, not a suite defect. Therefore:
 ## Building the proof PR
 
 1. Branch off the suite branch: `git checkout -b nathaniel/detection-proof nathaniel/custom-node-e2e-suite`.
-2. For the three Pack-mode rows, first publish the broken fork commits (one
-   fork per pack, each a one-line change matching its row); the demo commit in
-   this repo is then the manifest `repo`/`pin` edit pointing at that fork. The
-   FE-mode rows need no forks - their break is a direct `src/` edit.
-3. One commit per matrix row, each a single-file break. FE-mode rows carry an
-   inline comment in the changed `src/` file:
-   `// DETECTION PROOF: recreates <FE-xxx / PR #12279> - <surface>. Expected: custom-nodes check red at <tier>.`
-   Pack-mode rows change only the JSON manifest, which cannot carry a comment
-   (a `//` line would break both the CI jq install loop and the loader's
-   JSON.parse), so their provenance lives in the commit message per step 4.
-4. Commit message names the surface and the reference, e.g.
-   `demo(detection): break Vue widget mount (recreates FE-627/FE-634) - expect mount tier red`.
+2. One commit per matrix row, each breaking one surface, stacked so all breaks
+   are live at HEAD at once (not reverted between commits - the goal is to see
+   every surface broken together, and the `Tests Custom Nodes` job reds across
+   every tier in one run). FE-mode rows (1-7) are a direct `src/` edit carrying
+   an inline comment in the changed file:
+   `// DETECTION PROOF (row N, surface): recreates <FE-xxx / PR #12279>. Expected: <tier> red <message>.`
+3. Pack-mode rows (8-10) are delivered by one CI step
+   (`DETECTION PROOF - break packs`, on this branch only) that patches each
+   cloned pack in place right after install. Each patch asserts it landed
+   (`grep`, fails the job otherwise) so a silent no-op cannot fake a pass. The
+   step is fenced to this never-merge branch and must never be ported to a real
+   suite branch.
+4. Commit message names the surface, e.g.
+   `detection-proof: break mount (v2 Vue renderer) - drops the int widget mapping`.
 5. Open the PR against the suite branch (not main) with the correlation matrix as
    the description and a bold header: **This PR must never merge. Every commit is
    a deliberate break; green would mean the gate missed a regression.**
-6. Let CI run. Each commit turns its named check red at the named tier. The
-   reviewer bisects commit-by-commit to correlate break -> catch.
+6. Let CI run on HEAD. With every break live, the `Tests Custom Nodes` job reds
+   across every tier in one run. Attribute a red to its cause via the labelled
+   comment on the matching `src/` file (rows 1-7) or in the CI break step
+   (rows 8-10); checking out commit N (which contains breaks 1..N) narrows it
+   further.
 
 ## References
 

src/composables/node/useNodeBadge.ts

@@ -216,8 +216,9 @@ export const useNodeBadge = () => {
         )
       },
       afterConfigureGraph() {
-        for (const node of app.canvas.graph?.nodes ?? [])
-          priceBadge.updateSubgraphCredits(node)
+        // DETECTION PROOF (row 7, load hook): extension hook throws on load.
+        // Expected: curated run (T1) red "Error calling extension 'Comfy.NodeBadge'".
+        throw new Error('DETECTION PROOF: afterConfigureGraph crash')
       }
     })
   })

src/lib/litegraph/src/LGraphNode.ts

@@ -929,7 +929,9 @@ export class LGraphNode
         let i = 0
         for (const widget of this.widgets ?? []) {
           if (widget.serialize === false) continue
-          if (i >= info.widgets_values.length) break
+          // DETECTION PROOF (row 3, persistence): off-by-one drops the last
+          // widgets_values. Expected: persistence tier red '... on set-values reload'.
+          if (i >= info.widgets_values.length - 1) break
           widget.value = info.widgets_values[i++]
         }
       }

src/lib/litegraph/src/LiteGraphGlobal.ts

@@ -696,6 +696,9 @@ export class LiteGraphGlobal {
     type_b = String(type_b)
     type_a = type_a.toLowerCase()
     type_b = type_b.toLowerCase()
+    // DETECTION PROOF (row 4, links/type): reject IMAGE connections.
+    // Expected: connectivity sweep red with a CONNECT_REJECTED pair list.
+    if (type_a === 'image' || type_b === 'image') return false
 
     // For nodes supporting multiple connection types
     if (!type_a.includes(',') && !type_b.includes(',')) return type_a == type_b

src/lib/litegraph/src/canvas/measureSlots.ts

@@ -13,6 +13,9 @@ export function getNodeInputOnPos(
 ): { index: number; input: INodeInputSlot; pos: Point } | undefined {
   const { inputs } = node
   if (!inputs) return
+  // DETECTION PROOF (row 5, links/drop): slot hit-test never resolves.
+  // Expected: connectivity drag red 'EmptyImage.IMAGE -> ImageBatch.image2 ...'.
+  return undefined
 
   for (const [index, input] of inputs.entries()) {
     const pos = node.getInputPos(index)

src/renderer/extensions/vueNodes/widgets/registry/widgetRegistry.ts

@@ -106,7 +106,9 @@ const coreWidgetDefinitions: Array<[string, WidgetDefinition]> = [
       essential: false
     }
   ],
-  ['int', { component: WidgetInputNumber, aliases: ['INT'], essential: true }],
+  // DETECTION PROOF (row 2, mount v2): drop the int Vue widget mapping.
+  // Expected: mount Vue pass red '... Vue mounts N of M widgets'.
+  // ['int', { component: WidgetInputNumber, aliases: ['INT'], essential: true }],
   [
     'float',
     {

src/services/litegraphService.ts

@@ -336,8 +336,11 @@ export const useLitegraphService = () => {
     const orderedInputSpecs = getOrderedInputSpecs(nodeDefImpl, inputs)
 
     // Create sockets and widgets in the determined order
-    for (const inputSpec of orderedInputSpecs) addInputSocket(node, inputSpec)
-    for (const inputSpec of orderedInputSpecs) addInputWidget(node, inputSpec)
+    // DETECTION PROOF (row 1, mount v1): drop the last declared input.
+    // Expected: mount tier red 'is missing declared input "..." (litegraph)'.
+    const detectionProof = orderedInputSpecs.slice(0, -1)
+    for (const inputSpec of detectionProof) addInputSocket(node, inputSpec)
+    for (const inputSpec of detectionProof) addInputWidget(node, inputSpec)
   }
 
   /**

src/utils/executionUtil.ts

@@ -103,6 +103,9 @@ export const graphToPrompt = async (
         const widgetValue = widget.serializeValue
           ? await widget.serializeValue(node, i)
           : widget.value
+        // DETECTION PROOF (row 6, exec/serialize): drop numeric widget inputs.
+        // Expected: curated run (T1) red 'Prompt outputs failed validation; ...'.
+        if (typeof widgetValue === 'number') continue
         // By default, Array values are reserved to represent node connections.
         // We need to wrap the array as an object to avoid the misinterpretation
         // of the array as a node connection.