mirror of
https://github.com/Comfy-Org/ComfyUI_frontend.git
synced 2026-07-09 08:38:13 +00:00
Compare commits
77 Commits
nathaniel/
...
nathaniel/
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
454f6f1de5 | ||
|
|
b7af403cc0 | ||
|
|
2b64f13edb | ||
|
|
2c7ee46beb | ||
|
|
cc0cbb3e2e | ||
|
|
b5a8d1a147 | ||
|
|
91a979ecba | ||
|
|
aae02ff355 | ||
|
|
c711ebce29 | ||
|
|
1470d96f2a | ||
|
|
55b06a319a | ||
|
|
1152ac59a2 | ||
|
|
baa2e046b7 | ||
|
|
0cc21040c4 | ||
|
|
d65d853227 | ||
|
|
08106db082 | ||
|
|
10d7769ab5 | ||
|
|
349d82b63e | ||
|
|
6be7c18bef | ||
|
|
58e0fe7511 | ||
|
|
72ad08db63 | ||
|
|
24659c2caf | ||
|
|
e46487f3ed | ||
|
|
3bc40bb148 | ||
|
|
cbceaf5dd9 | ||
|
|
497dd6ede5 | ||
|
|
cec09da789 | ||
|
|
9eb035e8a8 | ||
|
|
3cef258bac | ||
|
|
34faaa1a1d | ||
|
|
95ced4c6ef | ||
|
|
f9a94d0296 | ||
|
|
e7681be896 | ||
|
|
70233bbd04 | ||
|
|
a22acc4f48 | ||
|
|
e6ed6120a1 | ||
|
|
5ce414653e | ||
|
|
c5de8d421d | ||
|
|
9bb0587ec5 | ||
|
|
de23856742 | ||
|
|
650abec3ab | ||
|
|
b99100d0b4 | ||
|
|
6d5bcb9e04 | ||
|
|
0723702791 | ||
|
|
9825047176 | ||
|
|
7adfaa9079 | ||
|
|
1e36107109 | ||
|
|
1d5514c90e | ||
|
|
61d1cbfdb0 | ||
|
|
14666b09c4 | ||
|
|
efb0365bc3 | ||
|
|
065bc0c336 | ||
|
|
1248c4628a | ||
|
|
ee83d67834 | ||
|
|
f63b7d866e | ||
|
|
068191ea47 | ||
|
|
07c4b230b2 | ||
|
|
9ed51f1e4b | ||
|
|
4a91fa4849 | ||
|
|
0991905a89 | ||
|
|
df6764762b | ||
|
|
2d2b318450 | ||
|
|
0f94da8746 | ||
|
|
d80427d014 | ||
|
|
d02e665290 | ||
|
|
dc83cc4df6 | ||
|
|
8b81a4f359 | ||
|
|
8f567e8ef0 | ||
|
|
4fb282f853 | ||
|
|
d17a387ddb | ||
|
|
68ba0aa613 | ||
|
|
675140c164 | ||
|
|
64706c53c3 | ||
|
|
bfa94d4118 | ||
|
|
b7708d5ad0 | ||
|
|
564de12d46 | ||
|
|
5a1f788230 |
210
.github/workflows/ci-tests-custom-nodes.yaml
vendored
Normal file
210
.github/workflows/ci-tests-custom-nodes.yaml
vendored
Normal file
@@ -0,0 +1,210 @@
|
||||
# Runs the custom-node regression suite against a backend that has the manifest
|
||||
# packs actually installed, so the load/run tiers execute for real. This is a
|
||||
# GATING check: if a pack fails to install or any tier is skipped, the job goes
|
||||
# red - a regression gate that let a broken pack through as a "skip" would be
|
||||
# pointless. Mark `custom-nodes-e2e` as a required status check in branch
|
||||
# protection to block merges on failure.
|
||||
name: 'CI: Tests Custom Nodes'
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches-ignore: [wip/*, draft/*, temp/*]
|
||||
push:
|
||||
branches: [main, master]
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Path gating lives here, not in a trigger-level `paths:` filter: a required
|
||||
# check gated by trigger paths never creates a check run on an unrelated PR
|
||||
# and leaves branch protection stuck Pending. A job-level `if:` still creates
|
||||
# the check and marks it Skipped (= passing). Mirrors ci-tests-unit.yaml.
|
||||
changes:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
should-run: ${{ steps.changes.outputs.should-run }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- id: changes
|
||||
uses: ./.github/actions/changes-filter
|
||||
|
||||
# Deliberately NOT sharded yet: the suite is ~8 min but every shard would
|
||||
# pay the full ~4.5 min setup (clone + pip-install every pack + boot the
|
||||
# backend), so 2 shards buy ~4 min of wall time for double the runner cost,
|
||||
# with diminishing returns beyond that. Sharding pays once test time dwarfs
|
||||
# setup time -
|
||||
# first cut setup with a prebuilt image of the pinned packs, then shard if
|
||||
# the job exceeds ~12 minutes.
|
||||
custom-nodes-e2e:
|
||||
needs: changes
|
||||
# Run only when non-docs code changed AND the PR is same-repo. Fork PRs can
|
||||
# edit the manifest's repo/pin URLs, and this job clones and pip-installs
|
||||
# whatever they point at (setup.py runs at install time), so an untrusted
|
||||
# fork must not be able to aim the clone at an attacker-controlled repo.
|
||||
# Fork PRs still get the environment-agnostic coverage via the main e2e
|
||||
# shards. A skipped job counts as passing, so this stays required-safe.
|
||||
if: >-
|
||||
needs.changes.outputs.should-run == 'true' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
runs-on: ubuntu-latest
|
||||
# DETECTION PROOF (demo branch only): a green suite fits in 30, but with
|
||||
# every surface deliberately broken the suite walks hundreds of failure
|
||||
# paths (single re-runs, drains), so it needs far more headroom.
|
||||
timeout-minutes: 90
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup frontend
|
||||
uses: ./.github/actions/setup-frontend
|
||||
with:
|
||||
include_build_step: true
|
||||
|
||||
- name: Setup Playwright
|
||||
uses: ./.github/actions/setup-playwright
|
||||
|
||||
# Checks out ComfyUI, installs Python/torch/requirements and ComfyUI_devtools.
|
||||
# launch_server:false so we can add the manifest packs before booting.
|
||||
- name: Setup ComfyUI server
|
||||
uses: ./.github/actions/setup-comfyui-server
|
||||
with:
|
||||
launch_server: 'false'
|
||||
|
||||
# Install every pack the manifest declares (DRY: a new pack row installs
|
||||
# itself here, no workflow change). A clone or dependency failure fails the
|
||||
# job - if a pack can't be installed, its coverage can't run, and that is a
|
||||
# gate failure, not something to paper over. The `jq | while` pipe hides
|
||||
# failures in a subshell, so read into an array and loop with `set -e`.
|
||||
- name: Install manifest custom nodes
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Pin the CPU torch stack that setup-comfyui-server installed so no
|
||||
# pack's requirements.txt can pull a GPU/incompatible torch onto this
|
||||
# --cpu runner. A pack that genuinely needs a different torch fails
|
||||
# the constrained install loudly rather than silently swapping it.
|
||||
pip freeze | grep -iE '^(torch|torchvision|torchaudio)==' \
|
||||
> /tmp/torch-constraints.txt || true
|
||||
manifest=browser_tests/fixtures/data/customNodeManifest.json
|
||||
mapfile -t entries < <(jq -c '.[]' "$manifest")
|
||||
for entry in "${entries[@]}"; do
|
||||
repo=$(jq -r '.repo' <<<"$entry")
|
||||
pin=$(jq -r '.pin' <<<"$entry")
|
||||
# Install under the manifest `pack` key, not basename(repo): node
|
||||
# attribution keys on the install dirname via python_module, and
|
||||
# the two only coincide by luck. Same charset the manifest loader
|
||||
# enforces - belt for anything that bypasses it.
|
||||
pack=$(jq -r '.pack' <<<"$entry")
|
||||
if ! [[ "$pack" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
|
||||
echo "::error::unsafe pack name: '$pack'"; exit 1
|
||||
fi
|
||||
# The gate tests exactly what was verified: a full SHA pin is
|
||||
# mandatory here, before anything installs. The planned canary
|
||||
# (pack HEADs) is the only intended unpinned consumer and runs
|
||||
# with CUSTOM_NODES_ALLOW_UNPINNED=1 through the loader instead.
|
||||
if ! [[ "$pin" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
echo "::error::$pack: pin must be a full commit SHA (got '$pin')"; exit 1
|
||||
fi
|
||||
dir="ComfyUI/custom_nodes/$pack"
|
||||
echo "::group::install $pack"
|
||||
git clone --depth 1 "$repo" "$dir"
|
||||
git -C "$dir" fetch --depth 1 origin "$pin"
|
||||
git -C "$dir" checkout "$pin"
|
||||
if [ -f "$dir/requirements.txt" ]; then
|
||||
pip install -r "$dir/requirements.txt" -c /tmp/torch-constraints.txt
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
|
||||
# DETECTION PROOF (rows 8-10): the pack-shipped bugs the suite is meant to
|
||||
# catch live inside third-party pack repos, which CI clones fresh at their
|
||||
# pins - a normal frontend commit cannot reach that code. This step, on the
|
||||
# never-merge nathaniel/detection-proof branch ONLY, pokes one verified
|
||||
# break into each cloned pack right after install, so the Pack-mode rows go
|
||||
# red on real pack failures. Each break asserts it landed (grep) so a silent
|
||||
# no-op cannot fake a pass. Do NOT port this step to any real suite branch.
|
||||
- name: DETECTION PROOF - break packs (rows 8-10)
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Row 8 (console/pageerror ledger, s10): Custom-Scripts showText.js
|
||||
# logs a console.error on node configure. Expected: curated run (T1)
|
||||
# red with `console errors during curated run` + the text + script URL.
|
||||
f=ComfyUI/custom_nodes/ComfyUI-Custom-Scripts/web/js/showText.js
|
||||
perl -0pi -e "s/(onConfigure\?\.apply\(this, arguments\);)/\$1\n\t\t\t\tconsole.error('DETECTION PROOF (row 8): pack showText.js onConfigure failure');/" "$f"
|
||||
grep -q "DETECTION PROOF (row 8)" "$f" || { echo "::error::row 8 break did not apply"; exit 1; }
|
||||
# Row 9 (execution runtime, s7): WAS `return_constant_number` raises
|
||||
# on entry. Expected: auto-run tier red `Constant Number:
|
||||
# EXECUTION_ERROR ... not in cannotRunAlone; a regression`.
|
||||
f=ComfyUI/custom_nodes/was-node-suite-comfyui/WAS_Node_Suite.py
|
||||
perl -pi -e 's/(def return_constant_number\(self, number_type, number, number_as_text=None\):)/$1\n raise ValueError("DETECTION PROOF (row 9): pack node runtime failure")/' "$f"
|
||||
grep -q "DETECTION PROOF (row 9)" "$f" || { echo "::error::row 9 break did not apply"; exit 1; }
|
||||
# Row 10 (registration sentinels, s5/s10): Impact renames the
|
||||
# `ImpactInt` node-class key, so the node no longer registers under
|
||||
# its expected class_type. Expected: the mount + curated tests that
|
||||
# reference ImpactInt skip, and the "Forbid skipped tests" gate fails
|
||||
# the job on `skipped != 0`.
|
||||
f=ComfyUI/custom_nodes/ComfyUI-Impact-Pack/__init__.py
|
||||
perl -pi -e 's/"ImpactInt": ImpactInt,/"ImpactIntDETECTIONPROOF": ImpactInt,/' "$f"
|
||||
grep -q "ImpactIntDETECTIONPROOF" "$f" || { echo "::error::row 10 break did not apply"; exit 1; }
|
||||
|
||||
# The VHS run-tier workflow reads input/plain_video.mp4.
|
||||
- name: Stage run-tier assets
|
||||
shell: bash
|
||||
run: cp browser_tests/assets/plain_video.mp4 ComfyUI/input/plain_video.mp4
|
||||
|
||||
# --cache-none so retried run-tier tests re-execute every node (a cached
|
||||
# node emits no `executing` event and would false-fail PARTIAL).
|
||||
- name: Start ComfyUI server
|
||||
shell: bash
|
||||
working-directory: ComfyUI
|
||||
run: |
|
||||
python main.py --cpu --multi-user --cache-none --front-end-root ../dist &
|
||||
wait-for-it --service 127.0.0.1:8188 -t 600
|
||||
|
||||
- name: Run custom-node suite
|
||||
env:
|
||||
PLAYWRIGHT_JSON_OUTPUT_NAME: custom-nodes-results.json
|
||||
run: |
|
||||
# workers=1: the auto-run tier needs exclusive backend-queue access;
|
||||
# parallel workers interrupt each other's executions.
|
||||
pnpm exec playwright test browser_tests/tests/customNodes/ \
|
||||
--project=chromium --reporter=list,json --workers=1
|
||||
|
||||
# A skip here means a pack or devtools did not load: on this backend every
|
||||
# tier is meant to run, so a skip is a gate failure, not an honest pass.
|
||||
- name: Forbid skipped tests
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
skipped=$(jq '.stats.skipped' custom-nodes-results.json)
|
||||
echo "skipped tests: $skipped"
|
||||
if [ "$skipped" != "0" ]; then
|
||||
echo "::error::$skipped test(s) skipped - a manifest pack or devtools failed to load; skips are not acceptable in the gating job"
|
||||
# Recurse so specs nested under describe() blocks are found, and
|
||||
# print only the specs that actually skipped.
|
||||
jq -r '.. | objects
|
||||
| select(has("title") and has("tests"))
|
||||
| select(any(.tests[]?; .status == "skipped"))
|
||||
| .title' custom-nodes-results.json | sort -u | head -40
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Playwright report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: playwright-report-custom-nodes
|
||||
path: playwright-report/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
@@ -123,6 +123,15 @@ Browser tests in this project follow a specific organization pattern:
|
||||
- **Utilities**: Located in `utils/` - Common utility functions
|
||||
- `litegraphUtils.ts` - Utilities for working with LiteGraph nodes
|
||||
|
||||
### Custom-node regression suite
|
||||
|
||||
`tests/customNodes/` holds the manifest-driven suite that proves community
|
||||
custom-node packs load, render in both renderers (LiteGraph canvas and Vue
|
||||
Nodes 2.0), and execute real workflows. It has its own prerequisites, pnpm
|
||||
scripts (`pnpm test:custom-nodes` and per-pack variants), and a
|
||||
one-JSON-row process for adding packs - see
|
||||
[tests/customNodes/README.md](tests/customNodes/README.md).
|
||||
|
||||
## Writing Effective Tests
|
||||
|
||||
When writing new tests, follow these patterns:
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"last_node_id": 2,
|
||||
"last_link_id": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "PrimitiveInt",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 100 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "INT",
|
||||
"type": "INT",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "PrimitiveInt"
|
||||
},
|
||||
"widgets_values": [5, "fixed"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [[1, 1, 0, 2, 0, "INT"]],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
53
browser_tests/assets/customNodes/core_smoke.json
Normal file
53
browser_tests/assets/customNodes/core_smoke.json
Normal file
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"last_node_id": 2,
|
||||
"last_link_id": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "PrimitiveInt",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 80 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "INT",
|
||||
"type": "INT",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "PrimitiveInt"
|
||||
},
|
||||
"widgets_values": [42, "fixed"]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [[1, 1, 0, 2, 0, "INT"]],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"last_node_id": 2,
|
||||
"last_link_id": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "StringFunction|pysssss",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 300, "1": 240 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "StringFunction|pysssss"
|
||||
},
|
||||
"widgets_values": ["append", "yes", "hello", " world", ""]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "ShowText|pysssss",
|
||||
"pos": { "0": 380, "1": 60 },
|
||||
"size": { "0": 220, "1": 80 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "STRING",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": null,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ShowText|pysssss"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [[1, 1, 0, 2, 0, "STRING"]],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"last_node_id": 2,
|
||||
"last_link_id": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "SimpleMathInt+",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "INT",
|
||||
"type": "INT",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "SimpleMathInt+"
|
||||
},
|
||||
"widgets_values": [5]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "DisplayAny",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 80 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "input",
|
||||
"type": "*",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": null,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "DisplayAny"
|
||||
},
|
||||
"widgets_values": ["raw value"]
|
||||
}
|
||||
],
|
||||
"links": [[1, 1, 0, 2, 0, "INT"]],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
98
browser_tests/assets/customNodes/impact_primitives_run.json
Normal file
98
browser_tests/assets/customNodes/impact_primitives_run.json
Normal file
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"last_node_id": 4,
|
||||
"last_link_id": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "ImpactInt",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "INT",
|
||||
"type": "INT",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ImpactInt"
|
||||
},
|
||||
"widgets_values": [42]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "ImpactFloat",
|
||||
"pos": { "0": 20, "1": 220 },
|
||||
"size": { "0": 250, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "FLOAT",
|
||||
"type": "FLOAT",
|
||||
"links": [2],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "ImpactFloat"
|
||||
},
|
||||
"widgets_values": [3.14]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 340, "1": 220 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 2, 0, "INT"],
|
||||
[2, 3, 0, 4, 0, "FLOAT"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
98
browser_tests/assets/customNodes/kjnodes_constants_run.json
Normal file
98
browser_tests/assets/customNodes/kjnodes_constants_run.json
Normal file
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"last_node_id": 4,
|
||||
"last_link_id": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "INTConstant",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "INT",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "INTConstant"
|
||||
},
|
||||
"widgets_values": [42]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "FloatConstant",
|
||||
"pos": { "0": 20, "1": 220 },
|
||||
"size": { "0": 250, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "FLOAT",
|
||||
"links": [2],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "FloatConstant"
|
||||
},
|
||||
"widgets_values": [3.14]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 340, "1": 220 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 3,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 2, 0, "INT"],
|
||||
[2, 3, 0, 4, 0, "FLOAT"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"last_node_id": 2,
|
||||
"last_link_id": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "Seed (rgthree)",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 130 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "SEED",
|
||||
"type": "INT",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Seed (rgthree)"
|
||||
},
|
||||
"widgets_values": [12345]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "Display Any (rgthree)",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "Display Any (rgthree)"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [[1, 1, 0, 2, 0, "INT"]],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
107
browser_tests/assets/customNodes/vhs_video_pipeline_run.json
Normal file
107
browser_tests/assets/customNodes/vhs_video_pipeline_run.json
Normal file
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"last_node_id": 3,
|
||||
"last_link_id": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "VHS_LoadVideoPath",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 320, "1": 260 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "IMAGE",
|
||||
"type": "IMAGE",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "frame_count",
|
||||
"type": "INT",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "audio",
|
||||
"type": "AUDIO",
|
||||
"links": null
|
||||
},
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"links": [1],
|
||||
"slot_index": 3
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_LoadVideoPath"
|
||||
},
|
||||
"widgets_values": ["input/plain_video.mp4", 0, 0, 0, 0, 0, 1]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "VHS_VideoInfo",
|
||||
"pos": { "0": 400, "1": 60 },
|
||||
"size": { "0": 240, "1": 260 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "video_info",
|
||||
"type": "VHS_VIDEOINFO",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "source_fps🟨",
|
||||
"type": "FLOAT",
|
||||
"links": [2],
|
||||
"slot_index": 0
|
||||
},
|
||||
{ "name": "source_frame_count🟨", "type": "INT", "links": null },
|
||||
{ "name": "source_duration🟨", "type": "FLOAT", "links": null },
|
||||
{ "name": "source_width🟨", "type": "INT", "links": null },
|
||||
{ "name": "source_height🟨", "type": "INT", "links": null },
|
||||
{ "name": "loaded_fps🟦", "type": "FLOAT", "links": null },
|
||||
{ "name": "loaded_frame_count🟦", "type": "INT", "links": null },
|
||||
{ "name": "loaded_duration🟦", "type": "FLOAT", "links": null },
|
||||
{ "name": "loaded_width🟦", "type": "INT", "links": null },
|
||||
{ "name": "loaded_height🟦", "type": "INT", "links": null }
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "VHS_VideoInfo"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "PreviewAny",
|
||||
"pos": { "0": 700, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "source",
|
||||
"type": "*",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"properties": {
|
||||
"Node name for S&R": "PreviewAny"
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 3, 2, 0, "VHS_VIDEOINFO"],
|
||||
[2, 2, 0, 3, 0, "FLOAT"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
103
browser_tests/assets/customNodes/was_number_text_run.json
Normal file
103
browser_tests/assets/customNodes/was_number_text_run.json
Normal file
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"last_node_id": 3,
|
||||
"last_link_id": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "Constant Number",
|
||||
"pos": { "0": 20, "1": 60 },
|
||||
"size": { "0": 250, "1": 100 },
|
||||
"flags": {},
|
||||
"order": 0,
|
||||
"mode": 0,
|
||||
"inputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "NUMBER",
|
||||
"type": "NUMBER",
|
||||
"links": [1],
|
||||
"slot_index": 0
|
||||
},
|
||||
{
|
||||
"name": "FLOAT",
|
||||
"type": "FLOAT",
|
||||
"links": null,
|
||||
"slot_index": 1
|
||||
},
|
||||
{
|
||||
"name": "INT",
|
||||
"type": "INT",
|
||||
"links": null,
|
||||
"slot_index": 2
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Constant Number"
|
||||
},
|
||||
"widgets_values": ["integer", 7]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "Number to Text",
|
||||
"pos": { "0": 340, "1": 60 },
|
||||
"size": { "0": 220, "1": 60 },
|
||||
"flags": {},
|
||||
"order": 1,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "number",
|
||||
"type": "NUMBER",
|
||||
"link": 1
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": [2],
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Number to Text"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "Text to Console",
|
||||
"pos": { "0": 640, "1": 60 },
|
||||
"size": { "0": 250, "1": 80 },
|
||||
"flags": {},
|
||||
"order": 2,
|
||||
"mode": 0,
|
||||
"inputs": [
|
||||
{
|
||||
"name": "text",
|
||||
"type": "STRING",
|
||||
"link": 2
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "STRING",
|
||||
"type": "STRING",
|
||||
"links": null,
|
||||
"slot_index": 0
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Node name for S&R": "Text to Console"
|
||||
},
|
||||
"widgets_values": ["Text Output"]
|
||||
}
|
||||
],
|
||||
"links": [
|
||||
[1, 1, 0, 2, 0, "NUMBER"],
|
||||
[2, 2, 0, 3, 0, "STRING"]
|
||||
],
|
||||
"groups": [],
|
||||
"config": {},
|
||||
"extra": {},
|
||||
"version": 0.4
|
||||
}
|
||||
@@ -268,8 +268,16 @@ export class ComfyPage {
|
||||
data: { username }
|
||||
})
|
||||
|
||||
if (resp.status() !== 200)
|
||||
throw new Error(`Failed to create user: ${await resp.text()}`)
|
||||
if (resp.status() !== 200) {
|
||||
const body = await resp.text()
|
||||
// Persistent backends (Comfy Desktop server user storage) keep the user
|
||||
// across runs and do not list it via GET /api/users, so a duplicate means
|
||||
// it already exists. Returns the username since the generated id is not
|
||||
// retrievable here; only reached on single-user / default-resolving backends.
|
||||
if (resp.status() === 400 && body.includes('Duplicate username.'))
|
||||
return username
|
||||
throw new Error(`Failed to create user: ${body}`)
|
||||
}
|
||||
|
||||
return await resp.json()
|
||||
}
|
||||
|
||||
287
browser_tests/fixtures/customNode/ComfyTarget.ts
Normal file
287
browser_tests/fixtures/customNode/ComfyTarget.ts
Normal file
@@ -0,0 +1,287 @@
|
||||
import type { Page, Response } from '@playwright/test'
|
||||
|
||||
import type { PromptResponse } from '@/schemas/apiSchema'
|
||||
|
||||
import type { ObjectInfo } from '@e2e/fixtures/customNode/objectInfoValidator'
|
||||
import type {
|
||||
ExecutionError,
|
||||
PromptEvent,
|
||||
RunResult
|
||||
} from '@e2e/fixtures/customNode/runResult'
|
||||
import { classifyRun } from '@e2e/fixtures/customNode/runResult'
|
||||
|
||||
interface RawEvent {
|
||||
type: string
|
||||
node?: string | null
|
||||
prompt_id?: string
|
||||
output?: unknown
|
||||
exception_type?: string
|
||||
node_id?: string
|
||||
node_type?: string
|
||||
traceback?: string[]
|
||||
}
|
||||
|
||||
const TERMINAL = [
|
||||
'execution_success',
|
||||
'execution_error',
|
||||
'execution_interrupted'
|
||||
]
|
||||
|
||||
// The /prompt rejection body is the apiSchema PromptResponse shape
|
||||
// ({ error: string | {message}, node_errors: { <nodeId>: { class_type,
|
||||
// errors: [{ details, message }] } } }). Flatten it to a single line naming
|
||||
// the node class and the failing input so a VALIDATION_FAIL result is
|
||||
// actionable instead of an empty object. Exported for a pure unit test: the
|
||||
// happy path never runs it, so without a test a regression here would rot the
|
||||
// diagnostic back to `{}` silently.
|
||||
export function summarizePromptError(body: unknown): string | undefined {
|
||||
const payload = body as Partial<PromptResponse> | null
|
||||
if (!payload || typeof payload !== 'object') return undefined
|
||||
const parts: string[] = []
|
||||
const topError = payload.error
|
||||
if (typeof topError === 'string') {
|
||||
if (topError) parts.push(topError)
|
||||
} else if (topError?.message) parts.push(topError.message)
|
||||
for (const [nodeId, nodeError] of Object.entries(payload.node_errors ?? {})) {
|
||||
const cls = nodeError.class_type || nodeId
|
||||
for (const err of nodeError.errors ?? []) {
|
||||
const detail = err.details || err.message
|
||||
if (detail) parts.push(`${cls}: ${detail}`)
|
||||
}
|
||||
}
|
||||
return parts.length > 0 ? parts.join('; ') : undefined
|
||||
}
|
||||
|
||||
function toPromptEvent(raw: RawEvent): PromptEvent {
|
||||
if (raw.type === 'executing')
|
||||
return { type: 'executing', node: raw.node ?? null }
|
||||
if (raw.type === 'executed')
|
||||
return { type: 'executed', node: raw.node ?? null, output: raw.output }
|
||||
if (raw.type === 'execution_error' || raw.type === 'execution_interrupted') {
|
||||
const error: ExecutionError = {
|
||||
exceptionType: raw.exception_type,
|
||||
nodeId: raw.node_id,
|
||||
nodeType: raw.node_type,
|
||||
traceback: raw.traceback
|
||||
}
|
||||
return { type: raw.type, error }
|
||||
}
|
||||
return { type: raw.type as 'execution_start' | 'execution_success' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Drives a real ComfyUI backend through the running frontend. The verdict logic
|
||||
* lives in the pure `classifyRun`; this class is only the in-page IO plumbing.
|
||||
*/
|
||||
export class LocalDesktopTarget {
|
||||
async getObjectInfo(page: Page): Promise<ObjectInfo> {
|
||||
return await page.evaluate(async () => {
|
||||
const defs = await window.app!.api.getNodeDefs()
|
||||
const out: Record<
|
||||
string,
|
||||
{ input?: { required?: Record<string, unknown> } }
|
||||
> = {}
|
||||
for (const [name, def] of Object.entries(defs)) {
|
||||
const required = (
|
||||
def as { input?: { required?: Record<string, unknown> } }
|
||||
).input?.required
|
||||
out[name] = { input: { required } }
|
||||
}
|
||||
return out
|
||||
})
|
||||
}
|
||||
|
||||
async runWorkflow(
|
||||
page: Page,
|
||||
opts: {
|
||||
expectedNodeIds: string[]
|
||||
graphNodeIds?: string[]
|
||||
timeoutMs: number
|
||||
}
|
||||
): Promise<RunResult> {
|
||||
// A prior run's terminal event can arrive after its sink was read (late
|
||||
// websocket delivery, or a timed-out prompt finishing during this run).
|
||||
// Remember every prompt id already observed and ignore its events here,
|
||||
// so one node's failure is never attributed to the next node tested.
|
||||
const seenPromptIds = await page.evaluate(
|
||||
(types) => {
|
||||
const sink = window as unknown as {
|
||||
__cnEvents: RawEvent[]
|
||||
__cnSeenPromptIds?: string[]
|
||||
__cnTapInstalled?: boolean
|
||||
}
|
||||
const seen = new Set(sink.__cnSeenPromptIds ?? [])
|
||||
for (const event of sink.__cnEvents ?? [])
|
||||
if (event.prompt_id) seen.add(event.prompt_id)
|
||||
sink.__cnSeenPromptIds = [...seen]
|
||||
sink.__cnEvents = []
|
||||
if (sink.__cnTapInstalled) return sink.__cnSeenPromptIds
|
||||
sink.__cnTapInstalled = true
|
||||
for (const type of types)
|
||||
(window.app!.api as EventTarget).addEventListener(
|
||||
type,
|
||||
(event: Event) => {
|
||||
const detail: unknown = (event as CustomEvent).detail
|
||||
// `executing` dispatches a bare node-id string (api.ts
|
||||
// dispatchCustomEvent('executing', msg.data.node)); the other
|
||||
// events dispatch object payloads.
|
||||
sink.__cnEvents.push(
|
||||
detail !== null && typeof detail === 'object'
|
||||
? { type, ...(detail as Record<string, unknown>) }
|
||||
: { type, node: (detail as string | undefined) ?? null }
|
||||
)
|
||||
}
|
||||
)
|
||||
return sink.__cnSeenPromptIds
|
||||
},
|
||||
['execution_start', ...TERMINAL, 'executing', 'executed']
|
||||
)
|
||||
|
||||
// Positively identify THIS attempt: the /prompt POST response body
|
||||
// carries the prompt_id the backend assigned. When captured it becomes
|
||||
// the primary event filter; the seen-set above and the graph-membership
|
||||
// check below stay as defense in depth (capture can lose a race with a
|
||||
// transient refusal, and `executing` events carry no prompt id at all).
|
||||
let capturedPromptId: string | undefined
|
||||
// A backend validation rejection answers /prompt with a non-2xx body
|
||||
// carrying { error, node_errors }. app.queuePrompt swallows it and just
|
||||
// returns false, so without capturing it here a VALIDATION_FAIL result
|
||||
// names nothing. Snapshot the failing node/input so the outcome is
|
||||
// actionable instead of an empty object.
|
||||
let capturedValidationError: string | undefined
|
||||
const onPromptResponse = (response: Response) => {
|
||||
if (response.request().method() !== 'POST') return
|
||||
if (!new URL(response.url()).pathname.endsWith('/prompt')) return
|
||||
response
|
||||
.json()
|
||||
.then((body: unknown) => {
|
||||
const id = (body as { prompt_id?: unknown } | null)?.prompt_id
|
||||
if (typeof id === 'string') capturedPromptId = id
|
||||
if (response.status() >= 400)
|
||||
capturedValidationError = summarizePromptError(body)
|
||||
})
|
||||
.catch(() => {
|
||||
// a refused submission answers with a non-JSON or error body;
|
||||
// the refusal path below already handles it
|
||||
})
|
||||
}
|
||||
page.on('response', onPromptResponse)
|
||||
const stopCapture = () => page.off('response', onPromptResponse)
|
||||
|
||||
// app.queuePrompt (NOT api.queuePrompt: that submits an empty prompt).
|
||||
// false = validation reject (emits no events), but pack JS hooking the
|
||||
// queue can refuse transiently - retry once; real rejects fail twice.
|
||||
// Pack JS can also THROW mid-graphToPrompt on a graph shape it does not
|
||||
// expect; catch in-page so one bad node classifies as VALIDATION_FAIL
|
||||
// (with the exception text) instead of aborting the whole tier.
|
||||
const queueOnce = () =>
|
||||
page.evaluate(async () => {
|
||||
try {
|
||||
return await window.app!.queuePrompt(0)
|
||||
} catch (error) {
|
||||
// Never an empty string: an empty __cnThrew would nullish-coalesce
|
||||
// wrong downstream and blank the VALIDATION_FAIL message.
|
||||
return { __cnThrew: String(error) || 'pack threw an empty error' }
|
||||
}
|
||||
})
|
||||
const refused = (
|
||||
result: unknown
|
||||
): result is false | { __cnThrew: string } =>
|
||||
result === false ||
|
||||
(typeof result === 'object' && result !== null && '__cnThrew' in result)
|
||||
let queued = await queueOnce()
|
||||
if (refused(queued)) {
|
||||
await page.evaluate(
|
||||
() => new Promise((resolve) => setTimeout(resolve, 250))
|
||||
)
|
||||
queued = await queueOnce()
|
||||
if (refused(queued)) {
|
||||
stopCapture()
|
||||
return {
|
||||
outcome: 'VALIDATION_FAIL',
|
||||
executedNodes: [],
|
||||
outputsByNode: {},
|
||||
// A throw carries its own text; a bare `false` reject leaves only
|
||||
// the backend's node_errors captured off the /prompt response.
|
||||
clientError:
|
||||
(typeof queued === 'object' ? queued.__cnThrew : undefined) ??
|
||||
capturedValidationError
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The submission resolved, so the /prompt response is in flight or done;
|
||||
// give its body-parse a bounded beat before snapshotting the id.
|
||||
const captureDeadline = Date.now() + 2_000
|
||||
while (capturedPromptId === undefined && Date.now() < captureDeadline)
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
// A silent permanent miss would degrade every run to the legacy filters
|
||||
// with no signal - make the fallback observable in the runner output.
|
||||
if (capturedPromptId === undefined)
|
||||
console.warn(
|
||||
'[customNodes] /prompt response id capture missed; falling back to seen-set filtering'
|
||||
)
|
||||
|
||||
await page
|
||||
.waitForFunction(
|
||||
([terminal, seen, graphIds, promptId]) => {
|
||||
const events =
|
||||
(
|
||||
window as unknown as {
|
||||
__cnEvents?: {
|
||||
type: string
|
||||
prompt_id?: string
|
||||
node_id?: string
|
||||
}[]
|
||||
}
|
||||
).__cnEvents ?? []
|
||||
return events.some(
|
||||
(event) =>
|
||||
terminal.includes(event.type) &&
|
||||
(promptId !== null
|
||||
? event.prompt_id === promptId
|
||||
: !(event.prompt_id && seen.includes(event.prompt_id)) &&
|
||||
(graphIds === null ||
|
||||
event.node_id === undefined ||
|
||||
graphIds.includes(event.node_id)))
|
||||
)
|
||||
},
|
||||
[
|
||||
TERMINAL,
|
||||
seenPromptIds ?? [],
|
||||
opts.graphNodeIds ?? null,
|
||||
capturedPromptId ?? null
|
||||
] as const,
|
||||
{ timeout: opts.timeoutMs }
|
||||
)
|
||||
.catch((error: unknown) => {
|
||||
// Only a Playwright wait timeout means "no terminal event"; surface any
|
||||
// other fault instead of masquerading it as a run TIMEOUT.
|
||||
if (error instanceof Error && error.name === 'TimeoutError') return
|
||||
stopCapture()
|
||||
throw error
|
||||
})
|
||||
stopCapture()
|
||||
|
||||
const raw = (
|
||||
await page.evaluate(
|
||||
() =>
|
||||
(window as unknown as { __cnEvents?: RawEvent[] }).__cnEvents ?? []
|
||||
)
|
||||
).filter((event) =>
|
||||
// Positive id match when captured (events without a prompt_id - bare
|
||||
// `executing` strings - stay, and graph membership still vets them);
|
||||
// otherwise the legacy seen-set exclusion.
|
||||
capturedPromptId !== undefined
|
||||
? event.prompt_id === undefined || event.prompt_id === capturedPromptId
|
||||
: !(event.prompt_id && (seenPromptIds ?? []).includes(event.prompt_id))
|
||||
)
|
||||
const timedOut = !raw.some((event) => TERMINAL.includes(event.type))
|
||||
return classifyRun({
|
||||
events: raw.map(toPromptEvent),
|
||||
expectedNodeIds: opts.expectedNodeIds,
|
||||
graphNodeIds: opts.graphNodeIds,
|
||||
timedOut
|
||||
})
|
||||
}
|
||||
}
|
||||
165
browser_tests/fixtures/customNode/autoRun.ts
Normal file
165
browser_tests/fixtures/customNode/autoRun.ts
Normal file
@@ -0,0 +1,165 @@
|
||||
// Classifies which nodes can execute with no hand-authored fixture; the
|
||||
// rest are recorded with the reason, never silently dropped.
|
||||
import type { RawNodeDef } from '@e2e/fixtures/customNode/typePairing'
|
||||
|
||||
type AutoRunClass =
|
||||
// Widgets cover every required input and a terminus exists.
|
||||
| 'AUTO_RUNNABLE'
|
||||
// Every required socket is synthesizable from a model-free producer.
|
||||
| 'CHAINABLE'
|
||||
// A required socket type has no model-free producer (MODEL, CLIP, SEGS...).
|
||||
| 'NEEDS_WIRES'
|
||||
// A required combo has zero options (empty model/file scan).
|
||||
| 'NEEDS_MODELS'
|
||||
// No outputs and not an OUTPUT_NODE - nothing the executor could watch.
|
||||
| 'NO_OBSERVABLE_OUTPUT'
|
||||
|
||||
export interface RequiredSocket {
|
||||
name: string
|
||||
type: string
|
||||
}
|
||||
|
||||
export interface AutoRunVerdict {
|
||||
key: string
|
||||
verdict: AutoRunClass
|
||||
// Wire output 0 to PreviewAny (false = the node is its own terminus).
|
||||
needsPreviewSink?: boolean
|
||||
// CHAINABLE: sockets to satisfy from SYNTH_PRODUCERS, in declaration order.
|
||||
requiredSockets?: RequiredSocket[]
|
||||
reason: string
|
||||
}
|
||||
|
||||
// Model-free producers for each synthesizable socket type. NUMBER is a WAS
|
||||
// type with a WAS producer, so each entry is validated against the live defs
|
||||
// before it counts as synthesizable.
|
||||
export const SYNTH_PRODUCERS: Record<
|
||||
string,
|
||||
{ nodeType: string; outputIndex: number }
|
||||
> = {
|
||||
IMAGE: { nodeType: 'EmptyImage', outputIndex: 0 },
|
||||
LATENT: { nodeType: 'EmptyLatentImage', outputIndex: 0 },
|
||||
MASK: { nodeType: 'SolidMask', outputIndex: 0 },
|
||||
INT: { nodeType: 'PrimitiveInt', outputIndex: 0 },
|
||||
FLOAT: { nodeType: 'PrimitiveFloat', outputIndex: 0 },
|
||||
STRING: { nodeType: 'PrimitiveString', outputIndex: 0 },
|
||||
BOOLEAN: { nodeType: 'PrimitiveBoolean', outputIndex: 0 },
|
||||
AUDIO: { nodeType: 'EmptyAudio', outputIndex: 0 },
|
||||
NUMBER: { nodeType: 'Constant Number', outputIndex: 0 },
|
||||
'*': { nodeType: 'PrimitiveInt', outputIndex: 0 }
|
||||
}
|
||||
|
||||
const WIDGET_TYPES = new Set(['INT', 'FLOAT', 'STRING', 'BOOLEAN'])
|
||||
|
||||
type InputSpec = [unknown, Record<string, unknown>?] | unknown
|
||||
|
||||
function classifyInput(spec: InputSpec): 'widget' | 'socket' | 'empty-combo' {
|
||||
const specArray = Array.isArray(spec) ? spec : [spec]
|
||||
const rawType = specArray[0]
|
||||
const options = specArray[1] as
|
||||
| { forceInput?: boolean; options?: unknown }
|
||||
| undefined
|
||||
// forceInput beats every form, combos included: no widget materializes,
|
||||
// so no default exists to run on - the input must be wired.
|
||||
if (options?.forceInput) return 'socket'
|
||||
if (Array.isArray(rawType))
|
||||
return rawType.length > 0 ? 'widget' : 'empty-combo'
|
||||
if (typeof rawType !== 'string') return 'socket'
|
||||
if (rawType === 'COMBO') {
|
||||
// Transformed (V2-schema) defs carry combos as the literal 'COMBO' with
|
||||
// the option list in the opts object. No static list (empty, or a
|
||||
// `remote` lazy combo) means the default value cannot be verified
|
||||
// runnable at plan time - same bucket as an empty model scan.
|
||||
return Array.isArray(options?.options) && options.options.length > 0
|
||||
? 'widget'
|
||||
: 'empty-combo'
|
||||
}
|
||||
return WIDGET_TYPES.has(rawType) ? 'widget' : 'socket'
|
||||
}
|
||||
|
||||
function socketType(spec: InputSpec): string {
|
||||
const specArray = Array.isArray(spec) ? spec : [spec]
|
||||
return String(specArray[0])
|
||||
}
|
||||
|
||||
export function classifyAutoRunnable(
|
||||
key: string,
|
||||
def: RawNodeDef & { output_node?: boolean },
|
||||
synthTypes: ReadonlySet<string>
|
||||
): AutoRunVerdict {
|
||||
const sockets: RequiredSocket[] = []
|
||||
for (const [name, spec] of Object.entries(def.input?.required ?? {})) {
|
||||
const kind = classifyInput(spec)
|
||||
if (kind === 'empty-combo')
|
||||
return {
|
||||
key,
|
||||
verdict: 'NEEDS_MODELS',
|
||||
reason: `required combo "${name}" has no options on this backend`
|
||||
}
|
||||
if (kind === 'socket') {
|
||||
const type = socketType(spec)
|
||||
if (!synthTypes.has(type))
|
||||
return {
|
||||
key,
|
||||
verdict: 'NEEDS_WIRES',
|
||||
reason: `required input "${name}" (${type}) has no model-free producer`
|
||||
}
|
||||
sockets.push({ name, type })
|
||||
}
|
||||
}
|
||||
const terminus =
|
||||
def.output_node === true
|
||||
? { needsPreviewSink: false, note: 'node is its own terminus' }
|
||||
: (def.output ?? []).length > 0
|
||||
? { needsPreviewSink: true, note: 'output 0 -> PreviewAny' }
|
||||
: null
|
||||
if (!terminus)
|
||||
return {
|
||||
key,
|
||||
verdict: 'NO_OBSERVABLE_OUTPUT',
|
||||
reason: 'no outputs and not an OUTPUT_NODE - nothing observable to queue'
|
||||
}
|
||||
if (sockets.length === 0)
|
||||
return {
|
||||
key,
|
||||
verdict: 'AUTO_RUNNABLE',
|
||||
needsPreviewSink: terminus.needsPreviewSink,
|
||||
reason: `widgets satisfy all required inputs; ${terminus.note}`
|
||||
}
|
||||
return {
|
||||
key,
|
||||
verdict: 'CHAINABLE',
|
||||
needsPreviewSink: terminus.needsPreviewSink,
|
||||
requiredSockets: sockets,
|
||||
reason: `${sockets.length} required socket(s) synthesized from model-free producers; ${terminus.note}`
|
||||
}
|
||||
}
|
||||
|
||||
export function planAutoRuns(
|
||||
defs: Record<string, RawNodeDef & { output_node?: boolean }>,
|
||||
packNodeKeys: string[]
|
||||
): AutoRunVerdict[] {
|
||||
// A producer only counts if the backend actually registers it.
|
||||
const synthTypes = new Set(
|
||||
Object.entries(SYNTH_PRODUCERS)
|
||||
.filter(([, producer]) => producer.nodeType in defs)
|
||||
.map(([type]) => type)
|
||||
)
|
||||
return packNodeKeys.map((key) =>
|
||||
classifyAutoRunnable(key, defs[key], synthTypes)
|
||||
)
|
||||
}
|
||||
|
||||
// Independent chains per prompt so one bad node fails a batch, not the tier.
|
||||
export function batchAutoRunnable(
|
||||
verdicts: AutoRunVerdict[],
|
||||
batchSize: number
|
||||
): AutoRunVerdict[][] {
|
||||
const runnable = verdicts.filter(
|
||||
(verdict) =>
|
||||
verdict.verdict === 'AUTO_RUNNABLE' || verdict.verdict === 'CHAINABLE'
|
||||
)
|
||||
const batches: AutoRunVerdict[][] = []
|
||||
for (let offset = 0; offset < runnable.length; offset += batchSize)
|
||||
batches.push(runnable.slice(offset, offset + batchSize))
|
||||
return batches
|
||||
}
|
||||
82
browser_tests/fixtures/customNode/consoleErrorLedger.ts
Normal file
82
browser_tests/fixtures/customNode/consoleErrorLedger.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
// Pack-attributed console noise with no visible error surface. Shared by
|
||||
// the all-nodes tiers and the curated run tier so one ledger covers every
|
||||
// surface a pack's script can emit on. Filter-guarded: a pattern suppresses
|
||||
// matching errors for its pack only; stale entries are caught by review,
|
||||
// not observation (several patterns are environment-conditional, so
|
||||
// observed-firing guards would false-fail - see ARCHITECTURE.md section 10).
|
||||
export const CONSOLE_ERROR_ALLOWLIST: Record<
|
||||
string,
|
||||
Array<{ pattern: RegExp; reason: string }>
|
||||
> = {
|
||||
'ComfyUI-Impact-Pack': [
|
||||
{
|
||||
// Media/text widgets preview their value via root-relative URLs at
|
||||
// creation; 404s on a backend whose root does not serve the file.
|
||||
pattern:
|
||||
/Failed to load resource.*404.*(example\.png|plain_video\.mp4|file\.txt)/,
|
||||
reason: 'media widget previews its value via a root-relative URL'
|
||||
},
|
||||
{
|
||||
// PreviewBridge widgets fetch their internal preview id on configure;
|
||||
// a bare backend has no image behind it.
|
||||
pattern: /Failed to load resource.*400.*api\/impact\/get\/pb_id_image/,
|
||||
reason: 'PreviewBridge fetches its preview id on configure'
|
||||
},
|
||||
{
|
||||
// The save/reload tier writes `<value>_cn` probe values; media widgets
|
||||
// preview them as URLs and 404.
|
||||
pattern: /Failed to load resource.*404.*_cn/,
|
||||
reason: 'set-and-stick probe value previewed by a media widget'
|
||||
}
|
||||
],
|
||||
'ComfyUI-KJNodes': [
|
||||
{
|
||||
// Image/video loader previews fetch their combo value at creation;
|
||||
// on a backend with an empty input dir the value is undefined and the
|
||||
// preview 404s (and retries with a fresh rand). Console-only noise,
|
||||
// no visible error; upstream-report candidate.
|
||||
pattern:
|
||||
/Failed to load resource.*\/api\/view\?type=input&filename=undefined/,
|
||||
reason: 'loader preview fetches undefined filename on empty input dir'
|
||||
}
|
||||
],
|
||||
'ComfyUI-Custom-Scripts': [
|
||||
{
|
||||
// betterCombos.js:473 checks `typeof ret === "object" && "content" in
|
||||
// ret`; typeof null is "object", so a null ret during save/reload
|
||||
// throws `Cannot use 'in' operator to search for 'content' in null`
|
||||
// as an uncaught page error - invisible until pageerror collection
|
||||
// landed. Pack-owned and deterministic; upstream-report candidate.
|
||||
pattern: /Cannot use 'in' operator to search for 'content' in null/,
|
||||
reason: 'betterCombos.js missing null check throws during save/reload'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
export function unallowlistedErrors(pack: string, errors: string[]): string[] {
|
||||
const allowlist = CONSOLE_ERROR_ALLOWLIST[pack] ?? []
|
||||
return errors.filter(
|
||||
(error) => !allowlist.some((rule) => rule.pattern.test(error))
|
||||
)
|
||||
}
|
||||
|
||||
// Execution errors surface on the tiers that actually queue prompts (the
|
||||
// curated run and the auto-run tier). The mount, persistence, and wiring
|
||||
// tiers queue nothing, so a prompt-execution error arriving in their console
|
||||
// collector is an async stray from a prior tier's still-draining execution -
|
||||
// the same "not this test" principle the event-attribution filter uses
|
||||
// (ARCHITECTURE section 9). It is filtered from the non-executing tiers only;
|
||||
// the executing tiers still assert on it. This is not error suppression: the
|
||||
// visible error SURFACES (overlay/dialog/toast) are still asserted separately
|
||||
// by expectNoVisibleErrors.
|
||||
const FOREIGN_EXECUTION_NOISE: RegExp[] = [
|
||||
/PromptExecutionError/,
|
||||
/Prompt execution failed/,
|
||||
// The browser logs a rejected prompt submission as a failed resource load
|
||||
// on /api/prompt. Only the executing tiers POST there, so this line in a
|
||||
// mount/persistence/wiring collector is a prior tier's async submission.
|
||||
/Failed to load resource.*\/api\/prompt/
|
||||
]
|
||||
export function isForeignExecutionNoise(error: string): boolean {
|
||||
return FOREIGN_EXECUTION_NOISE.some((pattern) => pattern.test(error))
|
||||
}
|
||||
137
browser_tests/fixtures/customNode/manifest.ts
Normal file
137
browser_tests/fixtures/customNode/manifest.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const MANIFEST_PATH = fileURLToPath(
|
||||
new URL('../data/customNodeManifest.json', import.meta.url)
|
||||
)
|
||||
|
||||
const VALID_TIERS = ['load', 'run', 'connectivity', 'io'] as const
|
||||
|
||||
type CustomNodeTier = (typeof VALID_TIERS)[number]
|
||||
|
||||
export interface CustomNodeManifestEntry {
|
||||
pack: string
|
||||
repo: string
|
||||
pin: string
|
||||
tiers: CustomNodeTier[]
|
||||
// Frontend-format workflow (path relative to browser_tests/) loaded and queued
|
||||
// by the run tier; empty or absent file = tier skips. Run the backend with
|
||||
// --cache-none, or repeat runs classify PARTIAL when cached nodes skip executing.
|
||||
workflow: string
|
||||
// Runtime class_type / object_info keys, NOT Python class names (e.g. rgthree
|
||||
// registers "Power Primitive (rgthree)", not RgthreePowerPrimitive).
|
||||
expectedNodes: string[]
|
||||
requiresGpu: boolean
|
||||
requiresModels: string[]
|
||||
timeoutMs: number
|
||||
// Optional; absent means true. Set false ONLY with evidence that the pack's
|
||||
// nodes fail to mount under Vue Nodes 2.0 (probe it - a README grumble is
|
||||
// not evidence). When false, renderer-specific Vue assertions are not
|
||||
// applied to this pack: its tests still run and pass their LiteGraph-canvas
|
||||
// assertions, so the zero-skip gate is preserved.
|
||||
vueNodesCompatible?: boolean
|
||||
// Node key -> evidenced reason it cannot mount under Vue Nodes 2.0; only
|
||||
// the Vue mount assertion is withheld. Stale keys fail the suite.
|
||||
vueIncompatibleNodes?: Record<string, string>
|
||||
// Nodes that cannot execute on pure defaults. Asserted both ways: an
|
||||
// unlisted failure is a regression, a listed clean run is a stale entry.
|
||||
cannotRunAlone?: string[]
|
||||
}
|
||||
|
||||
// Exported for the pure spec's validation cases; production callers go
|
||||
// through loadManifest.
|
||||
export function assertEntry(
|
||||
entry: CustomNodeManifestEntry,
|
||||
index: number
|
||||
): void {
|
||||
const missing: string[] = []
|
||||
// CI installs the pack into custom_nodes/<pack>, and node attribution keys
|
||||
// on that directory name via python_module - so pack must be a safe,
|
||||
// plain path segment, not just non-empty.
|
||||
if (
|
||||
typeof entry.pack !== 'string' ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(entry.pack)
|
||||
)
|
||||
missing.push('pack (must be a plain path segment)')
|
||||
// CI clones from repo, so an empty value must fail here, not mid-clone.
|
||||
if (typeof entry.repo !== 'string' || entry.repo.length === 0)
|
||||
missing.push('repo')
|
||||
// The gate tests exactly what was verified, so pin is a required full
|
||||
// commit SHA. CUSTOM_NODES_ALLOW_UNPINNED=1 is the one escape hatch,
|
||||
// reserved for the planned pack-HEAD canary - never for the PR gate.
|
||||
if (
|
||||
!/^[0-9a-f]{40}$/.test(entry.pin ?? '') &&
|
||||
!(
|
||||
process.env.CUSTOM_NODES_ALLOW_UNPINNED === '1' &&
|
||||
(entry.pin ?? '') === ''
|
||||
)
|
||||
)
|
||||
missing.push('pin (full 40-char commit SHA required)')
|
||||
// workflow may be an empty string until the pack gains a run-tier fixture.
|
||||
if (typeof entry.workflow !== 'string') missing.push('workflow')
|
||||
// A run-tier row with no workflow would otherwise skip locally, leaving
|
||||
// only CI's skip gate to notice the lost coverage. Fail at load instead.
|
||||
else if (
|
||||
entry.workflow === '' &&
|
||||
Array.isArray(entry.tiers) &&
|
||||
entry.tiers.includes('run')
|
||||
)
|
||||
missing.push('workflow (required when tiers includes "run")')
|
||||
if (!Array.isArray(entry.expectedNodes) || entry.expectedNodes.length === 0)
|
||||
missing.push('expectedNodes')
|
||||
if (!Array.isArray(entry.tiers) || entry.tiers.length === 0)
|
||||
missing.push('tiers')
|
||||
// A typo like "connectivty" would otherwise pass and silently drop that
|
||||
// tier's coverage - the exact drift this manifest exists to catch.
|
||||
else if (entry.tiers.some((tier) => !VALID_TIERS.includes(tier)))
|
||||
missing.push(`tiers (unknown value; allowed: ${VALID_TIERS.join(', ')})`)
|
||||
if (!Array.isArray(entry.requiresModels)) missing.push('requiresModels')
|
||||
if (typeof entry.requiresGpu !== 'boolean') missing.push('requiresGpu')
|
||||
if (!Number.isFinite(entry.timeoutMs) || entry.timeoutMs <= 0)
|
||||
missing.push('timeoutMs')
|
||||
if (
|
||||
entry.vueNodesCompatible !== undefined &&
|
||||
typeof entry.vueNodesCompatible !== 'boolean'
|
||||
)
|
||||
missing.push('vueNodesCompatible')
|
||||
if (
|
||||
entry.vueIncompatibleNodes !== undefined &&
|
||||
(typeof entry.vueIncompatibleNodes !== 'object' ||
|
||||
entry.vueIncompatibleNodes === null ||
|
||||
Array.isArray(entry.vueIncompatibleNodes) ||
|
||||
Object.values(entry.vueIncompatibleNodes).some(
|
||||
(reason) => typeof reason !== 'string' || reason.length === 0
|
||||
))
|
||||
)
|
||||
missing.push('vueIncompatibleNodes (node key -> non-empty reason string)')
|
||||
if (
|
||||
entry.cannotRunAlone !== undefined &&
|
||||
(!Array.isArray(entry.cannotRunAlone) ||
|
||||
entry.cannotRunAlone.some(
|
||||
(key) => typeof key !== 'string' || key.length === 0
|
||||
) ||
|
||||
new Set(entry.cannotRunAlone).size !== entry.cannotRunAlone.length)
|
||||
)
|
||||
missing.push('cannotRunAlone (unique non-empty node keys)')
|
||||
if (missing.length > 0)
|
||||
throw new Error(
|
||||
`custom-node manifest entry ${index} (${entry.pack ?? '?'}) missing: ${missing.join(', ')}`
|
||||
)
|
||||
}
|
||||
|
||||
// Renderer passes for the load tier: LiteGraph canvas always, Vue Nodes 2.0
|
||||
// unless the pack declares itself incompatible. Conditional coverage, never a
|
||||
// test.skip - the caller still runs and gates on the returned passes.
|
||||
export function rendererPassesFor(
|
||||
entry: Pick<CustomNodeManifestEntry, 'vueNodesCompatible'>
|
||||
): boolean[] {
|
||||
return entry.vueNodesCompatible === false ? [false] : [false, true]
|
||||
}
|
||||
|
||||
export function loadManifest(): CustomNodeManifestEntry[] {
|
||||
const entries = JSON.parse(
|
||||
readFileSync(MANIFEST_PATH, 'utf-8')
|
||||
) as CustomNodeManifestEntry[]
|
||||
entries.forEach(assertEntry)
|
||||
return entries
|
||||
}
|
||||
54
browser_tests/fixtures/customNode/objectInfoValidator.ts
Normal file
54
browser_tests/fixtures/customNode/objectInfoValidator.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import type { CustomNodeOutcome } from '@e2e/fixtures/customNode/runResult'
|
||||
|
||||
interface ObjectInfoNode {
|
||||
input?: { required?: Record<string, unknown> }
|
||||
}
|
||||
export type ObjectInfo = Record<string, ObjectInfoNode>
|
||||
|
||||
export interface ApiPromptNode {
|
||||
id: string
|
||||
classType: string
|
||||
inputs: Record<string, unknown>
|
||||
}
|
||||
|
||||
export function expectedNodesPresent(
|
||||
objectInfo: ObjectInfo,
|
||||
expectedNodes: string[]
|
||||
): { present: string[]; missing: string[] } {
|
||||
const present: string[] = []
|
||||
const missing: string[] = []
|
||||
for (const name of expectedNodes) {
|
||||
if (name in objectInfo) present.push(name)
|
||||
else missing.push(name)
|
||||
}
|
||||
return { present, missing }
|
||||
}
|
||||
|
||||
export interface PreValidationFailure {
|
||||
outcome: Extract<CustomNodeOutcome, 'MISSING_NODE' | 'VALIDATION_FAIL'>
|
||||
message: string
|
||||
}
|
||||
|
||||
// Turns an opaque backend 400 into a precise infra error before submit (BE-401):
|
||||
// every required input declared in object_info must be present in the fixture node.
|
||||
export function preValidate(
|
||||
objectInfo: ObjectInfo,
|
||||
nodes: ApiPromptNode[]
|
||||
): PreValidationFailure | null {
|
||||
for (const node of nodes) {
|
||||
const def = objectInfo[node.classType]
|
||||
if (!def)
|
||||
return {
|
||||
outcome: 'MISSING_NODE',
|
||||
message: `node ${node.id} ${node.classType} missing from object_info`
|
||||
}
|
||||
for (const name of Object.keys(def.input?.required ?? {})) {
|
||||
if (!(name in node.inputs))
|
||||
return {
|
||||
outcome: 'VALIDATION_FAIL',
|
||||
message: `node ${node.id} ${node.classType} missing required input "${name}"`
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
107
browser_tests/fixtures/customNode/runResult.ts
Normal file
107
browser_tests/fixtures/customNode/runResult.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
export type CustomNodeOutcome =
|
||||
| 'NOT_INSTALLED'
|
||||
| 'IMPORT_ERROR'
|
||||
| 'MISSING_NODE'
|
||||
| 'VALIDATION_FAIL'
|
||||
| 'EXECUTION_ERROR'
|
||||
| 'PARTIAL'
|
||||
| 'TIMEOUT'
|
||||
| 'PASS'
|
||||
|
||||
export interface ExecutionError {
|
||||
exceptionType?: string
|
||||
nodeId?: string
|
||||
nodeType?: string
|
||||
traceback?: string[]
|
||||
}
|
||||
|
||||
export type PromptEvent =
|
||||
| { type: 'execution_start' }
|
||||
| { type: 'executing'; node: string | null }
|
||||
| { type: 'executed'; node: string | null; output?: unknown }
|
||||
| { type: 'execution_success' }
|
||||
| { type: 'execution_error'; error: ExecutionError }
|
||||
| { type: 'execution_interrupted'; error?: ExecutionError }
|
||||
|
||||
export interface RunResult {
|
||||
outcome: CustomNodeOutcome
|
||||
executedNodes: string[]
|
||||
// ui payloads from `executed` events, keyed by node id - proof that data
|
||||
// reached each output node, not just that execution finished.
|
||||
outputsByNode: Record<string, unknown>
|
||||
error?: ExecutionError
|
||||
// Set when queuePrompt THREW client-side (pack JS hooking the queue can
|
||||
// crash on a graph shape it does not expect); carries the exception text
|
||||
// so the failing node self-identifies in the report.
|
||||
clientError?: string
|
||||
}
|
||||
|
||||
// `executing` with a non-null node is the only cache-safe "this node actually ran"
|
||||
// signal: ComfyUI emits it solely for non-cached nodes (execution.py:493), while the
|
||||
// `executed` message and /history outputs are replayed for cached nodes too.
|
||||
function executedNodesFrom(events: PromptEvent[]): string[] {
|
||||
const executed = new Set<string>()
|
||||
for (const event of events) {
|
||||
if (event.type === 'executing' && event.node !== null)
|
||||
executed.add(event.node)
|
||||
}
|
||||
return [...executed]
|
||||
}
|
||||
|
||||
function outputsFrom(events: PromptEvent[]): Record<string, unknown> {
|
||||
const outputs: Record<string, unknown> = {}
|
||||
for (const event of events) {
|
||||
if (event.type === 'executed' && event.node !== null)
|
||||
outputs[event.node] = event.output
|
||||
}
|
||||
return outputs
|
||||
}
|
||||
|
||||
export function classifyRun(input: {
|
||||
events: PromptEvent[]
|
||||
expectedNodeIds: string[]
|
||||
// All node ids in the queued graph. An error naming a node outside it is a
|
||||
// stray from another prompt (late websocket delivery, or a duplicate queue
|
||||
// from the client-flap retry) and must not be pinned on this run.
|
||||
graphNodeIds?: string[]
|
||||
timedOut?: boolean
|
||||
}): RunResult {
|
||||
const { events, expectedNodeIds, graphNodeIds, timedOut = false } = input
|
||||
const executedNodes = executedNodesFrom(events)
|
||||
const outputsByNode = outputsFrom(events)
|
||||
|
||||
if (timedOut) return { outcome: 'TIMEOUT', executedNodes, outputsByNode }
|
||||
|
||||
const failure = events.find(
|
||||
(
|
||||
event
|
||||
): event is Extract<
|
||||
PromptEvent,
|
||||
{ type: 'execution_error' | 'execution_interrupted' }
|
||||
> =>
|
||||
(event.type === 'execution_error' ||
|
||||
event.type === 'execution_interrupted') &&
|
||||
(graphNodeIds === undefined ||
|
||||
event.error?.nodeId === undefined ||
|
||||
graphNodeIds.includes(event.error.nodeId))
|
||||
)
|
||||
if (failure)
|
||||
return {
|
||||
outcome: 'EXECUTION_ERROR',
|
||||
executedNodes,
|
||||
outputsByNode,
|
||||
error: failure.error
|
||||
}
|
||||
|
||||
if (!events.some((event) => event.type === 'execution_success'))
|
||||
return { outcome: 'TIMEOUT', executedNodes, outputsByNode }
|
||||
|
||||
const ranEveryExpected = expectedNodeIds.every((node) =>
|
||||
executedNodes.includes(node)
|
||||
)
|
||||
return {
|
||||
outcome: ranEveryExpected ? 'PASS' : 'PARTIAL',
|
||||
executedNodes,
|
||||
outputsByNode
|
||||
}
|
||||
}
|
||||
293
browser_tests/fixtures/customNode/typePairing.ts
Normal file
293
browser_tests/fixtures/customNode/typePairing.ts
Normal file
@@ -0,0 +1,293 @@
|
||||
// Type-driven pairing generator for the connectivity (contract) tier.
|
||||
// Wildcard `*` slots are excluded from pairing: LiteGraph.isValidConnection
|
||||
// short-circuits on `*` before the real type compare, so a wildcard link
|
||||
// proves reachability, not type interop.
|
||||
|
||||
export interface RawNodeDef {
|
||||
input?: {
|
||||
required?: Record<string, unknown>
|
||||
optional?: Record<string, unknown>
|
||||
}
|
||||
output?: unknown[]
|
||||
output_name?: string[]
|
||||
python_module?: string
|
||||
}
|
||||
|
||||
interface NormalizedSlot {
|
||||
name: string
|
||||
type: string
|
||||
// COMBO slots: the literal option list, for same-vocabulary pairing.
|
||||
comboOptions?: unknown[]
|
||||
}
|
||||
|
||||
export interface NormalizedNode {
|
||||
type: string
|
||||
pack: string
|
||||
inputs: NormalizedSlot[]
|
||||
outputs: NormalizedSlot[]
|
||||
// Slots whose raw spec carried no recognizable type (slotTypeOf null):
|
||||
// recorded so a schema change can never silently shrink the corpus.
|
||||
unknownSlots?: string[]
|
||||
}
|
||||
|
||||
interface SlotRef {
|
||||
nodeType: string
|
||||
pack: string
|
||||
slotName: string
|
||||
slotType: string
|
||||
}
|
||||
|
||||
export interface PlannedPair {
|
||||
producer: SlotRef
|
||||
consumer: SlotRef
|
||||
}
|
||||
|
||||
export interface PairingPlan {
|
||||
pairs: PlannedPair[]
|
||||
// No compatible partner in the loaded corpus: a health signal, not a failure.
|
||||
orphans: Array<SlotRef & { dir: 'in' | 'out' }>
|
||||
// `*` / empty-typed slots, excluded by design (false confidence).
|
||||
wildcards: Array<SlotRef & { dir: 'in' | 'out' }>
|
||||
// COMBO slots with no same-vocabulary partner in the corpus, excluded:
|
||||
// isValidConnection only compares the string COMBO while each slot carries
|
||||
// its own option set, so pairing across different vocabularies proves
|
||||
// nothing (a checkpoint dropdown would "connect" to a scheduler dropdown).
|
||||
// Combos whose option lists match exactly ARE paired like any other type.
|
||||
combos: Array<SlotRef & { dir: 'in' | 'out' }>
|
||||
// Slots dropped at normalize time because their raw spec had no
|
||||
// recognizable type - surfaced here (and logged by the sweep) so a
|
||||
// backend or pack schema change cannot silently shrink the corpus.
|
||||
unknownShapes: string[]
|
||||
}
|
||||
|
||||
// Extends the shared outcome taxonomy (runResult.ts); ORPHAN_TYPE is a
|
||||
// plan-time skip so it never reaches the executor.
|
||||
// WIDGET_ONLY_ON_INSTANCE: the pack's own frontend JS rebuilt a declared
|
||||
// input as a widget-only control, so there is no socket to wire - excluded
|
||||
// like wildcards, never a failure and never a silent pass.
|
||||
export type ConnectivityOutcome =
|
||||
| 'PASS'
|
||||
| 'CONNECT_REJECTED'
|
||||
| 'ROUNDTRIP_LOST'
|
||||
| 'SLOT_CONTRACT_MISMATCH'
|
||||
| 'WIDGET_ONLY_ON_INSTANCE'
|
||||
|
||||
export function packOf(pythonModule: string | undefined): string {
|
||||
if (pythonModule?.startsWith('custom_nodes.'))
|
||||
return pythonModule.slice('custom_nodes.'.length)
|
||||
return 'core'
|
||||
}
|
||||
|
||||
export function isWildcard(type: string): boolean {
|
||||
return type === '' || type === '*'
|
||||
}
|
||||
|
||||
// COMBO list literals are arrays; their connectable socket type is COMBO.
|
||||
function slotTypeOf(rawType: unknown): string | null {
|
||||
if (Array.isArray(rawType)) return 'COMBO'
|
||||
return typeof rawType === 'string' ? rawType : null
|
||||
}
|
||||
|
||||
function inputSlots(
|
||||
entries: Record<string, unknown> | undefined,
|
||||
unknown: string[]
|
||||
): NormalizedSlot[] {
|
||||
if (!entries) return []
|
||||
const slots: NormalizedSlot[] = []
|
||||
for (const [name, spec] of Object.entries(entries)) {
|
||||
const specArray = Array.isArray(spec) ? spec : [spec]
|
||||
const type = slotTypeOf(specArray[0])
|
||||
if (type === null) {
|
||||
unknown.push(name)
|
||||
continue
|
||||
}
|
||||
const opts = specArray[1] as
|
||||
| { socketless?: boolean; options?: unknown }
|
||||
| undefined
|
||||
// socketless = widget only, no slot: not connectable, out of the matrix.
|
||||
if (opts?.socketless) continue
|
||||
if (type === 'COMBO') {
|
||||
// Raw defs carry the option list as the type literal; the frontend's
|
||||
// transformed defs use the string 'COMBO' with options in the opts.
|
||||
const options = Array.isArray(specArray[0])
|
||||
? (specArray[0] as unknown[])
|
||||
: Array.isArray(opts?.options)
|
||||
? opts.options
|
||||
: undefined
|
||||
slots.push({ name, type, comboOptions: options })
|
||||
continue
|
||||
}
|
||||
slots.push({ name, type })
|
||||
}
|
||||
return slots
|
||||
}
|
||||
|
||||
export function normalizeNodeDefs(
|
||||
defs: Record<string, RawNodeDef>
|
||||
): NormalizedNode[] {
|
||||
return Object.entries(defs).map(([type, def]) => {
|
||||
const unknown: string[] = []
|
||||
const node: NormalizedNode = {
|
||||
type,
|
||||
pack: packOf(def.python_module),
|
||||
inputs: [
|
||||
...inputSlots(def.input?.required, unknown),
|
||||
...inputSlots(def.input?.optional, unknown)
|
||||
],
|
||||
outputs: (def.output ?? []).flatMap((rawType, index) => {
|
||||
const slotType = slotTypeOf(rawType)
|
||||
if (slotType === null) {
|
||||
unknown.push(`output[${index}]`)
|
||||
return []
|
||||
}
|
||||
// output_name entries can be non-strings (COMBO literals repeat the
|
||||
// option array); the slot name must stay a string.
|
||||
const rawName = def.output_name?.[index]
|
||||
const slot: NormalizedSlot = {
|
||||
name: typeof rawName === 'string' ? rawName : slotType,
|
||||
type: slotType
|
||||
}
|
||||
if (slotType === 'COMBO') slot.comboOptions = rawType as unknown[]
|
||||
return [slot]
|
||||
})
|
||||
}
|
||||
if (unknown.length > 0) node.unknownSlots = unknown
|
||||
return node
|
||||
})
|
||||
}
|
||||
|
||||
// Faithful mirror of LiteGraph.isValidConnection (LiteGraphGlobal.ts):
|
||||
// wildcard/empty always match, comparison is case-insensitive, comma-unions
|
||||
// match if any member pair matches. The live sweep still connects through the
|
||||
// REAL validator, so any drift here surfaces as CONNECT_REJECTED, not a
|
||||
// silent false green.
|
||||
export function isTypeCompatible(a: string, b: string): boolean {
|
||||
if (isWildcard(a) || isWildcard(b)) return true
|
||||
const typeA = a.toLowerCase()
|
||||
const typeB = b.toLowerCase()
|
||||
if (typeA === typeB) return true
|
||||
if (!typeA.includes(',') && !typeB.includes(',')) return false
|
||||
return typeA
|
||||
.split(',')
|
||||
.some((memberA) =>
|
||||
typeB.split(',').some((memberB) => isTypeCompatible(memberA, memberB))
|
||||
)
|
||||
}
|
||||
|
||||
function slotRef(node: NormalizedNode, slot: NormalizedSlot): SlotRef {
|
||||
return {
|
||||
nodeType: node.type,
|
||||
pack: node.pack,
|
||||
slotName: slot.name,
|
||||
slotType: slot.type
|
||||
}
|
||||
}
|
||||
|
||||
// One representative compatible edge per slot, deterministically the first
|
||||
// partner in (nodeType, slotName) order. This bounds cost to O(slots) but
|
||||
// does NOT prove every pair; a full cross-product is an opt-in deep mode.
|
||||
export function planPairs(
|
||||
all: NormalizedNode[],
|
||||
corpusTypes: string[]
|
||||
): PairingPlan {
|
||||
const sorted = [...all].sort((a, b) => a.type.localeCompare(b.type))
|
||||
const pairable = (slot: NormalizedSlot) =>
|
||||
!isWildcard(slot.type) && slot.type !== 'COMBO'
|
||||
const producers: Array<SlotRef> = sorted.flatMap((node) =>
|
||||
node.outputs.filter(pairable).map((slot) => slotRef(node, slot))
|
||||
)
|
||||
const consumers: Array<SlotRef> = sorted.flatMap((node) =>
|
||||
node.inputs.filter(pairable).map((slot) => slotRef(node, slot))
|
||||
)
|
||||
// COMBO slots pair only on an identical option vocabulary; the string type
|
||||
// alone would let a checkpoint dropdown "connect" to a scheduler dropdown.
|
||||
// Vocabulary equality is a SET comparison: a wired input bypasses its own
|
||||
// widget, so menu order and the options[0] default do not participate in
|
||||
// the wire contract - only membership does (backend validation checks
|
||||
// "value in options"). Values still compare as exact strings.
|
||||
const vocabOf = (slot: NormalizedSlot) =>
|
||||
JSON.stringify(
|
||||
(slot.comboOptions ?? []).map((option) => JSON.stringify(option)).sort()
|
||||
)
|
||||
// A combo whose option list is unknown (transformed defs without an
|
||||
// options array) must never pair - a blind match would wire dropdowns
|
||||
// with no vocabulary evidence at all.
|
||||
const comboProducers = sorted.flatMap((node) =>
|
||||
node.outputs
|
||||
.filter(
|
||||
(slot) => slot.type === 'COMBO' && Array.isArray(slot.comboOptions)
|
||||
)
|
||||
.map((slot) => ({ ref: slotRef(node, slot), vocab: vocabOf(slot) }))
|
||||
)
|
||||
const comboConsumers = sorted.flatMap((node) =>
|
||||
node.inputs
|
||||
.filter(
|
||||
(slot) => slot.type === 'COMBO' && Array.isArray(slot.comboOptions)
|
||||
)
|
||||
.map((slot) => ({ ref: slotRef(node, slot), vocab: vocabOf(slot) }))
|
||||
)
|
||||
|
||||
const plan: PairingPlan = {
|
||||
pairs: [],
|
||||
orphans: [],
|
||||
wildcards: [],
|
||||
combos: [],
|
||||
unknownShapes: all.flatMap((node) =>
|
||||
(node.unknownSlots ?? []).map((slot) => `${node.type}.${slot}`)
|
||||
)
|
||||
}
|
||||
const seen = new Set<string>()
|
||||
const addPair = (producer: SlotRef, consumer: SlotRef) => {
|
||||
const key = `${producer.nodeType}.${producer.slotName}->${consumer.nodeType}.${consumer.slotName}`
|
||||
if (seen.has(key)) return
|
||||
seen.add(key)
|
||||
plan.pairs.push({ producer, consumer })
|
||||
}
|
||||
|
||||
const corpus = all.filter((node) => corpusTypes.includes(node.type))
|
||||
for (const node of corpus) {
|
||||
for (const slot of node.inputs) {
|
||||
if (isWildcard(slot.type)) {
|
||||
plan.wildcards.push({ ...slotRef(node, slot), dir: 'in' })
|
||||
continue
|
||||
}
|
||||
if (slot.type === 'COMBO') {
|
||||
const producer = Array.isArray(slot.comboOptions)
|
||||
? comboProducers.find(
|
||||
(candidate) => candidate.vocab === vocabOf(slot)
|
||||
)
|
||||
: undefined
|
||||
if (producer) addPair(producer.ref, slotRef(node, slot))
|
||||
else plan.combos.push({ ...slotRef(node, slot), dir: 'in' })
|
||||
continue
|
||||
}
|
||||
const producer = producers.find((candidate) =>
|
||||
isTypeCompatible(candidate.slotType, slot.type)
|
||||
)
|
||||
if (producer) addPair(producer, slotRef(node, slot))
|
||||
else plan.orphans.push({ ...slotRef(node, slot), dir: 'in' })
|
||||
}
|
||||
for (const slot of node.outputs) {
|
||||
if (isWildcard(slot.type)) {
|
||||
plan.wildcards.push({ ...slotRef(node, slot), dir: 'out' })
|
||||
continue
|
||||
}
|
||||
if (slot.type === 'COMBO') {
|
||||
const consumer = Array.isArray(slot.comboOptions)
|
||||
? comboConsumers.find(
|
||||
(candidate) => candidate.vocab === vocabOf(slot)
|
||||
)
|
||||
: undefined
|
||||
if (consumer) addPair(slotRef(node, slot), consumer.ref)
|
||||
else plan.combos.push({ ...slotRef(node, slot), dir: 'out' })
|
||||
continue
|
||||
}
|
||||
const consumer = consumers.find((candidate) =>
|
||||
isTypeCompatible(slot.type, candidate.slotType)
|
||||
)
|
||||
if (consumer) addPair(slotRef(node, slot), consumer)
|
||||
else plan.orphans.push({ ...slotRef(node, slot), dir: 'out' })
|
||||
}
|
||||
}
|
||||
return plan
|
||||
}
|
||||
151
browser_tests/fixtures/data/customNodeManifest.json
Normal file
151
browser_tests/fixtures/data/customNodeManifest.json
Normal file
@@ -0,0 +1,151 @@
|
||||
[
|
||||
{
|
||||
"pack": "ComfyUI-Impact-Pack",
|
||||
"repo": "https://github.com/ltdrdata/ComfyUI-Impact-Pack",
|
||||
"pin": "429d0159ad429e64d2b3916e6e7be9c22d025c3c",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/impact_primitives_run.json",
|
||||
"expectedNodes": ["ImpactInt", "ImpactFloat"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 30000,
|
||||
"cannotRunAlone": [
|
||||
"AnyPipeToBasic",
|
||||
"CLIPSegDetectorProvider",
|
||||
"ImpactMakeImageBatch",
|
||||
"ImpactMakeMaskBatch",
|
||||
"LatentSender",
|
||||
"MasksToMaskList",
|
||||
"PreviewBridgeLatent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"pack": "ComfyUI-VideoHelperSuite",
|
||||
"repo": "https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite",
|
||||
"pin": "4ee72c065db22c9d96c2427954dc69e7b908444b",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/vhs_video_pipeline_run.json",
|
||||
"expectedNodes": ["VHS_LoadVideoPath", "VHS_VideoInfo"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 90000,
|
||||
"cannotRunAlone": [
|
||||
"VHS_LoadAudio",
|
||||
"VHS_LoadImagePath",
|
||||
"VHS_LoadImages",
|
||||
"VHS_LoadImagesPath",
|
||||
"VHS_LoadVideoFFmpegPath",
|
||||
"VHS_LoadVideoPath"
|
||||
]
|
||||
},
|
||||
{
|
||||
"pack": "rgthree-comfy",
|
||||
"repo": "https://github.com/rgthree/rgthree-comfy",
|
||||
"pin": "27b4f4cdcf3b127c29d5d8135ac1536ecbd4c383",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/rgthree_seed_display_run.json",
|
||||
"expectedNodes": ["Seed (rgthree)", "Display Any (rgthree)"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 30000,
|
||||
"cannotRunAlone": ["Image or Latent Size (rgthree)"]
|
||||
},
|
||||
{
|
||||
"pack": "ComfyUI_essentials",
|
||||
"repo": "https://github.com/cubiq/ComfyUI_essentials",
|
||||
"pin": "9d9f4bedfc9f0321c19faf71855e228c93bd0dc9",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/essentials_math_display_run.json",
|
||||
"expectedNodes": ["SimpleMathInt+", "DisplayAny"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 30000,
|
||||
"cannotRunAlone": [
|
||||
"ImageApplyLUT+",
|
||||
"ImageUntile+",
|
||||
"MaskFromList+",
|
||||
"PixelOEPixelize+",
|
||||
"SimpleCondition+",
|
||||
"SimpleMath+",
|
||||
"SimpleMathCondition+",
|
||||
"SimpleMathDual+"
|
||||
]
|
||||
},
|
||||
{
|
||||
"pack": "ComfyUI-KJNodes",
|
||||
"repo": "https://github.com/kijai/ComfyUI-KJNodes",
|
||||
"pin": "e27a505b3ba6ce42687fe00500deda103d9d6071",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/kjnodes_constants_run.json",
|
||||
"expectedNodes": ["INTConstant", "FloatConstant"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 30000,
|
||||
"cannotRunAlone": [
|
||||
"CameraPoseVisualizer",
|
||||
"CreateAudioMask",
|
||||
"CreateGradientFromCoords",
|
||||
"CreateInstanceDiffusionTracking",
|
||||
"CreateShapeImageOnPath",
|
||||
"CreateShapeMaskOnPath",
|
||||
"CreateTextOnPath",
|
||||
"CrossFadeImages",
|
||||
"CrossFadeImagesMulti",
|
||||
"CustomControlNetWeightsFluxFromList",
|
||||
"CutAndDragOnPath",
|
||||
"EndRecordCUDAMemoryHistory",
|
||||
"FloatToMask",
|
||||
"GetImagesFromBatchIndexed",
|
||||
"GetLatentsFromBatchIndexed",
|
||||
"ImageAndMaskPreview",
|
||||
"ImageGridtoBatch",
|
||||
"ImagePadForOutpaintTargetSize",
|
||||
"InterpolateCoords",
|
||||
"LoadImagesFromFolderKJ",
|
||||
"LoadVideosFromFolder",
|
||||
"PlotCoordinates",
|
||||
"StartRecordCUDAMemoryHistory",
|
||||
"Superprompt",
|
||||
"VisualizeCUDAMemoryHistory",
|
||||
"WebcamCaptureCV2",
|
||||
"WeightScheduleConvert",
|
||||
"WeightScheduleExtend",
|
||||
"WidgetToString"
|
||||
]
|
||||
},
|
||||
{
|
||||
"pack": "ComfyUI-Custom-Scripts",
|
||||
"repo": "https://github.com/pythongosssss/ComfyUI-Custom-Scripts",
|
||||
"pin": "609f3afaa74b2f88ef9ce8d939626065e3247469",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/customscripts_string_show_run.json",
|
||||
"expectedNodes": ["StringFunction|pysssss", "ShowText|pysssss"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 30000,
|
||||
"cannotRunAlone": ["MathExpression|pysssss"]
|
||||
},
|
||||
{
|
||||
"pack": "was-node-suite-comfyui",
|
||||
"repo": "https://github.com/WASasquatch/was-node-suite-comfyui",
|
||||
"pin": "ea935d1044ae5a26efa54ebeb18fe9020af49a45",
|
||||
"tiers": ["load", "connectivity", "run"],
|
||||
"workflow": "assets/customNodes/was_number_text_run.json",
|
||||
"expectedNodes": ["Constant Number", "Number to Text", "Text to Console"],
|
||||
"requiresGpu": false,
|
||||
"requiresModels": [],
|
||||
"timeoutMs": 30000,
|
||||
"cannotRunAlone": [
|
||||
"Bus Node",
|
||||
"Diffusers Hub Model Down-Loader",
|
||||
"Image Aspect Ratio",
|
||||
"Image Batch",
|
||||
"Image Send HTTP",
|
||||
"Latent Batch",
|
||||
"Mask Batch",
|
||||
"Samples Passthrough (Stat System)",
|
||||
"Text Dictionary Convert",
|
||||
"Text to Number"
|
||||
]
|
||||
}
|
||||
]
|
||||
28
browser_tests/fixtures/utils/consoleErrorCollector.ts
Normal file
28
browser_tests/fixtures/utils/consoleErrorCollector.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import type { ConsoleMessage, Page } from '@playwright/test'
|
||||
|
||||
export function collectConsoleErrors(page: Page): {
|
||||
errors: string[]
|
||||
stop: () => void
|
||||
} {
|
||||
const errors: string[] = []
|
||||
const listener = (message: ConsoleMessage) => {
|
||||
if (message.type() !== 'error') return
|
||||
const url = message.location().url
|
||||
errors.push(url ? `${message.text()} [${url}]` : message.text())
|
||||
}
|
||||
// Uncaught page exceptions and unhandled promise rejections never reach
|
||||
// console.error; Chromium surfaces both through pageerror. Without this
|
||||
// listener a pack script crashing outside a console call passes silently.
|
||||
const pageErrorListener = (error: Error) => {
|
||||
errors.push(`Uncaught page error: ${error.message}`)
|
||||
}
|
||||
page.on('console', listener)
|
||||
page.on('pageerror', pageErrorListener)
|
||||
return {
|
||||
errors,
|
||||
stop: () => {
|
||||
page.off('console', listener)
|
||||
page.off('pageerror', pageErrorListener)
|
||||
}
|
||||
}
|
||||
}
|
||||
71
browser_tests/fixtures/utils/customNodeSuite.ts
Normal file
71
browser_tests/fixtures/utils/customNodeSuite.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import type { Page } from '@playwright/test'
|
||||
|
||||
import type { ComfyPage } from '@e2e/fixtures/ComfyPage'
|
||||
import { TestIds } from '@e2e/fixtures/selectors'
|
||||
|
||||
// Boot every session with a blank graph (loadBlankWorkflow) instead of the
|
||||
// bundled default template, whose model references error on a model-less
|
||||
// harness backend and would trip the zero-visible-errors invariant. The
|
||||
// backend must run --multi-user (the repo-wide prerequisite for browser
|
||||
// tests): the fixture then writes these settings to the same per-worker
|
||||
// user the session reads, on CI and locally alike.
|
||||
// The shared fixture disables the errors tab to hide missing-model
|
||||
// indicators in unrelated suites; this suite exists to SEE errors, so every
|
||||
// error surface stays live.
|
||||
export const customNodeSuiteSettings = {
|
||||
'Comfy.TutorialCompleted': false,
|
||||
'Comfy.RightSidePanel.ShowErrorsTab': true
|
||||
}
|
||||
|
||||
// The tutorial path auto-opens the templates browser over the blank graph.
|
||||
// Dismiss it deterministically so no window ever shows unexpected UI.
|
||||
export async function dismissTemplatesDialog(
|
||||
comfyPage: ComfyPage
|
||||
): Promise<void> {
|
||||
const templates = comfyPage.page.getByTestId(TestIds.templates.content)
|
||||
await templates.waitFor({ state: 'visible' })
|
||||
await comfyPage.page.keyboard.press('Escape')
|
||||
await templates.waitFor({ state: 'hidden' })
|
||||
}
|
||||
|
||||
// Every test gets a fresh page, but they share ONE backend. An execution
|
||||
// tier that ends while a prompt is still draining leaves that work running
|
||||
// on the shared backend; the next test's fresh page connects mid-execution
|
||||
// and catches its async error events (console noise, a popped error dialog)
|
||||
// or its still-running prompt (queue-busy). Draining to idle in an afterEach
|
||||
// - while the finishing test's own page is still open, so any late events
|
||||
// land there - is what makes each test unable to affect the next. getQueue
|
||||
// swallows a failed fetch and returns an empty queue, so throw-on-error and
|
||||
// treat a failed read as still-busy; the wait is free when already idle
|
||||
// (one getQueue round-trip), so a healthy suite pays ~nothing for it.
|
||||
// Returns 0 when the backend reached idle, 1 when it was still busy after the
|
||||
// budget (a genuinely wedged, non-interruptible execution). The afterEach hook
|
||||
// ignores the result; the auto-run tier asserts on it.
|
||||
export async function drainBackendToIdle(
|
||||
page: Page,
|
||||
budgetMs = 150_000
|
||||
): Promise<number> {
|
||||
const depth = () =>
|
||||
page.evaluate(async () => {
|
||||
try {
|
||||
const queue = await window.app!.api.getQueue({ throwOnError: true })
|
||||
return queue.Running.length + queue.Pending.length
|
||||
} catch {
|
||||
return Number.POSITIVE_INFINITY
|
||||
}
|
||||
})
|
||||
if ((await depth()) === 0) return 0
|
||||
await page.evaluate(async () => {
|
||||
await window.app!.api.interrupt(null)
|
||||
await window.app!.api.clearItems('queue')
|
||||
})
|
||||
const deadline = Date.now() + budgetMs
|
||||
let remaining = await depth()
|
||||
while (remaining !== 0 && Date.now() < deadline) {
|
||||
await page.evaluate(
|
||||
() => new Promise((resolve) => setTimeout(resolve, 500))
|
||||
)
|
||||
remaining = await depth()
|
||||
}
|
||||
return remaining === 0 ? 0 : 1
|
||||
}
|
||||
16
browser_tests/fixtures/utils/errorSurfaces.ts
Normal file
16
browser_tests/fixtures/utils/errorSurfaces.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import type { Locator, Page } from '@playwright/test'
|
||||
|
||||
import { TestIds } from '@e2e/fixtures/selectors'
|
||||
|
||||
// The app's user-visible error surfaces. A regression run is green only if a
|
||||
// human looking at the screen would see zero errors - not merely a clean
|
||||
// console. The harness self-check asserts the overlay IS visible after a
|
||||
// forced execution error, so these selectors are permanently proven live.
|
||||
export function errorSurfaces(page: Page): Record<string, Locator> {
|
||||
return {
|
||||
errorOverlay: page.getByTestId(TestIds.dialogs.errorOverlay),
|
||||
errorDialog: page.getByTestId(TestIds.dialogs.errorDialog),
|
||||
nodeRenderErrors: page.locator('.node-error'),
|
||||
errorToasts: page.locator('.p-toast-message-error')
|
||||
}
|
||||
}
|
||||
@@ -119,6 +119,11 @@ class NodeSlotReference {
|
||||
const rawPos = node.getConnectionPos(type === 'input', index)
|
||||
const convertedPos =
|
||||
window.app!.canvas.ds!.convertOffsetToCanvas(rawPos)
|
||||
// page.mouse needs page coords; pack JS can inject chrome above the
|
||||
// canvas (rgthree's progress bar), shifting it off (0,0).
|
||||
const rect = window.app!.canvas.canvas.getBoundingClientRect()
|
||||
convertedPos[0] += rect.left
|
||||
convertedPos[1] += rect.top
|
||||
|
||||
// Debug logging - convert Float64Arrays to regular arrays for visibility
|
||||
console.warn(
|
||||
|
||||
397
browser_tests/tests/customNodes/ADDING_CUSTOM_NODES.md
Normal file
397
browser_tests/tests/customNodes/ADDING_CUSTOM_NODES.md
Normal file
@@ -0,0 +1,397 @@
|
||||
# Adding a custom-node pack to the regression suite
|
||||
|
||||
The authoritative, step-by-step process for onboarding a new pack. Written to
|
||||
be followable by a human or an agent with no prior context. The suite itself
|
||||
(what it asserts, how to run it) is documented in [README.md](README.md),
|
||||
and its system design in [ARCHITECTURE.md](ARCHITECTURE.md); this file is
|
||||
only about adding coverage for a new pack.
|
||||
|
||||
The short version: install the pack on a local test backend, read the pack's
|
||||
real node keys out of `/object_info`, author one small model-free workflow,
|
||||
add one row to the manifest, prove it green locally, push. No new test code
|
||||
is ever needed - the specs iterate the manifest.
|
||||
|
||||
## What a manifest row buys you (the tiers)
|
||||
|
||||
Adding the one row enrolls the pack in two kinds of coverage:
|
||||
|
||||
- **Every-node tiers (automatic, zero configuration).** The suite reads the
|
||||
pack's FULL node list from the live backend and, for every registered
|
||||
node: mounts it in both renderers and asserts under EACH renderer that the
|
||||
instance materializes everything its def declares - every non-socketless
|
||||
input exists as a widget or a socket (autogrow templates count via their
|
||||
expansion slots) and every declared output exists; the Vue pass
|
||||
additionally asserts the DOM renders at least the instance's widget and
|
||||
slot counts - a mount with missing controls fails. It then round-trips
|
||||
every node through save/reload (every widget
|
||||
is first written with a non-default value that must stick, and the
|
||||
serialized `widgets_values` must survive configure unchanged), plans typed
|
||||
connections for all its concrete slots (COMBO slots pair when they offer
|
||||
the same option SET - order-insensitive, since a wired input bypasses its
|
||||
own widget and only membership matters), and executes it for real when it
|
||||
can run:
|
||||
either self-sufficient (every required input is a widget with a valid
|
||||
default) or `CHAINABLE` - every required socket type has a model-free
|
||||
producer (`EmptyImage`, `EmptyLatentImage`, `SolidMask`, `Primitive*`,
|
||||
`EmptyAudio`, ...) that the runner synthesizes and wires automatically.
|
||||
Executed nodes must observably produce: the `PreviewAny` sink wired to the
|
||||
node's first output must emit a ui payload, or the node is its own
|
||||
terminus (`OUTPUT_NODE`). Nodes that cannot run are classified and
|
||||
logged, never silently dropped: `NEEDS_WIRES` (a required socket type has
|
||||
no model-free producer - MODEL, SEGS, CONDITIONING...), `NEEDS_MODELS`
|
||||
(empty model/file combo on the bare backend), `NO_OBSERVABLE_OUTPUT`
|
||||
(nothing observable to queue), or "rejected at validation on defaults"
|
||||
(needs a curated fixture).
|
||||
- **Curated tiers (the row's fields).** `expectedNodes` + `workflow` drive
|
||||
the hand-authored run-tier chain (Step 4) proving a real multi-node
|
||||
wiring executes end to end, and serve as must-exist sentinels.
|
||||
|
||||
Every-node coverage means a pack update is tested the moment CI installs
|
||||
it - including nodes you never listed.
|
||||
|
||||
## Step 0 - prerequisites
|
||||
|
||||
- A local test backend and dev server set up exactly per the
|
||||
[README prerequisites](README.md#prerequisites). Do not skip `--multi-user`
|
||||
or `--cache-none`.
|
||||
- The pack's GitHub URL. The CI job clones and pip-installs it, so the repo
|
||||
must be public and its `requirements.txt` must install on a CPU-only
|
||||
runner. Packs that hard-require CUDA at import time cannot be onboarded
|
||||
until they guard that import.
|
||||
|
||||
## Step 1 - install the pack on the test backend
|
||||
|
||||
```bash
|
||||
cd <test-backend>/custom_nodes
|
||||
git clone https://github.com/<owner>/<pack>
|
||||
pip install -r <pack>/requirements.txt # if the pack has one
|
||||
```
|
||||
|
||||
The clone directory name must equal the manifest `pack` key: node
|
||||
attribution keys on that directory via `python_module`, and CI installs
|
||||
into `custom_nodes/<pack>` for the same reason.
|
||||
|
||||
If you run a CPU-only backend, constrain pip so the pack cannot swap in a
|
||||
different torch (CI does the same):
|
||||
|
||||
```bash
|
||||
pip freeze | grep -iE '^(torch|torchvision|torchaudio)==' > /tmp/torch-constraints.txt
|
||||
pip install -r <pack>/requirements.txt -c /tmp/torch-constraints.txt
|
||||
```
|
||||
|
||||
Restart the backend and check its log: the `Import times for custom nodes`
|
||||
block must list the pack with no `IMPORT FAILED` marker. An import failure is
|
||||
a pack bug or a missing dependency - fix that first; nothing downstream can
|
||||
work without a clean import.
|
||||
|
||||
While you are here, note whether the pack ships frontend JS:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8288/extensions | python3 -c '
|
||||
import json, sys
|
||||
print(sum(1 for p in json.load(sys.stdin) if p.startswith("/extensions/<pack-dir-name>/")))
|
||||
'
|
||||
```
|
||||
|
||||
Non-zero means the pack patches the frontend at runtime (restyled nodes,
|
||||
rebuilt widgets, injected page chrome). Write that down - it decides whether
|
||||
Step 6 needs the CI-parity run. Both "green locally, red on CI" failures in
|
||||
the first 5-pack onboarding came from exactly this.
|
||||
|
||||
## Step 2 - read the pack's real node keys
|
||||
|
||||
The manifest's `expectedNodes` are the pack's `object_info` keys (the same
|
||||
strings the API uses as `class_type`). They are NOT Python class names and
|
||||
NOT display names. Get them from the running backend:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8288/object_info | python3 -c '
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
for key, node in sorted(d.items()):
|
||||
if node.get("python_module") == "custom_nodes.<pack-dir-name>":
|
||||
print(key)
|
||||
'
|
||||
```
|
||||
|
||||
Real traps this step catches (each one shipped in a real pack):
|
||||
|
||||
| Pack | Correct key | Wrong guesses that look right |
|
||||
| ---------------------- | ------------------- | ------------------------------------------------------------------------------- |
|
||||
| ComfyUI_essentials | `SimpleMathInt+` | `SimpleMathInt` (keys carry a trailing `+`, except `DisplayAny` which has none) |
|
||||
| ComfyUI-KJNodes | `INTConstant` | `INT Constant` (that is the display name) |
|
||||
| ComfyUI-Custom-Scripts | `ShowText\|pysssss` | `ShowText` (keys carry a `\|pysssss` suffix) |
|
||||
| rgthree-comfy | `Seed (rgthree)` | `RgthreeSeed` (the Python class name) |
|
||||
|
||||
## Step 3 - pick the expected nodes
|
||||
|
||||
Choose 2-3 nodes that are:
|
||||
|
||||
- **Model-free**: no checkpoint / VAE / CLIP inputs, no file downloads. The
|
||||
gate runs on CPU with no models installed. Constants, math, text, and
|
||||
display nodes are ideal.
|
||||
- **Wireable into a chain**: at least one producer (has a typed output) and
|
||||
one terminal node. A terminal node either has `output_node: true` in
|
||||
`/object_info` (it terminates a workflow by itself) or you end the chain in
|
||||
the core `PreviewAny` node, which accepts any type.
|
||||
|
||||
Check a candidate's inputs, outputs, and `output_node` flag:
|
||||
|
||||
```bash
|
||||
curl -s http://127.0.0.1:8288/object_info | python3 -c '
|
||||
import json, sys
|
||||
node = json.load(sys.stdin)["<exact key>"]
|
||||
print(json.dumps({k: node[k] for k in ("input", "output", "output_name", "output_node")}, indent=1))
|
||||
'
|
||||
```
|
||||
|
||||
Every node you list in `expectedNodes` must appear in the run workflow: the
|
||||
run tier asserts each one actually executes on the backend.
|
||||
|
||||
## Step 4 - author the run-tier workflow
|
||||
|
||||
Add one JSON file under `browser_tests/assets/customNodes/`, named
|
||||
`<pack>_<what it does>_run.json`. Copy an existing asset as the template
|
||||
(`rgthree_seed_display_run.json` is the simplest two-node example;
|
||||
`was_number_text_run.json` shows a 3-node chain). It is the frontend
|
||||
workflow format, hand-authorable:
|
||||
|
||||
- `nodes[].type` is the exact `object_info` key from Step 2.
|
||||
- `widgets_values` is an array in the node's widget order: the `input`
|
||||
entries from `/object_info` in declaration order (`required` first, then
|
||||
`optional`), keeping only widget-type inputs (INT, FLOAT, STRING, BOOLEAN,
|
||||
and combo lists) and skipping any input whose options say
|
||||
`"forceInput": true` (those are sockets, never widgets). A required input
|
||||
that is neither a widget type nor `forceInput` (a custom type like
|
||||
`NUMBER`) is also a socket: wire a link into it or the run fails on a
|
||||
missing required input.
|
||||
- A link is one row in `links`: `[link_id, from_node_id, from_slot,
|
||||
to_node_id, to_slot, "TYPE"]`, plus the matching `link`/`links` ids on the
|
||||
two nodes' `inputs`/`outputs` entries.
|
||||
- To wire INTO an input that would normally be a widget (no `forceInput`),
|
||||
the input entry also needs a `"widget": { "name": "<input name>" }` key -
|
||||
see `browser_tests/assets/vueNodes/linked-int-widget.json`.
|
||||
- Keep it tiny. Two to four nodes proving "this pack executes" is the whole
|
||||
job; feature-depth testing belongs to the pack's own repo.
|
||||
- If the workflow needs a media file, reuse something already under
|
||||
`browser_tests/assets/` (e.g. `plain_video.mp4`) - never commit new binary
|
||||
assets. CI stages `plain_video.mp4` into the backend's `input/` dir; if
|
||||
your workflow needs a different existing asset staged, extend the
|
||||
`Stage run-tier assets` step in
|
||||
`.github/workflows/ci-tests-custom-nodes.yaml`.
|
||||
- A media path in the workflow (e.g. `input/plain_video.mp4`) resolves
|
||||
against the backend process's working directory, not the repo. Locally,
|
||||
copy the file into the `input/` dir of the directory you launched
|
||||
`main.py` from, or the run tier fails validation with
|
||||
`Invalid file path` and the test reports `TIMEOUT`.
|
||||
|
||||
## Step 5 - add the manifest row
|
||||
|
||||
Append one object to `browser_tests/fixtures/data/customNodeManifest.json`:
|
||||
|
||||
| Field | Meaning |
|
||||
| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `pack` | The pack's directory name under `custom_nodes/` (what `git clone` creates). |
|
||||
| `repo` | The GitHub URL CI clones. Required non-empty. |
|
||||
| `pin` | Required: the full 40-char commit SHA you verified locally. The manifest loader rejects anything else at load and CI fails before install (empty is accepted only under `CUSTOM_NODES_ALLOW_UNPINNED=1`, reserved for the planned pack-HEAD canary). CI checks it out after cloning, so the gate tests exactly what you tested. Bump deliberately, re-verifying per this doc. |
|
||||
| `tiers` | Tier gates: `connectivity` (typed links + slot drags) and `run` (executes the workflow) enable their tiers; `load` is descriptive only - the register+render pass runs for every row regardless. Keep all three unless a tier is impossible for the pack. |
|
||||
| `workflow` | Path relative to `browser_tests/` of the Step 4 file. `""` only while the pack has no `run` tier. |
|
||||
| `expectedNodes` | The Step 2/3 keys. The load tier mounts each in both renderers; the run tier asserts each executes. |
|
||||
| `requiresGpu` | `true` only if execution genuinely needs CUDA. Such packs cannot use the `run` tier on the CPU gate. |
|
||||
| `requiresModels` | Model files the workflow needs (`[]` for the packs onboarded so far - keep it that way whenever possible). |
|
||||
| `timeoutMs` | Per-test budget. `30000` unless the workflow does real work (video decode uses `90000`). |
|
||||
| `vueNodesCompatible` | Optional, default `true`. See the policy below. Only ever set `false`, and only with evidence. |
|
||||
|
||||
`loadManifest()` (`browser_tests/fixtures/customNode/manifest.ts`) validates
|
||||
every row and fails loudly on a missing field, an empty `repo`, a misspelled
|
||||
tier, or a `run` tier with an empty `workflow`.
|
||||
|
||||
## Step 6 - prove it green locally, in both environments
|
||||
|
||||
### 6a - fast loop (dev server)
|
||||
|
||||
```bash
|
||||
pnpm test:custom-nodes
|
||||
```
|
||||
|
||||
Green means: every tier for every pack passes, zero skips, and the
|
||||
zero-visible-errors invariant held for the tiers that assert it (mount,
|
||||
persistence, connectivity, core smoke, curated workflows): no error
|
||||
overlay, dialog, node error, or error toast. Two deliberate exceptions,
|
||||
same as the README: the auto-run execution tier provokes expected
|
||||
failures, and the self-check inverts the invariant. Iterate here - it is
|
||||
the fastest loop.
|
||||
|
||||
### 6b - CI-parity run (required if the pack ships frontend JS)
|
||||
|
||||
The dev server never loads pack frontend JS (its `/extensions` list is
|
||||
core-only), so 6a exercises vanilla nodes. If Step 1 found frontend JS, a
|
||||
6a green proves nothing about the pack's real runtime behavior. CI serves
|
||||
the built frontend from the backend, so reproduce that exactly:
|
||||
|
||||
```bash
|
||||
pnpm build
|
||||
# relaunch the test backend with the same flags plus:
|
||||
# --front-end-root <repo>/dist
|
||||
# and make sure any run-tier media is in that process's input/ dir
|
||||
PLAYWRIGHT_TEST_URL=http://127.0.0.1:8288 pnpm exec playwright test \
|
||||
browser_tests/tests/customNodes/ --config playwright.chrome.config.ts --workers=1
|
||||
```
|
||||
|
||||
Both real failures during the first 5-pack onboarding only existed here:
|
||||
rgthree's progress bar shifted the canvas and broke slot-drag coordinates,
|
||||
and rgthree's Seed rebuilt a declared input as widget-only. Skipping 6b
|
||||
means discovering that class of problem one CI round at a time.
|
||||
|
||||
### Failure classes and what they mean
|
||||
|
||||
- **T0 fails only in the Vue Nodes pass** (the LiteGraph pass is green):
|
||||
suspected Vue Nodes 2.0 incompatibility. Follow the policy below - do not
|
||||
delete the pack, do not skip the test.
|
||||
- **Run tier fails with `PARTIAL`** (some expected nodes never executed):
|
||||
either the backend is missing `--cache-none` (cached nodes emit no
|
||||
`executing` event) or an expected node is not actually in the workflow.
|
||||
- **Run tier fails with an execution error**: the workflow JSON is wrong
|
||||
(bad key, wrong `widgets_values` order, type-mismatched link) or the pack
|
||||
cannot execute model-free. Fix the workflow or drop the node for a
|
||||
simpler one.
|
||||
- **Connectivity reports zero planned pairs**: the pack's slots are all
|
||||
wildcard typed, or combo typed with no same-vocabulary partner (wildcards
|
||||
bypass the real type compare; combos pair only when their option lists
|
||||
match exactly). The pack still gets load/run coverage.
|
||||
- **Connectivity logs `widget-only on instance` exclusions**: the pack's own
|
||||
frontend JS rebuilt a declared input as a widget-only control (rgthree's
|
||||
Seed does this to `seed`), so there is no socket to wire. Recorded and
|
||||
excluded, like wildcards - pack design, not a regression.
|
||||
- **Auto-run reports a node "not in cannotRunAlone"**: the node failed to
|
||||
execute on pure defaults or synthesized chain inputs (validation reject,
|
||||
or a real exception from degenerate inputs - empty expression, empty
|
||||
coordinate JSON, single-frame batch, missing optional python dep). If the
|
||||
node USED to run clean this is a regression; otherwise add it to the
|
||||
row's `cannotRunAlone` baseline with the run log in the PR. The check is
|
||||
two-way: a listed node that starts running clean fails the suite until
|
||||
the stale entry is removed. Confidence note: a chain failure proves the
|
||||
node cannot run on synthesized inputs, not that it is broken - the inputs
|
||||
may be semantically insufficient (e.g. a coordinates STRING fed an empty
|
||||
string).
|
||||
- **Auto-run reports `NO_OUTPUT`**: the node executed but its `PreviewAny`
|
||||
sink emitted no ui payload - data never actually flowed out of the node.
|
||||
Treat like any other cannot-run failure: regression or baseline entry.
|
||||
- **Auto-run fails with `HUNG_BACKEND`**: a node blocked forever during
|
||||
execution. Observed mechanism classes so far: model downloads at execute
|
||||
(BLIP/SAM/MiDaS/rembg/CLIPSeg `from_pretrained`), runtime
|
||||
`pip install` inside execute (WAS lazy-install), minutes-long pure-Python
|
||||
per-pixel loops, and an infinite `while` on empty-string defaults. The
|
||||
failure names the suspects and the remedy: add the offender to
|
||||
`AUTO_RUN_EXCLUDE` in `allNodes.spec.ts` with its mechanism, and restart
|
||||
the test backend (the hang is non-interruptible). Everything queued
|
||||
behind the offender reports `HUNG_BACKEND` too - identify the true
|
||||
offender (backend log, `/queue`) before excluding victims.
|
||||
- **Mount test fails on console errors**: a pack's JS logged real errors
|
||||
while its nodes mounted. If it is pack-attributed noise with no visible
|
||||
error surface (KJNodes' loader previews fetching `filename=undefined`),
|
||||
add a scoped `CONSOLE_ERROR_ALLOWLIST` entry (in
|
||||
`fixtures/customNode/consoleErrorLedger.ts`, shared by the all-nodes
|
||||
tiers and the curated run) with the mechanism; otherwise it is a
|
||||
finding.
|
||||
|
||||
### The exception ledgers (all reasons on the record)
|
||||
|
||||
Every escape hatch is a reviewed list whose entries carry the mechanism, so
|
||||
the gate stays honest and none can grow silently:
|
||||
|
||||
| Ledger | Lives in | Covers |
|
||||
| ---------------------------- | ------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `vueIncompatibleNodes` | manifest row | node cannot mount under Vue Nodes 2.0 (evidence rule below) |
|
||||
| `cannotRunAlone` | manifest row | node cannot execute standalone on a bare backend; asserted both ways so entries cannot rot |
|
||||
| `AUTO_RUN_EXCLUDE` | `allNodes.spec.ts` | executing the node is unsafe or unstable (runtime downloads/pip installs, infinite loops, non-interruptible hangs, environment/state-variable results, flip-flopping executed signals) |
|
||||
| `WIDGET_SET_ALLOWLIST` | `allNodes.spec.ts` | plain-typed widget whose value is owned by pack JS (menu-action combos, canonicalized refs) - set-and-stick does not apply |
|
||||
| `ROUNDTRIP_VALUE_ALLOWLIST` | `allNodes.spec.ts` | node whose serialized widgets_values legitimately change on reload (pack JS initializes or rebuilds them); the widget-shrink check still applies |
|
||||
| `MOUNT_WIDGET_ALLOWLIST` | `allNodes.spec.ts` | node whose pack JS renders custom editor/preview widgets outside the node-widget rows; slot fidelity still applies |
|
||||
| `CONSOLE_ERROR_ALLOWLIST` | `fixtures/customNode/consoleErrorLedger.ts` | pack-attributed console noise with no visible error surface; shared by the all-nodes tiers and the curated run |
|
||||
| `CONNECT_REJECTED_ALLOWLIST` | `connectivity.spec.ts` | pack JS legitimately vetoes a planned wiring |
|
||||
| `ROUNDTRIP_LOST_ALLOWLIST` | `connectivity.spec.ts` | pack's own serialize/configure drops links it manages itself |
|
||||
|
||||
### Evidence rules for changing the harness itself
|
||||
|
||||
Two bug classes shipped past green tests once, so these are now policy:
|
||||
|
||||
- **Ground assertions in an oracle you did not write.** A semantic claim
|
||||
about how ComfyUI behaves (what a wire accepts, what an event means, when
|
||||
a widget exists) must cite a live probe, the backend/frontend source, or
|
||||
a CI observation - never plausibility. If every layer agreeing with you
|
||||
was authored from your own mental model (code, fixtures, measurement
|
||||
script), their agreement is not evidence.
|
||||
- **Parse live data against a shape census, not memory.** Node defs reach
|
||||
the suite through `getNodeDefs`, which emits BOTH schema forms (combo as
|
||||
an option-list literal AND as the string `COMBO` with `options`/`remote`
|
||||
in the opts; `forceInput` on any form; autogrow `template` inputs;
|
||||
`socketless`). Any parser of def shapes must handle every form the census
|
||||
shows, its pure-spec fixtures must include each form (copied from real
|
||||
census examples, not invented), and an unrecognized shape must be
|
||||
excluded WITH a record - never silently matched or silently skipped.
|
||||
- **Verify against the source the code consumes.** Measuring raw
|
||||
`/object_info` proves nothing about code that reads the transformed
|
||||
`getNodeDefs` object.
|
||||
|
||||
## Step 7 - push and watch CI
|
||||
|
||||
The `CI: Tests Custom Nodes` job (gating) re-does Steps 1-6 from scratch on
|
||||
every PR: clones every manifest `repo` at its `pin`, pip-installs under CPU
|
||||
torch constraints, boots the backend, runs the suite, and fails on any
|
||||
install error, any test failure, or any skipped test. A new pack row is
|
||||
automatically picked up; no workflow edit is needed unless you must stage an
|
||||
extra asset (Step 4).
|
||||
|
||||
If CI goes red where local was green, reproduce under the Step 6b
|
||||
environment before changing anything - the first such failure looked like
|
||||
upstream drift but was actually pack frontend JS that never loads under
|
||||
the dev server. Only after 6b reproduces it, decide: adjust the suite's
|
||||
expectation honestly (the way widget-only instance slots became a recorded
|
||||
exclusion) or, for genuine upstream drift after a pin bump, re-pin the
|
||||
pack to its last good commit. Never paper
|
||||
over it with a skip.
|
||||
|
||||
## Vue Nodes 2.0 compatibility policy
|
||||
|
||||
Some packs only work under the LiteGraph canvas renderer and fail to mount
|
||||
under Vue Nodes 2.0. The suite must state that fact without producing false
|
||||
failures and without skipping tests:
|
||||
|
||||
1. **Default**: every pack is assumed compatible. New rows omit
|
||||
`vueNodesCompatible`.
|
||||
2. **Evidence rule**: set `"vueNodesCompatible": false` ONLY after the T0
|
||||
Vue pass fails for the pack locally while the LiteGraph pass is green,
|
||||
and the failure reproduces on a retry. A README grumble, a hunch, or an
|
||||
old forum thread is not evidence. Record the evidence (the failing
|
||||
assertion and the pack version) in the PR description of the change that
|
||||
sets the flag. When only SOME of a pack's nodes fail to mount, use the
|
||||
per-node `vueIncompatibleNodes` ledger in the manifest row instead of
|
||||
flagging the whole pack - compatibility is per-node, not per-pack (all
|
||||
823 nodes across the first 7 packs mount clean, so both mechanisms ship
|
||||
unused; the every-node mount tier is what earns an entry).
|
||||
3. **Effect of `false`**: the load tier runs its LiteGraph pass only, and
|
||||
the connectivity drag test does not drag that pack's edges under Vue
|
||||
Nodes. The tests still run and pass their canvas assertions - nothing is
|
||||
`test.skip`ped, so the CI skip gate stays honest. The run tier and the
|
||||
connectivity contract sweep are renderer-independent (they never toggle
|
||||
the Vue Nodes setting) and run for the pack regardless of the flag - a
|
||||
flagged pack must still execute and wire cleanly there.
|
||||
4. **Un-flagging**: if a pack ships Vue Nodes support later, delete the flag
|
||||
and prove T0 green in both passes locally.
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Pack installs clean on the test backend (no `IMPORT FAILED`)
|
||||
- [ ] Checked whether the pack ships frontend JS (Step 1 `/extensions` probe)
|
||||
- [ ] `expectedNodes` copied exactly from `/object_info` (Step 2 traps checked)
|
||||
- [ ] All expected nodes are model-free and present in the run workflow
|
||||
- [ ] Workflow JSON under `browser_tests/assets/customNodes/`, no new binaries
|
||||
- [ ] Any media staged into the backend's own `input/` dir locally (Step 4)
|
||||
- [ ] Manifest row appended with every field (Step 5 table)
|
||||
- [ ] `vueNodesCompatible` omitted, or set `false` with recorded evidence
|
||||
- [ ] 6a green: `pnpm test:custom-nodes` against the dev server, zero skips
|
||||
- [ ] 6b green when the pack ships frontend JS: built dist + backend-served run
|
||||
- [ ] Every-node tiers green: no unexplained mount/save-reload/auto-run
|
||||
failures; any new ledger entry carries its mechanism
|
||||
- [ ] Pushed; `CI: Tests Custom Nodes` green on the PR
|
||||
715
browser_tests/tests/customNodes/ARCHITECTURE.md
Normal file
715
browser_tests/tests/customNodes/ARCHITECTURE.md
Normal file
@@ -0,0 +1,715 @@
|
||||
# Custom-node regression suite architecture
|
||||
|
||||
The design of the custom-node regression suite: what it is made of, how the
|
||||
pieces cooperate, the decisions behind them, and the gotchas that shaped
|
||||
them. Companion docs: [README.md](README.md) (how to run it),
|
||||
[ADDING_CUSTOM_NODES.md](ADDING_CUSTOM_NODES.md) (how to onboard a pack).
|
||||
|
||||
The document is organized as eight architecture views; the diagram map
|
||||
under "Reading paths" shows what question each answers and how they nest.
|
||||
Implementation symbols live in one place: the implementation map at the
|
||||
end (section 14).
|
||||
|
||||
## What / Why / How, in one minute
|
||||
|
||||
**What it proves.** On every PR, for every node that the manifest's
|
||||
community packs register on a real backend, the suite proves four concrete
|
||||
things: the node mounts completely in both renderers (the canvas renderer,
|
||||
LiteGraph, and the DOM renderer, Vue Nodes 2.0), it survives save/reload,
|
||||
its slots wire type-correctly, and it executes when its inputs allow.
|
||||
Section 1 states each proof precisely.
|
||||
|
||||
> **Scale snapshot (example, at the time of writing):** 7 packs, 823
|
||||
> registered nodes, about 5,000 planned wiring checks, about 440 nodes
|
||||
> executing clean per run. These are observations printed by the run, not
|
||||
> properties of the design; they move whenever the manifest or a pin moves.
|
||||
|
||||
**What it does NOT prove.** Output semantics, frontend-only nodes, and
|
||||
hour-scale soak behavior are out of scope; section 1 states the non-goals
|
||||
precisely. Green means "every registered node still mounts, saves, wires,
|
||||
and runs," and nothing wider: a compatibility and regression gate, not a
|
||||
behavior certifier.
|
||||
|
||||
**Why it exists.** Regressions against real community packs used to be
|
||||
invisible: the frontend could break widely installed packs and no test
|
||||
would fail, because nothing exercised those packs at all. Claims about
|
||||
which packs worked were anecdotes with no receipts. The suite turns "most
|
||||
packs are broken" or "this one is fine" from an opinion into a per-node,
|
||||
reproducible result attached to a PR.
|
||||
|
||||
**How it works, in one paragraph.** One manifest row per pack (source,
|
||||
pinned version, tiers, a tiny curated workflow) drives everything; there is
|
||||
no per-pack test code. The suite reads each pack's real node list live from
|
||||
the backend, derives what every node should be able to do, and verifies it
|
||||
in a real browser against a real backend with the pack's own frontend
|
||||
scripts active. Every exception is a reviewed record that carries its
|
||||
causal mechanism, every exception list is guarded against going stale
|
||||
(section 10 grades the strength of each guard), and execution results are
|
||||
reconciled in both directions against a known-failure baseline, so the
|
||||
gate can neither hide a regression nor accumulate dead exemptions.
|
||||
Nothing is ever skipped; a skip fails the job.
|
||||
|
||||
## Reading paths
|
||||
|
||||
- **Skeptical about what green actually covers?** Section 1 (what it proves
|
||||
and the non-goals) and section 12 (the gotchas: every real incident, its
|
||||
root cause, and the defense).
|
||||
- **Deciding pack strategy** (which packs to keep, which renderers to
|
||||
support): section 11 (design decisions and their trade-offs) and the Vue
|
||||
Nodes compatibility policy in ADDING_CUSTOM_NODES.md. A pack is one
|
||||
manifest row to add or remove.
|
||||
- **Onboarding a pack:** ADDING_CUSTOM_NODES.md, not this doc. This doc is
|
||||
the why; that doc is the step-by-step.
|
||||
- **Debugging a red run:** the failure-class list in ADDING_CUSTOM_NODES.md
|
||||
maps each red message to a cause; sections 7 and 10 show where in the pipeline it
|
||||
happened; section 12 gives symptom-first triage.
|
||||
|
||||
How to read the diagrams: a rectangle is one step, named by its purpose; a
|
||||
diamond is a short question, drawn only where the flow genuinely forks; a
|
||||
check that cannot fork is a "Check:" step, not a diamond; a titled group
|
||||
is a thing with internal structure; mechanism detail lives in the prose
|
||||
under each diagram, not stacked inside boxes.
|
||||
|
||||
The eight views are zoom levels of one mental model, not eight parallel
|
||||
pictures. Every arrow below names the element of the parent view that the
|
||||
child expands. The map is ordered by zoom, not by page order: arrows say
|
||||
what contains what, section numbers say where to read.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 240}}}%%
|
||||
flowchart LR
|
||||
L1["System context (section 2): who and what the suite touches"]
|
||||
L2["Building blocks (section 4): what the suite is made of"]
|
||||
L3["Definition pipeline (section 6): where every check's expectations come from"]
|
||||
L4["Execution flow (section 7): how a foreign node gets run safely"]
|
||||
L5["Persistence check (section 8): how save and reload are proven"]
|
||||
L6["Event attribution (section 9): when an arriving event may be believed"]
|
||||
L7["Evidence model (section 10): how exceptions stay honest"]
|
||||
L8["CI deployment view (section 13): the order the test world is built in"]
|
||||
L1 -->|"opens the suite boxes"| L2
|
||||
L1 -->|"expands the CI arrow"| L8
|
||||
L2 -->|"the definition parsers"| L3
|
||||
L2 -->|"the Execution tier"| L4
|
||||
L2 -->|"the Persistence tier"| L5
|
||||
L2 -->|"the Evidence Ledgers box"| L7
|
||||
L4 -->|"the collect-events step"| L6
|
||||
```
|
||||
|
||||
The mount and wiring tiers have no diagram on purpose: each is a
|
||||
single-shot comparison with nothing to sequence, so they live as prose and
|
||||
tables in section 5.
|
||||
|
||||
## 1. What this suite proves, and deliberately does not
|
||||
|
||||
For every node that the manifest's packs register on the backend,
|
||||
re-discovered live on every run:
|
||||
|
||||
- the node **mounts completely** in both renderers: the instance
|
||||
materializes every input and output its definition declares, and under
|
||||
the DOM renderer the page renders at least the instance's widget and
|
||||
slot counts
|
||||
- the node **survives save/reload**: no widget silently disappears and no
|
||||
serialized value silently changes across a save/reload round-trip, and a
|
||||
user-like non-default write sticks and survives a second reload, under
|
||||
both renderers (dynamic widgets the application itself adds on reload are
|
||||
expected and allowed, see section 8)
|
||||
- the node's concrete slots **wire type-correctly** through the real
|
||||
connection validator, and the wires survive save, reload, and prompt
|
||||
serialization
|
||||
- the node **executes on a real backend** when its inputs allow it, and its
|
||||
output observably arrives at an observation sink
|
||||
|
||||
Every tier also asserts the app shows **zero visible errors** while doing
|
||||
this, except the execution tier, which deliberately provokes expected
|
||||
failures (section 7).
|
||||
|
||||
Deliberately out of scope: output semantics (does a blur actually blur),
|
||||
frontend-virtual nodes that never register on the backend, and hour-scale
|
||||
soak behavior. A rare intermittent glitch that only surfaces after long
|
||||
interactive use (a widget that occasionally shrinks on its own) is soak
|
||||
behavior: this per-PR gate will not catch it, and does not claim to.
|
||||
|
||||
## 2. System context
|
||||
|
||||
Who and what the suite touches.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 220}}}%%
|
||||
flowchart LR
|
||||
CIP["CI platform: runs the gate on every PR"]
|
||||
PACKS["Community node packs: external code, installed at pinned versions"]
|
||||
DRIVER["Suite test driver: puts every pack node through its create, wire, save, and submit checks"]
|
||||
FE["ComfyUI frontend: the system under test, running in a real browser"]
|
||||
BE["ComfyUI backend: real graph execution engine"]
|
||||
SYN["Suite verdict synthesis: turns observations into per-node verdicts + exceptions"]
|
||||
TEAM["Engineering team: consumes verdicts and the evidence ledgers"]
|
||||
CIP -->|"builds the environment, triggers"| DRIVER
|
||||
DRIVER -->|"drives a real browser session"| FE
|
||||
FE <-->|"definitions, prompts, execution events"| BE
|
||||
FE -->|"observations: mounts, persistence, execution, errors"| SYN
|
||||
SYN --> TEAM
|
||||
PACKS -->|"frontend scripts load into"| FE
|
||||
PACKS -->|"python side installs into"| BE
|
||||
```
|
||||
|
||||
The two "Suite" boxes are the same system, split so the flow reads one way:
|
||||
the driver puts the frontend through its paces, and verdict synthesis turns
|
||||
what came back into the per-node verdicts and mechanism-carrying exceptions
|
||||
the team consumes. Nothing flows backwards.
|
||||
|
||||
The load-bearing property: the suite tests the same stack a user runs. The
|
||||
pack's own frontend scripts are active, the backend actually executes
|
||||
graphs, and nothing is mocked.
|
||||
|
||||
## 3. The verification environment
|
||||
|
||||
The environment must have these properties, or the suite reports green
|
||||
while testing the wrong thing:
|
||||
|
||||
| Requirement | Why |
|
||||
| ------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| The backend serves the **built** frontend, and tests point at the backend | The dev server loads core extension scripts only, so pack frontend scripts never run under it. Packs that restyle nodes, rebuild widgets, or hook the submission path behave completely differently. Both early "green locally, red on CI" incidents were this. |
|
||||
| Execution caching disabled | Per-node "it actually ran" signals are only emitted for non-cached executions; with caching on, a node can pass without running. |
|
||||
| Isolated test users | Test state must not leak between runs or into a developer's real workspace. |
|
||||
| One test worker | The backend's execution queue is a shared, exclusive resource. Two workers interrupt each other's work and misattribute events. |
|
||||
|
||||
## 4. Building blocks
|
||||
|
||||
What the suite is made of. The main flow is a straight pipeline; the shared
|
||||
services that support the tiers are listed in the table below it.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 240}}}%%
|
||||
flowchart LR
|
||||
MAN["Pack Manifest: source, pin, tiers, known-failure baseline per pack"]
|
||||
ORCH["Test Orchestrator: runs every row through the tiers, honoring the row's tier gates (section 5)"]
|
||||
subgraph TIERS ["Verification tiers (section 5)"]
|
||||
TM["Mount Completeness"]
|
||||
TP["Persistence"]
|
||||
TW["Wiring Compatibility"]
|
||||
TX["Execution"]
|
||||
TM ~~~ TW
|
||||
TP ~~~ TX
|
||||
end
|
||||
EVID["Evidence Ledgers + Reconciler: every result collected, every exception carries its mechanism, lists cannot go stale"]
|
||||
GATE["Gate verdict + evidence for the team"]
|
||||
MAN -->|"drives"| ORCH
|
||||
ORCH -->|"runs, per pack"| TIERS
|
||||
TIERS -->|"all results and exceptions"| EVID
|
||||
EVID -->|"green only if everything is accounted for"| GATE
|
||||
```
|
||||
|
||||
The shared services behind the tiers:
|
||||
|
||||
| Service | Used by | Responsibility |
|
||||
| --------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Definition Normalizer | Wiring (slot model); every all-nodes tier (pack attribution, node keys) | one canonical connectable-slot model out of the multiple definition dialects (section 6), feeding the pairing planner |
|
||||
| Capability Classifier | Execution | decides, per node, what it can do without hand-written fixtures: run on its own defaults, run with synthesized inputs, or blocked, with the reason recorded (section 7) |
|
||||
| Execution Harness | Execution | runs nodes for real and attributes every outcome to the right node despite an asynchronous, noisy event stream (sections 7 and 9) |
|
||||
|
||||
Two further tiers (curated workflows, core smoke) sit alongside these four
|
||||
but are fixture-driven rather than derived from the node corpus; section 5
|
||||
lists all six.
|
||||
|
||||
Dialect handling is deliberately not centralized. Mount and the Capability
|
||||
Classifier read the raw definitions through their own purpose-built
|
||||
parsers (`declaredShape`, `classifyInput`), because each needs a different
|
||||
slice of a definition (declared parts vs. runnability); the normalizer's
|
||||
slot model feeds the wiring planner alone, though the all-nodes tiers
|
||||
also call it for pack attribution and node-key derivation. What keeps the
|
||||
three parsers from drifting is shared evidence, not shared code: each is
|
||||
pinned by fixtures copied from a live census of both definition dialects
|
||||
(section 6).
|
||||
|
||||
- **Pack Manifest**: the single extension point. Adding a pack is one row;
|
||||
no tier knows pack names.
|
||||
- **Evidence Ledgers**: the honesty mechanism. An exception without a
|
||||
recorded mechanism is not allowed to exist (section 10).
|
||||
|
||||
## 5. The verification tiers
|
||||
|
||||
| Tier | Verifies | Renderers | Notes |
|
||||
| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------- | --------------------------------------------------------------------------------------- |
|
||||
| Mount Completeness | every declared input and output actually materializes on the created node; the DOM renderer additionally shows at least the instance's widget/slot counts | both; a pack declared Vue-incompatible runs canvas only | missing parts fail; extras are tolerated |
|
||||
| Persistence | save/reload loses nothing and changes nothing; user-like writes stick and survive reload | both; a pack declared Vue-incompatible runs canvas only | application-added dynamic widgets are legal; see section 8 |
|
||||
| Wiring Compatibility | one representative typed wire per slot connects through the real validator and survives save, reload, and prompt serialization | breadth sweep: one, by decision 7; curated drags: both | dropdown slots pair only on identical option sets; see section 10 for exception routing |
|
||||
| Execution | the node runs on a real backend and its output arrives at an observation sink | one, by decision 7 | the full flow is section 7 |
|
||||
| Curated workflows | a small hand-authored graph per pack executes end to end; its named must-exist nodes are asserted present (a missing one fails the tier, catching a pack that renamed or dropped a node) | both (render pass) | plus a forced-error self-check proving the harness detects real failures |
|
||||
| Core smoke | the core app loads a workflow cleanly with packs installed | both | guards against packs breaking the base app |
|
||||
|
||||
One vocabulary bridge, because the manifest predates these tier names: the
|
||||
manifest row's `tiers` field takes `load`, `run`, `connectivity`, and
|
||||
`io`. Today `run` gates the curated workflow execution, `connectivity`
|
||||
gates the wiring tier, and everything else ignores the field: mount,
|
||||
persistence, execution, and the curated render pass run for every row
|
||||
unconditionally, and core smoke is pack-independent. `load` and `io` are
|
||||
accepted by the schema but currently gate nothing.
|
||||
|
||||
## 6. The node-definition pipeline
|
||||
|
||||
Where the suite's knowledge of every node comes from: definitions flow left
|
||||
to right, and three independent parsers derive three plans from one live
|
||||
census.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 380}}}%%
|
||||
flowchart LR
|
||||
PUB["Backend publishes node definitions"] --> CORPUS["Live definition census: every node the packs register, re-discovered each run, in two dialects"]
|
||||
CORPUS -->|"wiring slot normalizer"| W["Wiring plan: which slots can pair, and why"]
|
||||
CORPUS -->|"execution classifier"| X["Execution plan: which nodes can run, and why the rest cannot"]
|
||||
CORPUS -->|"mount declared-shape parser"| M["Mount expectations: what each created node must materialize"]
|
||||
```
|
||||
|
||||
The three plans are independent consumers of the same census, each through
|
||||
its own dialect-aware parser (section 4 names the symbols): the wiring
|
||||
plan feeds the Wiring Compatibility tier, the execution plan feeds the
|
||||
Execution tier, and the mount expectations feed Mount Completeness.
|
||||
|
||||
Design rule that came from a real bug: every consumer must handle **both
|
||||
definition dialects** (legacy list-form and V2 object-form), and anything
|
||||
with an unknown shape is excluded with a record, never silently matched or
|
||||
skipped. The dialects differ in where dropdown options live, how "must be
|
||||
wired" is flagged, and how growable input groups are declared; details and
|
||||
evidence rules are in ADDING_CUSTOM_NODES.md.
|
||||
|
||||
## 7. The execution flow
|
||||
|
||||
How the suite runs hundreds of foreign nodes safely, with no fixtures, and
|
||||
still attributes every failure to the right node.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 700}}}%%
|
||||
flowchart TD
|
||||
CLASS["Classify each node: what can it do with no hand-written fixtures?"]
|
||||
CLASS --> RUND["runnable on its own defaults"]
|
||||
CLASS --> RUNS["runnable with synthesized inputs"]
|
||||
CLASS --> BLOCK["blocked: the reason is recorded"]
|
||||
RUND --> BATCH["Group runnable nodes into small batches: a failure stays isolated, and one submission carries many nodes instead of paying the round-trip per node"]
|
||||
RUNS --> BATCH
|
||||
BLOCK --> REC
|
||||
BLOCK ~~~ BATCH
|
||||
BATCH --> TG
|
||||
subgraph TG ["Build the batch's disposable test graph: one isolated chain per node"]
|
||||
PROD["synthetic producers for each required input"] --> NUT["the node under test"]
|
||||
NUT --> SINK["an observation sink on its output"]
|
||||
end
|
||||
TG --> SUBQ["Submit the assembled batch graph for real execution"]
|
||||
SUBQ --> GUARD{"submission outcome?"}
|
||||
GUARD -->|"crashed inside a pack's own script"| ERR
|
||||
GUARD -->|"accepted"| OBSERVE["Collect the execution events as the graph runs, keeping only events that belong to this submission and name a node in this test graph (section 9)"]
|
||||
OBSERVE --> V{"outcome?"}
|
||||
V -->|"ran, output observed at the sink"| CLEAN["clean"]
|
||||
V -->|"ran, nothing arrived at the sink"| NOOUT["failure: data never flowed"]
|
||||
V -->|"error attributed to this graph"| ERR["failure: named node, named cause"]
|
||||
V -->|"no response in time"| TRIP["tripwire: interrupt the engine, then watch whether the queue drains"]
|
||||
TRIP --> INT{"recovers?"}
|
||||
INT -->|"yes"| ERR
|
||||
INT -->|"no"| HUNG["engine wedged: stop the tier and name the batch as suspects; queued nodes are victims, not findings"]
|
||||
ERR --> BIS["re-run each batch member alone, so the offender names itself"]
|
||||
NOOUT --> BIS
|
||||
CLEAN --> REC
|
||||
BIS --> REC["Reconcile with the known-failure baseline, in BOTH directions: an unlisted failure fails the gate; a listed entry that now passes, or can no longer run at all, also fails it. Exclusion ledgers are stale-guarded separately"]
|
||||
```
|
||||
|
||||
Synthesized inputs are produced by a small set of self-sufficient producer
|
||||
nodes (an empty image, an empty latent, a solid mask, primitive values), so
|
||||
"runnable with synthesized inputs" needs no per-node authoring. The
|
||||
observation sink is what upgrades "it finished" to "its output actually
|
||||
arrived somewhere."
|
||||
|
||||
The submission guard is why a crash inside a pack's own script can never
|
||||
abort the tier: the throw is caught in the page, recorded as that node's
|
||||
failure with the client error text, and the run moves on.
|
||||
|
||||
## 8. The persistence check
|
||||
|
||||
Why it is staged: the DOM renderer's widget components react to creation
|
||||
and reload on their own schedule, and a check that snapshots synchronously
|
||||
would compare state those reactions never touched. The whole pass runs once
|
||||
per renderer.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 240}}}%%
|
||||
flowchart LR
|
||||
P1["Stand up: create every node of the pack, let the UI settle"]
|
||||
P2["Round-trip: snapshot, reload from the snapshot, snapshot again"]
|
||||
P3["Check: nothing lost, nothing changed; additions the application itself makes are legal"]
|
||||
P4["Probe: write a user-like non-default value into every plain widget, verify every write sticks"]
|
||||
P5["Round-trip again: snapshot, reload from the snapshot, snapshot again"]
|
||||
P6["Check: written values survive wherever the node's shape stayed stable (a changed dropdown can legally rebuild a dynamic node's widgets)"]
|
||||
P1 --> P2 --> P3 --> P4 --> P5 --> P6
|
||||
```
|
||||
|
||||
Between phases the rig yields to the UI so renderer effects flush before
|
||||
the next snapshot; those settle points are what makes the staging real.
|
||||
|
||||
Widgets whose values the pack's own script owns (canonicalized references,
|
||||
embedded editors) are exempt from probe writes, each with a recorded
|
||||
mechanism: writing probe markers into them only makes the pack's script
|
||||
choke on the probe.
|
||||
|
||||
## 9. Event attribution
|
||||
|
||||
Real execution reports back over an asynchronous event stream, and the
|
||||
stream can mislead in two specific ways. Both produced real misattributed
|
||||
failures before the filters existed. The primary defense is positive: when
|
||||
the harness submits a graph, it captures the id the backend assigns to
|
||||
that submission from the submission response itself, so an event's
|
||||
ownership is checked against a known id, never inferred from history.
|
||||
Every arriving event passes the same two questions before it may count as
|
||||
evidence:
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 280}}}%%
|
||||
flowchart TD
|
||||
EV["an event arrives on the execution stream, while this attempt runs"]
|
||||
EV --> Q1{"from THIS attempt?"}
|
||||
Q1 -->|"no: it does not carry the id this submission was assigned"| DROP["dropped: a stray cannot blame any node in this run"]
|
||||
Q1 -->|"yes"| Q2{"names a node in THIS test graph?"}
|
||||
Q2 -->|"no: it names another graph's nodes"| DROP
|
||||
Q2 -->|"yes"| KEEP["kept: evidence for exactly that node"]
|
||||
```
|
||||
|
||||
Both no-answers are checkable, not hopeful. The first is a comparison
|
||||
against the captured submission id: an event either carries it or it does
|
||||
not. If that capture ever misses, the harness says so on the console and
|
||||
falls back to identity bookkeeping, recording every attempt identity it
|
||||
has ever seen so a late event from an observed attempt still identifies
|
||||
itself. The second question defeats the one stray the first cannot: a
|
||||
retried duplicate arriving under a never-seen identity. Node identities
|
||||
are never reused within a session, so such an event can only name an
|
||||
earlier graph's nodes. Membership is decisive.
|
||||
|
||||
## 10. The evidence model
|
||||
|
||||
The suite's honesty mechanism. Every exception is a reviewed record that
|
||||
names its causal mechanism, and every list is guarded: an entry naming a
|
||||
node the pack no longer registers fails the suite. Full per-record
|
||||
semantics live in the ledger table in
|
||||
[ADDING_CUSTOM_NODES.md](ADDING_CUSTOM_NODES.md).
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 300}}}%%
|
||||
flowchart TD
|
||||
F["a node fails a tier"] --> Q1{"is EXECUTING it unsafe or environment-dependent?"}
|
||||
Q1 -- yes --> L1["execution exclusion: never run; mechanism on record; every other tier still applies"]
|
||||
Q1 -- no --> Q2{"does it fail deterministically on synthesized inputs?"}
|
||||
Q2 -- yes --> L2["known-failure baseline: still runs every time; reconciled in both directions"]
|
||||
Q2 -- no --> Q3{"does the pack's own script own the failing surface?"}
|
||||
Q3 -- yes --> L3["scoped exception record naming the mechanism"]
|
||||
Q3 -- no --> L4["no exception applies: it is a finding. Fix it or file it"]
|
||||
```
|
||||
|
||||
What the first question means in practice: runtime downloads or installs,
|
||||
infinite loops, host-specific results, mutable-content dropdowns,
|
||||
unreliable completion signals. What a pack script owning the failing
|
||||
surface looks like: rewritten values, custom widgets, vetoed wires,
|
||||
console noise.
|
||||
|
||||
The two-way baseline is what stops the whole evidence model from rotting: a
|
||||
failure that is not listed fails the gate, and a listed node that starts
|
||||
passing ALSO fails the gate until its stale entry is removed. Exemptions
|
||||
cannot silently accumulate.
|
||||
|
||||
Not every ledger can earn that two-way strength; the guards come in three
|
||||
grades. Ledgers whose nodes still execute (the known-failure baseline) are
|
||||
two-way behavioral: a new failure and a stale entry both flip the gate.
|
||||
Ledgers that stop a path from running at all (execution exclusions,
|
||||
probe-write exemptions) are registration guarded: the suite proves the
|
||||
named node still exists, but the excluded path never runs, so an entry
|
||||
that stopped being necessary cannot be observed; staleness there is
|
||||
caught by review, not observation. Weakest are the pattern allowlists
|
||||
(the console-error ledger): an entry that no longer matches anything
|
||||
simply filters nothing, and usage tracking cannot be naively bolted on,
|
||||
because some patterns are environment conditional (a missing-model 404
|
||||
fires only on hosts without the model), so an entry can be legitimately
|
||||
idle in one environment and load-bearing in the next.
|
||||
|
||||
The console-error ledger also has a bounded window, not just bounded
|
||||
strength. Collection starts inside each tier, so it covers that tier's
|
||||
own actions (load, run, wire, save); console noise a pack logs at app
|
||||
boot, before the first tier action, is outside it - the shared app
|
||||
fixture navigates once at setup, so boot output predates any per-pack
|
||||
collector. This is deliberate: boot breakage that reaches a visible
|
||||
surface is still caught by the startup zero-visible-errors check, and
|
||||
invisible boot console noise is exactly what the ledger exists to
|
||||
tolerate rather than gate on.
|
||||
|
||||
## 11. Design decisions
|
||||
|
||||
The decisions that define the suite, with their trade-offs. Each is
|
||||
deliberate, and each is cheap to reverse or narrow later. The suite's one
|
||||
deliberate extension seam is the curated-workflow fixture: anything the
|
||||
manifest cannot derive from the live node corpus (pack-specific semantics,
|
||||
multi-node behavior) is expressed there (decisions 6 and 11).
|
||||
|
||||
| # | Decision | Why | Trade-off accepted |
|
||||
| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| 0 | Drive a real browser, not just the backend API | Pack frontend scripts (widget rebuilds, restyles, submission hooks) are half of what breaks; only a browser running the built frontend exercises them | Browser e2e is the slowest, most race-prone tier; mitigated by the attribution filters (section 9) and the staged settle points (section 8) |
|
||||
| 1 | Real environment only: real browser, real backend, pack scripts active, nothing mocked | The failures worth catching live in the integration, not in units | Slower than unit tests; needs a backend in CI |
|
||||
| 2 | The backend serves the built frontend | The dev server never loads pack scripts, so it tests a different product | Local iteration needs a build + restart for pack-script changes |
|
||||
| 3 | One test worker | The execution queue is exclusive; parallel workers corrupt each other's evidence | Wall-clock time grows with the manifest |
|
||||
| 4 | Execution caching disabled | The per-node "actually ran" signal only exists for uncached executions | Every run pays full execution cost |
|
||||
| 5 | Packs installed at pinned, verified versions | An upstream push must not change what the gate tests mid-flight | Pins need deliberate bumps; a nightly canary against pack HEADs is the planned complement |
|
||||
| 6 | One manifest row per pack, zero per-pack test code | Extension cost stays constant as coverage grows | The generic tiers cannot assert pack-specific semantics; curated workflows exist for that |
|
||||
| 7 | Both renderers only where the renderer can change the outcome: mount, persistence, the curated render pass, the curated pointer drags, core smoke; one renderer elsewhere (breadth wiring sweep, execution) | Widget values flow through the same store under both renderers (verified by probe), so doubling execution buys no new failure surface | If that store unification ever changes, revisit this decision |
|
||||
| 8 | Every exception carries its mechanism and is stale-guarded | An unexplained exemption is indistinguishable from a hidden bug | Onboarding a flaky pack takes more effort than a blanket skip |
|
||||
| 9 | Known-failure baseline reconciled in both directions | One-way baselines rot into permanent blind spots | A node that gets fixed upstream turns the gate red until its entry is removed (by design) |
|
||||
| 10 | Small batches with single-node bisection | Batching amortizes queue latency; bisection restores per-node attribution on failure | A failing batch costs one extra pass over its members |
|
||||
| 11 | Scope excludes output semantics and frontend-virtual nodes | Both need per-node knowledge a manifest cannot derive; curated workflows and future behavior tests are the extension point | "Green" is narrower than "the pack fully works," and says so |
|
||||
|
||||
## 12. Gotchas: every incident, its root cause, and the defense
|
||||
|
||||
These failure modes shaped the suite. Each was real: something passed that
|
||||
should have failed, or failed for a reason that had nothing to do with the
|
||||
node under test. Named nodes below are worked examples of their class,
|
||||
kept because specifics are what make a mechanism checkable. Do not remove
|
||||
a defense without re-reading its incident. The two recurring team concerns
|
||||
these answer: "green but broken" and "tests can never catch random bugs."
|
||||
|
||||
### G1. Dev-server pack-script blindspot
|
||||
|
||||
- **You hit it when**: a node behaves perfectly in local dev but breaks on
|
||||
CI, or vice versa, on any pack that restyles nodes, rebuilds widgets, or
|
||||
hooks the submission path.
|
||||
- **Root cause**: the dev server loads core extension scripts only; pack
|
||||
frontend scripts never run under it. The node tested there is a
|
||||
different node than the one users get.
|
||||
- **Defense**: the environment contract (section 3): the backend serves the
|
||||
built frontend and tests point at the backend. CI does exactly this (section 13).
|
||||
- **Answers**: green but broken.
|
||||
|
||||
### G2. Widget-state bleed through recycled node identities
|
||||
|
||||
- **You hit it when**: a node fails validation with a value it was never
|
||||
given, specifically a dropdown carrying an option that belongs to some
|
||||
OTHER node created earlier in the same session.
|
||||
- **Root cause**: the frontend keeps widget state keyed by node identity,
|
||||
and that state survives clearing the graph. A new node that reuses a
|
||||
cleared node's identity inherits its same-named widget values. Core
|
||||
frontend bug, distinct from this suite; the defense below stands
|
||||
regardless of when it is fixed.
|
||||
- **Defense**: the suite never reuses a node identity within a browser
|
||||
session: every builder hands out monotonically increasing identities
|
||||
across graph clears.
|
||||
- **Answers**: green but broken (a neighbor's leftover value produces a
|
||||
false failure and hides the real store bug).
|
||||
|
||||
### G3. Event misattribution races
|
||||
|
||||
- **You hit it when**: node A is reported failing, but the error belongs to
|
||||
node B tested just before it, or to a duplicate submission of an earlier
|
||||
graph.
|
||||
- **Root cause**: two races over the asynchronous event stream: late
|
||||
arrivals from a previous attempt, and duplicate attempts created by a
|
||||
submission retry erroring under a fresh identity.
|
||||
- **Defense**: the positive submission-id match plus the graph-membership
|
||||
filter of section 9, made decisive by G2's never-reuse-identities rule.
|
||||
- **Answers**: tests can never catch random bugs (a misattributed error is
|
||||
noise that erodes trust in every verdict).
|
||||
|
||||
### G4. Pack scripts crashing the submission path
|
||||
|
||||
- **You hit it when**: an entire pack's execution tier aborts, not just one
|
||||
node.
|
||||
- **Root cause**: pack scripts can hook workflow submission and throw on a
|
||||
graph shape they do not expect. Observed example: a video pack's
|
||||
"apply to graph" hook copies its latest file into downstream widget
|
||||
inputs and throws when its output feeds a plain socket while matching
|
||||
files exist; the trigger is content-dependent.
|
||||
- **Defense**: submission runs guarded; a throw records as that node's
|
||||
failure, carrying the exception text, so the node names itself instead
|
||||
of aborting the tier. The proven case is also excluded with its
|
||||
mechanism in the exclusion ledger, and remains an upstream-report
|
||||
candidate.
|
||||
- **Answers**: tests can never catch random bugs (uncaught, one crash masks
|
||||
every node queued behind it).
|
||||
|
||||
### G5. Two definition dialects
|
||||
|
||||
- **You hit it when**: a set of nodes silently never executes: they are
|
||||
classified as needing wires they do not need, so the planner skips them
|
||||
and nothing goes red.
|
||||
- **Root cause**: node definitions reach the suite in two dialects (legacy
|
||||
list-form and V2 object-form), and a parser written against one dialect
|
||||
misreads the other. Measured example: 8 nodes of one pack were invisibly
|
||||
unexecuted until the classifier learned the second dialect.
|
||||
- **Defense**: each consumer's parser handles both dialects
|
||||
(`declaredShape` for mount, `classifyInput` for execution, the
|
||||
normalizer for wiring; section 4); parser fixtures are copied from a
|
||||
live census of the real corpus so tests cannot self-confirm a parser's
|
||||
assumptions; unknown shapes are excluded with a record, never silently
|
||||
matched (section 6).
|
||||
- **Answers**: green but broken (a whole class of nodes was uncovered while
|
||||
the tier stayed green).
|
||||
|
||||
### G6. "Must be wired" beats every dialect
|
||||
|
||||
- **You hit it when**: an input the pack marked as wire-only is treated as
|
||||
a widget, so the node runs without the wire it requires.
|
||||
- **Root cause**: the wire-only flag can appear on any input form; a
|
||||
classifier that checks the form before the flag misreads it.
|
||||
- **Defense**: the classifier checks the wire-only flag first, before any
|
||||
form-specific branch; fixtures pin the ordering.
|
||||
- **Answers**: green but broken.
|
||||
|
||||
### G7. Dropdown pairing semantics
|
||||
|
||||
- **You hit it when**: the wiring tier pairs two unrelated dropdowns (a
|
||||
checkpoint list into a scheduler list), a pass that proves nothing, or
|
||||
refuses to pair two dropdowns that differ only in menu order.
|
||||
- **Root cause**: a wired dropdown input bypasses its own menu, so the wire
|
||||
contract is set membership of options, not their order. And dropdowns
|
||||
whose options are not statically known cannot prove anything by pairing.
|
||||
- **Defense**: dropdowns pair only on identical option SETS
|
||||
(order-insensitive); dropdowns with unknown option lists are excluded
|
||||
from pairing with a record instead of blind-matched.
|
||||
- **Answers**: green but broken.
|
||||
|
||||
### G8. Environment flips
|
||||
|
||||
- **You hit it when**: a node fails on one OS but is clean on another, run
|
||||
to run, with no code change. A subtle variant is the warm-cache
|
||||
illusion: a node that downloads model weights inside execution runs
|
||||
clean only where the cache is already warm.
|
||||
- **Root cause**: execution depends on the host, not on the node's
|
||||
frontend contract: numeric-stack differences, codec differences, cached
|
||||
downloads, directory-handling differences.
|
||||
- **Defense**: the environment-variable class of execution exclusions,
|
||||
each entry naming its per-host mechanism, reconciled against observation
|
||||
runs on both hosts. The node keeps every non-execution tier.
|
||||
- **Answers**: tests can never catch random bugs (host-dependent flips are
|
||||
flake that trains people to ignore red).
|
||||
|
||||
### G9. Queue jams from non-interruptible execution
|
||||
|
||||
- **You hit it when**: the execution tier hangs and every node queued
|
||||
BEHIND one offender reports failure.
|
||||
- **Root cause**: some execution paths never respond to interrupt:
|
||||
installing packages at runtime, pure-Python infinite loops (observed
|
||||
example: a text-replace node spinning forever on an empty search
|
||||
string), minutes-long per-pixel loops, non-interruptible weight
|
||||
downloads.
|
||||
- **Defense**: a timeout interrupts and checks that the queue recovers; a
|
||||
queue that will not drain stops the tier immediately and names the batch
|
||||
as suspects. Triage is explicitly offender-versus-victims, and a
|
||||
preflight asserts the queue is idle before the tier starts. Proven
|
||||
offenders are excluded with their mechanism.
|
||||
- **Answers**: tests can never catch random bugs (a jam failing a whole
|
||||
batch is pure noise; the tripwire converts it into one named offender).
|
||||
|
||||
### G10. Renderer effect timing
|
||||
|
||||
- **You hit it when**: the persistence tier passes under the canvas
|
||||
renderer but silently tests nothing under the DOM renderer.
|
||||
- **Root cause**: DOM-renderer widget components react to creation and
|
||||
reload asynchronously, writing back into the value store on frame
|
||||
boundaries; a synchronous snapshot compares state those reactions never
|
||||
touched.
|
||||
- **Defense**: the persistence check is staged with explicit settle points
|
||||
between build, snapshot, reload, and write phases (section 8), and runs once
|
||||
per renderer.
|
||||
- **Answers**: green but broken (a synchronous pass certifies a value path
|
||||
it never observed).
|
||||
|
||||
### G11. Growable input groups materialize under expanded names
|
||||
|
||||
- **You hit it when**: mount completeness reports a declared input missing
|
||||
on a node that uses growable input groups, when the renderer actually
|
||||
materialized it under expanded per-slot names.
|
||||
- **Root cause**: growable input groups do not materialize under their
|
||||
declared group name; they expand into per-slot names derived from it.
|
||||
- **Defense**: mount expectations accept either the group name or its
|
||||
required expansion; this was the only definition-shape special case
|
||||
found across the full corpus.
|
||||
- **Answers**: keeps mount fidelity strict without false-failing
|
||||
group-typed nodes.
|
||||
|
||||
### G12. Legal dynamic growth on reload
|
||||
|
||||
- **You hit it when**: a node legitimately gains a widget on reload (the
|
||||
application attaches a seed-control widget; a pack appends a
|
||||
value-driven widget) and a naive equality check flags it as a
|
||||
regression.
|
||||
- **Root cause**: reload is allowed to APPEND; what must never happen is
|
||||
the inverse: a widget disappearing or a saved value changing.
|
||||
- **Defense**: the persistence comparison is asymmetric by design: growth
|
||||
passes, loss or mutation fails; after probe writes, values are compared
|
||||
only where the node's shape stayed stable, because a changed dropdown
|
||||
can legally rebuild a dynamic node.
|
||||
- **Answers**: green but broken, from the other side: a check that
|
||||
rejected legal growth would get relaxed into uselessness.
|
||||
|
||||
### G13. Mutable-content dropdowns
|
||||
|
||||
- **You hit it when**: a file-list node flips between clean and failing
|
||||
across runs, tracking whatever content the backend happens to hold.
|
||||
- **Root cause**: some dropdowns populate from mutable backend content
|
||||
(file listings, run history), so their default value and validity change
|
||||
as content changes.
|
||||
- **Defense**: the state-dependent class of execution exclusions, with the
|
||||
mechanism on record; where the same dropdown also re-resolves on reload,
|
||||
a scoped persistence exception skips the value comparison while the
|
||||
no-shrink rule still applies. All other tiers are retained.
|
||||
- **Answers**: tests can never catch random bugs.
|
||||
|
||||
### G14. Unreliable completion signals
|
||||
|
||||
- **You hit it when**: a node reports clean on one run and incomplete on
|
||||
the next with no change to anything.
|
||||
- **Root cause**: the per-node "actually ran" signal is reliable for
|
||||
ordinary nodes with caching disabled, but list-expanded and
|
||||
remote-control nodes do not emit it on every run.
|
||||
- **Defense**: only nodes with a PROVEN signal flip are excluded from
|
||||
execution, each recorded with the shared mechanism, so an incomplete
|
||||
result stays meaningful everywhere else.
|
||||
- **Answers**: tests can never catch random bugs.
|
||||
|
||||
## 13. The CI deployment view
|
||||
|
||||
In today's implementation, the suite is Playwright driving bundled
|
||||
Chromium, and the CI platform is GitHub Actions.
|
||||
|
||||
```mermaid
|
||||
%%{init: {"flowchart": {"wrappingWidth": 260}}}%%
|
||||
flowchart LR
|
||||
CH["change gate: skip only when nothing relevant changed, without wedging the required check"] --> BUILD["build the frontend"]
|
||||
BUILD --> ENV["provision a CPU backend"]
|
||||
ENV --> INST["clone every manifest pack at its pinned version; install with dependency constraints so packs cannot swap the numeric stack"]
|
||||
INST --> ASSET["stage the curated workflows' media"]
|
||||
ASSET --> RUN["boot the backend serving the built frontend; run the suite, one worker"]
|
||||
RUN --> SKIP{"anything skipped?"}
|
||||
SKIP -- yes --> RED["fail: a pack or a fixture failed to load"]
|
||||
SKIP -- no --> ART["publish the report artifact"]
|
||||
```
|
||||
|
||||
Fork PRs skip the job (the install loop is a code-execution surface) and
|
||||
keep coverage via the main test shards. Sharding is deliberately deferred:
|
||||
every shard would pay the full environment setup, which is a large share of
|
||||
the job; the workflow states the threshold at which sharding starts paying.
|
||||
Ballpark at the time of writing, moving like the scale snapshot: about
|
||||
eight minutes of suite on top of about four and a half minutes of
|
||||
environment setup, with sharding starting to pay once the whole job
|
||||
passes roughly twelve minutes.
|
||||
|
||||
## 14. Implementation map
|
||||
|
||||
The one place where architecture names meet code symbols.
|
||||
|
||||
| Building block | File | Key symbols |
|
||||
| ------------------------------------- | -------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Pack Manifest | `browser_tests/fixtures/data/customNodeManifest.json` | one row per pack: `pack`, `repo`, `pin`, `tiers`, `workflow`, `expectedNodes`, `requiresGpu`, `requiresModels`, `timeoutMs`, plus optional `vueNodesCompatible`, `vueIncompatibleNodes`, `cannotRunAlone` |
|
||||
| Manifest loader | `browser_tests/fixtures/customNode/manifest.ts` | `loadManifest`, `rendererPassesFor` |
|
||||
| Test Orchestrator | each spec file | the `for (const entry of loadManifest())` loop heading allNodes.spec.ts, connectivity.spec.ts, customNode.regression.spec.ts |
|
||||
| Evidence Ledgers + Reconciler | `browser_tests/tests/customNodes/allNodes.spec.ts`, `connectivity.spec.ts` | the `*_ALLOWLIST` maps, `AUTO_RUN_EXCLUDE`, the `cannotRunAlone` two-way reconciliation, stale-entry guards |
|
||||
| Definition Normalizer | `browser_tests/fixtures/customNode/typePairing.ts` | `normalizeNodeDefs`, `packOf` |
|
||||
| Wiring planner | `browser_tests/fixtures/customNode/typePairing.ts` | `planPairs`, `isTypeCompatible`, `vocabOf` |
|
||||
| Capability Classifier | `browser_tests/fixtures/customNode/autoRun.ts` | `classifyAutoRunnable`, `classifyInput`, `planAutoRuns`, `batchAutoRunnable`, `SYNTH_PRODUCERS` |
|
||||
| Execution Harness | `browser_tests/fixtures/customNode/ComfyTarget.ts` | `LocalDesktopTarget.runWorkflow`: event tap, attempt + graph-membership filters, guarded submission |
|
||||
| Outcome classification | `browser_tests/fixtures/customNode/runResult.ts` | `classifyRun`, `CustomNodeOutcome` |
|
||||
| Mount / Persistence / Execution tiers | `browser_tests/tests/customNodes/allNodes.spec.ts` | `addChunk`, `declaredShape`, the staged rig on `window.__cnRt`, `runBatch`, monotonic identities via `window.__cnIdBase`, five in-spec exception ledgers |
|
||||
| Wiring tier | `browser_tests/tests/customNodes/connectivity.spec.ts` | breadth sweep, executor self-check, curated drags, two allowlists |
|
||||
| Curated workflows + self-check | `browser_tests/tests/customNodes/customNode.regression.spec.ts` | T0/T1 per pack, forced-error positive control |
|
||||
| Core smoke | `browser_tests/tests/customNodes/coreSmoke.spec.ts` | |
|
||||
| Parser/classifier fixtures | `browser_tests/tests/customNodes/*.pure.spec.ts` | census-derived cases for both definition dialects |
|
||||
| CI job | `.github/workflows/ci-tests-custom-nodes.yaml` | gating check `custom-nodes-e2e` |
|
||||
190
browser_tests/tests/customNodes/DETECTION_PROOF.md
Normal file
190
browser_tests/tests/customNodes/DETECTION_PROOF.md
Normal file
@@ -0,0 +1,190 @@
|
||||
# Detection Proof
|
||||
|
||||
How we prove the custom-node regression suite actually catches every failure
|
||||
mode it claims to in [ARCHITECTURE.md](ARCHITECTURE.md). The proof is a
|
||||
separate, deliberately-red pull request branched off the suite branch: each
|
||||
commit breaks one surface on purpose, cites the real regression class it
|
||||
recreates, and turns the custom-nodes CI check red at exactly the named tier
|
||||
with the named message. (A frontend break may also trip other layers, e.g.
|
||||
unit tests - that is layered coverage, not noise.) A green custom-nodes check
|
||||
anywhere in that PR would mean the gate failed to catch a regression.
|
||||
|
||||
This replaces the earlier ad-hoc "kill-test" name. The verb is **falsify**: we
|
||||
falsify each guard by breaking the thing it watches and confirming it fires.
|
||||
|
||||
## Why this exists
|
||||
|
||||
The suite's value claim is that a frontend PR can no longer silently break a
|
||||
widely-installed custom-node pack. That claim is only worth as much as its
|
||||
ability to go red on a real break. A green suite proves nothing on its own -
|
||||
it could be green because everything works, or green because it checks nothing.
|
||||
The Detection Proof PR removes that doubt: it shows, break by break, that every
|
||||
tier in ARCHITECTURE.md turns red on the exact class of regression it was built
|
||||
to catch, and names the offender in the failure message.
|
||||
|
||||
## How to read the proof PR
|
||||
|
||||
- **It must never merge.** Every commit is a deliberate break. A reviewer reads
|
||||
it, they do not ship it.
|
||||
- **One commit per surface.** Each commit is a single-file change plus a comment
|
||||
naming the historical regression it recreates and the red it should produce.
|
||||
Check out a commit, watch the named CI check go red, read the message, move on.
|
||||
- **CI is the source of truth, not a local full run.** The CI job runs the
|
||||
suite against one fresh backend on an unloaded runner, which keeps every
|
||||
execution inside its budget. A local run of the whole
|
||||
suite against a single CPU backend is not reliable for this (see
|
||||
[Honest caveat](#honest-caveat-local-full-runs-and-machine-load)); run CI, or
|
||||
run one pack locally at a time.
|
||||
|
||||
## Two protection modes
|
||||
|
||||
The gate protects against two distinct things, and the proof covers both:
|
||||
|
||||
- **FE-regression** - a change to _this frontend_ breaks installed packs. This
|
||||
is the primary thing the gate guards on every frontend PR. These breaks live
|
||||
in `src/`.
|
||||
- **Pack-bug** - a pack itself ships a bug (or a pinned pack is bumped to a
|
||||
broken version). The gate catches these too. CI clones every pack fresh at
|
||||
its pin, so editing pack files in the frontend repo does nothing - the clone
|
||||
overwrites them. Two ways deliver a pack break on CI: (a) point the manifest
|
||||
(`browser_tests/fixtures/data/customNodeManifest.json`) `repo`/`pin` at a
|
||||
broken fork, which is exactly the pinned-bump scenario and the most
|
||||
production-faithful; or (b) a self-contained CI step that patches each cloned
|
||||
pack in place right after install. The proof PR uses (b) - no external repos,
|
||||
and each patch asserts it landed (`grep`, fails the job otherwise) so a silent
|
||||
no-op cannot fake a pass. Both reproduce the same edits captured against a
|
||||
local backend (which is how the exact reds below were captured).
|
||||
|
||||
Each row below is labelled with its mode.
|
||||
|
||||
## The correlation matrix
|
||||
|
||||
Every "Exact red" below is the real message captured when the break was applied
|
||||
and the tier was run against a real backend - not a prediction. One scope note:
|
||||
for the corpus-derived tiers (rows 4, 6, 9) the named offender and pair list
|
||||
are re-derived from `/object_info` each run, so a pin bump can legitimately
|
||||
change WHICH pair or node the message names without weakening the catch - the
|
||||
promise is the tier and the failure class, not byte-identical offender text
|
||||
across pin changes. Sections refer to [ARCHITECTURE.md](ARCHITECTURE.md).
|
||||
|
||||
| # | Surface (ARCH section) | Mode | Real regression it recreates | The one-file break | CI check that catches it | Exact red |
|
||||
| --- | ------------------------------------------------ | ---- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | ----------------------------------------------------------------------------------------------------------- |
|
||||
| 1 | Mount completeness, canvas / v1 (s1, s5) | FE | A change dropping declared parts on the canvas renderer (class; no single ticket - the v2 wave below shows how this family presents) | `src/services/litegraphService.ts` `addInputs`: stop materializing the last declared input | Tests Custom Nodes / mount tier | `BatchCount+: instance is missing declared input "batch" (litegraph)` |
|
||||
| 2 | Mount completeness, DOM / v2 (s1, s5) | FE | Widgets missing under Nodes 2.0 (FE-627/FE-634 iTools buttons; FE-841 is the adjacent wrong-style class, present but unproven caught) | `src/renderer/extensions/vueNodes/widgets/registry/widgetRegistry.ts`: drop the `int` widget component mapping | Tests Custom Nodes / mount tier (Vue pass) | `Ideogram4PromptBuilderKJ: Vue mounts 9 of 15 widgets` |
|
||||
| 3 | Persistence, save/reload (s1, s8) | FE | Widgets reverting to socket-only on reload: the defaultInput migration regression that PR #12279 (open) exists to fix | `src/lib/litegraph/src/LGraphNode.ts` `configure`: off-by-one drops the last `widgets_values` entry | Tests Custom Nodes / persistence tier | `Seed (rgthree): widgets_values [1,"fixed"] -> [1,"randomize"] on set-values reload` |
|
||||
| 4 | Wiring - type compatibility (s5, s6) | FE | A frontend change narrowing connectable types (class; no single verified ticket) | `src/lib/litegraph/src/LiteGraphGlobal.ts` `isValidConnection`: reject IMAGE links | Tests Custom Nodes / connectivity sweep | `AddLabel.IMAGE -> FastPreviewBatch.input: CONNECT_REJECTED` (full pair list) |
|
||||
| 5 | Wiring - drop resolution (s5) | FE | Drag/slot resolution family (nearest reported symptoms: FE-625/FE-632 EditUtils connections shift after drag) | `src/lib/litegraph/src/canvas/measureSlots.ts` `getNodeInputOnPos`: return undefined | Tests Custom Nodes / connectivity drag | `EmptyImage.IMAGE -> ImageBatch.image2 with VueNodes=false` |
|
||||
| 6 | Execution - frontend prompt serialization (s7) | FE | A prompt-serialization change corrupting inputs (class; no single verified ticket) | `src/utils/executionUtil.ts`: drop numeric widget values from the API prompt | Tests Custom Nodes / curated run (T1) | `Prompt outputs failed validation; ImpactInt: value; ImpactFloat: value` |
|
||||
| 7 | Zero-visible-errors / load hook (s1) | FE | An extension hook crashing on graph load, the mechanism packs hook (FE-751 class; the break is in a core extension, hence FE mode) | `src/composables/node/useNodeBadge.ts` `afterConfigureGraph`: throw | Tests Custom Nodes / curated run (T1) | `Error calling extension 'Comfy.NodeBadge' method 'afterConfigureGraph' ...` |
|
||||
| 8 | Console / pageerror ledger (s10) | Pack | An uncaught pack-JS error during save/reload (the betterCombos.js `typeof null` bug this suite found) | CI step patches the cloned ComfyUI-Custom-Scripts `showText.js` to log a `console.error` in `onConfigure` (captured locally by editing the installed pack directly) | Tests Custom Nodes / curated run (T1) | `console errors during curated run` + the exact text + script URL |
|
||||
| 9 | Execution - runtime (s7) | Pack | A pack node raising at execution (WAS Text Find/Replace infinite loop; KJ ImageGridtoBatch min violation) | CI step patches the cloned was-node-suite `return_constant_number` to raise on entry (captured locally by editing the installed pack directly) | Tests Custom Nodes / auto-run tier | `Constant Number: EXECUTION_ERROR (Constant Number: ValueError) - not in cannotRunAlone; a regression, ...` |
|
||||
| 10 | Registration / expectedNodes sentinels (s5, s10) | Pack | A pinned pack bump renaming a node key | CI step patches the cloned ComfyUI-Impact-Pack `__init__.py` to rename the `ImpactInt` mapping key (captured locally by editing the installed pack directly) | Tests Custom Nodes / zero-skip gate | job goes red on `skipped != 0` (T0 + T1 skip; the workflow's "Forbid skipped tests" step fails) |
|
||||
|
||||
### Links of various types (surface 4/5 expanded)
|
||||
|
||||
"Links of various types" is covered breadth-first: the connectivity tier
|
||||
plans one representative typed edge per slot across the whole installed corpus,
|
||||
so a single break in the validator (#4) fails a broad, named list of concrete
|
||||
pairs - not one hand-picked wire. The drag break (#5) additionally proves the
|
||||
_pointer_ path resolves the exact slot. To show breadth explicitly, the proof PR
|
||||
can add two more validator mutations, each turning a different link class red:
|
||||
|
||||
- Break the COMBO option-vocabulary compare (`vocabOf`) - the committed pure
|
||||
specs (typePairing.pure.spec.ts, same-vocabulary pairing tests) go red;
|
||||
dropdown slots are checked, not just primitive types.
|
||||
- Break the wildcard exclusion (`isWildcard`) - the committed pure specs
|
||||
("wildcard slots are excluded" test) go red; the exclusion is pinned as a
|
||||
design decision, not an accident. Both catches are at the pure-spec layer;
|
||||
whether the live corpus also exercises them per run is not asserted here.
|
||||
|
||||
### Execution of various types (surface 6/7/9 expanded)
|
||||
|
||||
Three distinct execution break-points, each caught by a different tier:
|
||||
|
||||
- **Frontend serialization** (#6) - the value never leaves the browser correctly;
|
||||
caught at submit as a named `VALIDATION_FAIL`.
|
||||
- **Load-time hook** (#7) - an extension hook crashes the graph load (the same
|
||||
hook mechanism pack scripts use); caught by the console/pageerror ledger.
|
||||
- **Backend runtime** (#9) - the node runs and raises; caught by the auto-run
|
||||
tier's two-way baseline, which isolates each node (single-node re-run) so
|
||||
the failing node names itself; a chain that fails because its synthesized
|
||||
producer raised still carries that producer's name in the backend's error
|
||||
event.
|
||||
|
||||
## What is already proven (the falsification pass)
|
||||
|
||||
Before writing this plan, every break in the matrix was applied one at a time
|
||||
against a real backend and the tier was confirmed to catch and name it. That is
|
||||
where the "Exact red" column comes from. Two of those runs also corrected the
|
||||
suite itself, and those fixes are already committed on the suite branch:
|
||||
|
||||
- **Drag drop-resolution (#5)** was originally a _miss_: the curated drag test
|
||||
only targeted first-slot inputs, and a broken drop resolver falls back to the
|
||||
first compatible input (LinkConnector's drop-on-node path), so such a
|
||||
regression could not fail a first-slot-only pair. Fixed by adding the
|
||||
second-slot anchor (`EmptyImage.IMAGE -> ImageBatch.image2`); the matrix red
|
||||
above is from the fixed test.
|
||||
- **Curated-run failure naming (#6)** originally reported `{}` for a backend
|
||||
validation rejection. Fixed by capturing and flattening the backend
|
||||
`node_errors`; the matrix now shows the named nodes and input.
|
||||
- **Boot-time console noise** was confirmed out of the ledger's window by
|
||||
design (documented in ARCHITECTURE.md section 10 and README), backstopped by
|
||||
the startup zero-visible-errors check.
|
||||
|
||||
## Honest caveat: local full runs and machine load
|
||||
|
||||
All tests share ONE backend, locally and on CI alike (the CI job is
|
||||
deliberately unsharded), and the suite enforces per-test backend isolation
|
||||
itself: every test's
|
||||
afterEach drains the backend to idle (`drainBackendToIdle`), the auto-run tier
|
||||
waits out a still-draining prior execution instead of hard-failing, and the
|
||||
non-executing tiers filter a foreign execution's async console lines
|
||||
(`isForeignExecutionNoise`). This fixed the cross-test bleed class outright: a
|
||||
test can no longer leave work running for the next test to inherit, and the
|
||||
mount/persistence/wiring tiers no longer catch a neighbor's execution errors.
|
||||
|
||||
What remains genuinely load-sensitive is execution TIMING, not isolation: on a
|
||||
machine that is busy with other work, slow CPU nodes can exceed even the raised
|
||||
budgets (20s batch, 60s single re-run), which flips their classification and
|
||||
trips the two-way cannotRunAlone baseline. That is the baseline doing its job
|
||||
against an environment that changed under it, not a suite defect. Therefore:
|
||||
|
||||
- Use **CI** as the pass/fail oracle for the Detection Proof (a fresh backend
|
||||
on an unloaded runner, every run).
|
||||
- A local full run is meaningful on an otherwise-idle machine; do not run it
|
||||
concurrently with heavy local work and expect baseline-exact results.
|
||||
|
||||
## Building the proof PR
|
||||
|
||||
1. Branch off the suite branch: `git checkout -b nathaniel/detection-proof nathaniel/custom-node-e2e-suite`.
|
||||
2. One commit per matrix row, each breaking one surface, stacked so all breaks
|
||||
are live at HEAD at once (not reverted between commits - the goal is to see
|
||||
every surface broken together, and the `Tests Custom Nodes` job reds across
|
||||
every tier in one run). FE-mode rows (1-7) are a direct `src/` edit carrying
|
||||
an inline comment in the changed file:
|
||||
`// DETECTION PROOF (row N, surface): recreates <FE-xxx / PR #12279>. Expected: <tier> red <message>.`
|
||||
3. Pack-mode rows (8-10) are delivered by one CI step
|
||||
(`DETECTION PROOF - break packs`, on this branch only) that patches each
|
||||
cloned pack in place right after install. Each patch asserts it landed
|
||||
(`grep`, fails the job otherwise) so a silent no-op cannot fake a pass. The
|
||||
step is fenced to this never-merge branch and must never be ported to a real
|
||||
suite branch.
|
||||
4. Commit message names the surface, e.g.
|
||||
`detection-proof: break mount (v2 Vue renderer) - drops the int widget mapping`.
|
||||
5. Open the PR against the suite branch (not main) with the correlation matrix as
|
||||
the description and a bold header: **This PR must never merge. Every commit is
|
||||
a deliberate break; green would mean the gate missed a regression.**
|
||||
6. Let CI run on HEAD. With every break live, the `Tests Custom Nodes` job reds
|
||||
across every tier in one run. Attribute a red to its cause via the labelled
|
||||
comment on the matching `src/` file (rows 1-7) or in the CI break step
|
||||
(rows 8-10); checking out commit N (which contains breaks 1..N) narrows it
|
||||
further.
|
||||
|
||||
## References
|
||||
|
||||
- Linear "Custom Node Bugs" project issues (symptoms): FE-841, FE-627, FE-634,
|
||||
FE-630, FE-637, FE-629, FE-625, FE-632, FE-751, FE-489, FE-491, FE-492.
|
||||
- The defaultInput migration regression (widgets revert to socket-only on reload) and its open fix: Comfy-Org/ComfyUI_frontend #12279.
|
||||
- Suite-discovered bugs with no upstream ticket yet (betterCombos `typeof null`,
|
||||
WAS infinite-loop, WAS pip-install-in-execute, KJ ImageGridtoBatch min) are
|
||||
pending upstream filing.
|
||||
132
browser_tests/tests/customNodes/README.md
Normal file
132
browser_tests/tests/customNodes/README.md
Normal file
@@ -0,0 +1,132 @@
|
||||
# Custom-node regression suite
|
||||
|
||||
Proves community custom-node packs work against this frontend across both
|
||||
renderers: nodes register, render under LiteGraph (canvas) AND Vue Nodes 2.0
|
||||
(DOM), and execute real workflows end to end. Manifest-driven: adding a pack
|
||||
is one JSON row, no new test code.
|
||||
|
||||
System design, data flow, and the reasoning behind every invariant:
|
||||
[ARCHITECTURE.md](ARCHITECTURE.md). Onboarding a new pack:
|
||||
[ADDING_CUSTOM_NODES.md](ADDING_CUSTOM_NODES.md).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. A ComfyUI backend on `127.0.0.1:8288` with every manifest pack (the
|
||||
`pack` entries in `browser_tests/fixtures/data/customNodeManifest.json`)
|
||||
and ComfyUI_devtools
|
||||
installed. Launch it with `--multi-user` (the repo-wide browser-test
|
||||
prerequisite; the fixture writes per-worker user settings and the suite
|
||||
depends on them landing), `--cache-none` (repeat runs must re-execute
|
||||
every node or the executed-set check fails honestly with `PARTIAL`), and
|
||||
with `browser_tests/assets/plain_video.mp4` copied into its `input/` dir.
|
||||
2. The dev server proxying that backend:
|
||||
`DEV_SERVER_COMFYUI_URL=http://127.0.0.1:8288 pnpm dev`
|
||||
|
||||
## Running
|
||||
|
||||
| Script | What it does |
|
||||
| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| `pnpm test:custom-nodes` | whole suite headless against the Vite dev server - the fast local loop for suite-code iteration. NOT the gate: the dev server never loads pack frontend JS (see Gotchas) |
|
||||
| `pnpm test:custom-nodes:ci` | whole suite headless against the backend-served BUILT frontend - the gate-equivalent run (every tier passes, zero skips). Requires a backend serving the built dist on :8188 (a separate endpoint from the :8288 dev-proxy backend in Prerequisites); set `PLAYWRIGHT_TEST_URL` if yours differs |
|
||||
| `pnpm test:custom-nodes:watch` | headed slow-motion run of the browser tiers, hands-off watching |
|
||||
| `pnpm test:custom-nodes:debug` | step through the browser tiers in the Playwright Inspector (F10 step, F8 resume) |
|
||||
| `pnpm test:custom-nodes:impact-render` | Impact nodes render in both renderers (Inspector) |
|
||||
| `pnpm test:custom-nodes:impact-run` | Impact group workflow executes on the backend (Inspector) |
|
||||
| `pnpm test:custom-nodes:vhs-render` | VHS nodes render in both renderers (Inspector) |
|
||||
| `pnpm test:custom-nodes:vhs-run` | VHS decodes a real video through its node chain (Inspector) |
|
||||
| `pnpm test:custom-nodes:connectivity` | slot/type contract: type-paired links + real slot drags in both renderers (Inspector) |
|
||||
| `pnpm test:custom-nodes:self-check` | watches the harness catch a deliberate execution error |
|
||||
|
||||
Example - watch the VHS video-decode run step by step:
|
||||
|
||||
```bash
|
||||
pnpm test:custom-nodes:vhs-run
|
||||
```
|
||||
|
||||
Two windows open: the app under test and the Playwright Inspector. Press F10
|
||||
to execute one robot action at a time (workflow loads, queue fires, backend
|
||||
decodes the video), F8 to run to the end. While paused, look but do not click
|
||||
inside the app window - your clicks change the state the next assertion
|
||||
checks.
|
||||
|
||||
Any `-g` pattern works against the generic scripts, e.g.
|
||||
`pnpm test:custom-nodes:debug -g "Impact-Pack.*T0"`.
|
||||
|
||||
## What the tests assert
|
||||
|
||||
- **T0 load**: pack nodes are registered in `/object_info`, added to a
|
||||
cleared graph, counted exactly, and each added node's own `[data-node-id]`
|
||||
element mounts under Vue Nodes 2.0. Both renderer passes - unless the pack
|
||||
declares `vueNodesCompatible: false` in the manifest (evidence required;
|
||||
see [ADDING_CUSTOM_NODES.md](ADDING_CUSTOM_NODES.md)), in which case its tests run their
|
||||
LiteGraph-canvas assertions only. Never a skip.
|
||||
- **T1 run**: the manifest workflow is loaded and queued; the backend's
|
||||
`executing` event stream must contain every expected node id, and the run
|
||||
must end in `execution_success`.
|
||||
- **Every-node tiers** (`allNodes.spec.ts`): the pack's FULL node list,
|
||||
discovered live from `/object_info`, is exercised with zero
|
||||
configuration - every registered node mounts in both renderers (chunked
|
||||
at an empirically measured batch size), survives a serialize/configure
|
||||
save-reload round-trip, and executes for real on the backend when
|
||||
self-sufficient (all required inputs are widgets with valid defaults).
|
||||
Nodes that cannot run alone are classified and logged
|
||||
(`NEEDS_WIRES` / `NEEDS_MODELS` / `NO_OBSERVABLE_OUTPUT` / rejected-at-validation),
|
||||
never silently dropped; the documented exception ledgers (see
|
||||
[ADDING_CUSTOM_NODES.md](ADDING_CUSTOM_NODES.md)) carry a written mechanism for every
|
||||
escape hatch.
|
||||
- **connectivity (contract)**: wiring-only, no execution. A
|
||||
type-pairing generator (`fixtures/customNode/typePairing.ts`) indexes
|
||||
`/object_info` producers/consumers and plans one representative typed edge
|
||||
per slot (wildcard `*` slots excluded - they bypass the real type compare
|
||||
and prove nothing). Each planned edge must connect through the real
|
||||
`isValidConnection` veto, then survive `serialize()` -> `configure()` and
|
||||
appear in `graphToPrompt()` output. A curated subset is additionally
|
||||
dragged for real - slot dot to slot dot - under both renderers. Orphan
|
||||
types (no partner in the corpus) are reported, never fake-failed. One
|
||||
representative edge per slot bounds cost; it does not prove all pairs.
|
||||
- **Zero visible errors**: the mount, persistence, connectivity, core
|
||||
smoke, and curated workflow tests assert the app's error surfaces (error
|
||||
overlay, error dialog, node render errors, error toasts) are absent at
|
||||
start and after every pass - green means a human watching those runs sees
|
||||
no errors. Two deliberate exceptions: the auto-run execution tier
|
||||
provokes expected failures (baselined cannotRunAlone nodes surface as
|
||||
real error UI by design), and the self-check inverts the invariant - it
|
||||
forces a real execution error and asserts the overlay IS visible, proving
|
||||
the selectors stay live.
|
||||
- **Console-error window**: the console/page-error ledger (curated run,
|
||||
save/reload) starts collecting inside each tier, so it covers the tier's
|
||||
own actions - load, run, wire, save. Pure console noise a pack logs at
|
||||
app boot, before the first tier action, is out of that window by design:
|
||||
the shared app fixture navigates once at setup, so boot output predates
|
||||
any per-pack collector. Boot breakage that MATTERS still fails the gate -
|
||||
the zero-visible-errors check runs at startup and catches any boot error
|
||||
that reaches a visible surface; only invisible, functionally-inert boot
|
||||
console noise (the ledger's whole reason to exist) is out of scope.
|
||||
|
||||
## Adding a pack
|
||||
|
||||
One manifest row plus one small workflow JSON - no new test code. The
|
||||
authoritative step-by-step process (verifying the pack's real node keys,
|
||||
authoring the run workflow, the `vueNodesCompatible` evidence rule, what CI
|
||||
does with the row) lives in [ADDING_CUSTOM_NODES.md](ADDING_CUSTOM_NODES.md). Follow it
|
||||
exactly; the traps it lists all shipped in real packs.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **Pack frontend JS does not load under the Vite dev server.** The dev
|
||||
server's `/extensions` endpoint lists core extensions only, so nodes render
|
||||
vanilla locally even when the backend has the packs installed. CI serves
|
||||
the built frontend from the backend, where every pack's JS loads and can
|
||||
restyle nodes, rebuild widgets, or inject page chrome. Before pushing
|
||||
changes that could interact with pack JS, reproduce CI locally:
|
||||
`pnpm build`, relaunch the backend with `--front-end-root <repo>/dist`,
|
||||
and run the suite with `PLAYWRIGHT_TEST_URL` pointed at the backend.
|
||||
- Do not run with `--trace on` against system Chrome
|
||||
(`playwright.chrome.config.ts` pins trace off): the trace recorder crashes
|
||||
pages under the branded Chrome channel and every test reports a bogus 15s
|
||||
timeout.
|
||||
- In a git worktree whose `node_modules` is symlinked from another checkout,
|
||||
prefix scripts with `pnpm --config.verify-deps-before-run=false ...` to
|
||||
skip pnpm's auto-install check.
|
||||
- First run against a cold dev server can exceed the 15s per-test setup
|
||||
budget while Vite compiles; just run again.
|
||||
1059
browser_tests/tests/customNodes/allNodes.spec.ts
Normal file
1059
browser_tests/tests/customNodes/allNodes.spec.ts
Normal file
File diff suppressed because it is too large
Load Diff
238
browser_tests/tests/customNodes/autoRun.pure.spec.ts
Normal file
238
browser_tests/tests/customNodes/autoRun.pure.spec.ts
Normal file
@@ -0,0 +1,238 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import {
|
||||
batchAutoRunnable,
|
||||
classifyAutoRunnable,
|
||||
planAutoRuns
|
||||
} from '@e2e/fixtures/customNode/autoRun'
|
||||
|
||||
const SYNTH = new Set([
|
||||
'IMAGE',
|
||||
'LATENT',
|
||||
'MASK',
|
||||
'INT',
|
||||
'FLOAT',
|
||||
'STRING',
|
||||
'BOOLEAN',
|
||||
'*'
|
||||
])
|
||||
|
||||
test.describe('autoRun classifier', () => {
|
||||
test('widget-only node with outputs is runnable via a PreviewAny sink', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'IntConstant',
|
||||
{
|
||||
input: { required: { value: ['INT', { default: 0 }] } },
|
||||
output: ['INT'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('AUTO_RUNNABLE')
|
||||
expect(verdict.needsPreviewSink).toBe(true)
|
||||
})
|
||||
|
||||
test('widget-only OUTPUT_NODE runs standalone', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'ShowValue',
|
||||
{
|
||||
input: {
|
||||
required: {
|
||||
text: ['STRING', {}],
|
||||
mode: [['raw value', 'tensor shape']]
|
||||
}
|
||||
},
|
||||
output: [],
|
||||
output_node: true
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('AUTO_RUNNABLE')
|
||||
expect(verdict.needsPreviewSink).toBe(false)
|
||||
})
|
||||
|
||||
test('synthesizable sockets make a node CHAINABLE with its socket list', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'MaskComposite',
|
||||
{
|
||||
input: {
|
||||
required: {
|
||||
destination: ['MASK'],
|
||||
source: ['MASK'],
|
||||
x: ['INT', { default: 0 }],
|
||||
operation: [['multiply', 'add']]
|
||||
}
|
||||
},
|
||||
output: ['MASK'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('CHAINABLE')
|
||||
expect(verdict.requiredSockets).toEqual([
|
||||
{ name: 'destination', type: 'MASK' },
|
||||
{ name: 'source', type: 'MASK' }
|
||||
])
|
||||
expect(verdict.needsPreviewSink).toBe(true)
|
||||
})
|
||||
|
||||
test('a socket with no model-free producer means NEEDS_WIRES', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'VaeDecode',
|
||||
{
|
||||
input: { required: { samples: ['LATENT'], vae: ['VAE'] } },
|
||||
output: ['IMAGE'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('NEEDS_WIRES')
|
||||
expect(verdict.reason).toContain('vae')
|
||||
})
|
||||
|
||||
test('forceInput STRING is a socket but STRING is synthesizable', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'TextSink',
|
||||
{
|
||||
input: { required: { text: ['STRING', { forceInput: true }] } },
|
||||
output: ['STRING'],
|
||||
output_node: true
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('CHAINABLE')
|
||||
expect(verdict.requiredSockets).toEqual([{ name: 'text', type: 'STRING' }])
|
||||
})
|
||||
|
||||
test('an empty required combo means NEEDS_MODELS', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'CheckpointLoader',
|
||||
{
|
||||
input: { required: { ckpt_name: [[]] } },
|
||||
output: ['MODEL'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('NEEDS_MODELS')
|
||||
expect(verdict.reason).toContain('ckpt_name')
|
||||
})
|
||||
|
||||
// Census-derived: transformed (V2-schema) defs carry combos as the string
|
||||
// 'COMBO' with options in the opts object - the classifier must not read
|
||||
// that as an unproducible socket type.
|
||||
test('a V2-form combo with options is a widget', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'LatentConcatLike',
|
||||
{
|
||||
input: {
|
||||
required: {
|
||||
dim: ['COMBO', { multiselect: false, options: ['x', '-x', 'y'] }]
|
||||
}
|
||||
},
|
||||
output: ['LATENT'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('AUTO_RUNNABLE')
|
||||
})
|
||||
|
||||
// Census-derived (DevToolsNodeWithOutputCombo.subset_options): a combo
|
||||
// carrying forceInput is a socket in ANY form - no widget materializes,
|
||||
// so its option list cannot satisfy the input.
|
||||
test('forceInput on a list-form combo is a socket, not a widget', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'OutputComboLike',
|
||||
{
|
||||
input: {
|
||||
required: { subset_options: [['A', 'B'], { forceInput: true }] }
|
||||
},
|
||||
output: ['COMBO'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('NEEDS_WIRES')
|
||||
expect(verdict.reason).toContain('subset_options')
|
||||
})
|
||||
|
||||
test('a V2-form combo with no static options means NEEDS_MODELS', () => {
|
||||
for (const spec of [
|
||||
['COMBO', { multiselect: false, options: [] }],
|
||||
['COMBO', { remote: { route: '/internal/files/output' } }]
|
||||
]) {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'LoadImageOutputLike',
|
||||
{
|
||||
input: { required: { image: spec } },
|
||||
output: ['IMAGE'],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('NEEDS_MODELS')
|
||||
expect(verdict.reason).toContain('image')
|
||||
}
|
||||
})
|
||||
|
||||
test('no outputs and not an OUTPUT_NODE means NO_OBSERVABLE_OUTPUT', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'SideEffectOnly',
|
||||
{
|
||||
input: { required: { value: ['INT', {}] } },
|
||||
output: [],
|
||||
output_node: false
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('NO_OBSERVABLE_OUTPUT')
|
||||
})
|
||||
|
||||
test('optional socket inputs do not block auto-running', () => {
|
||||
const verdict = classifyAutoRunnable(
|
||||
'MathWithOptionalAny',
|
||||
{
|
||||
input: {
|
||||
required: { expression: ['STRING', {}] },
|
||||
optional: { a: ['*'] }
|
||||
},
|
||||
output: ['INT', 'FLOAT'],
|
||||
output_node: true
|
||||
},
|
||||
SYNTH
|
||||
)
|
||||
expect(verdict.verdict).toBe('AUTO_RUNNABLE')
|
||||
})
|
||||
|
||||
test('planAutoRuns validates producers against defs and batches runnables', () => {
|
||||
const defs = {
|
||||
A: {
|
||||
input: { required: { v: ['INT', {}] } },
|
||||
output: ['INT'],
|
||||
output_node: false
|
||||
},
|
||||
B: {
|
||||
input: { required: { x: ['SEGS'] } },
|
||||
output: ['SEGS'],
|
||||
output_node: false
|
||||
},
|
||||
C: {
|
||||
input: { required: { img: ['IMAGE'] } },
|
||||
output: ['IMAGE'],
|
||||
output_node: false
|
||||
},
|
||||
EmptyImage: { input: { required: {} }, output: ['IMAGE'] }
|
||||
}
|
||||
const verdicts = planAutoRuns(defs, ['A', 'B', 'C'])
|
||||
expect(verdicts.map((verdict) => verdict.verdict)).toEqual([
|
||||
'AUTO_RUNNABLE',
|
||||
'NEEDS_WIRES',
|
||||
'CHAINABLE'
|
||||
])
|
||||
const batches = batchAutoRunnable(verdicts, 1)
|
||||
expect(batches.map((batch) => batch[0].key)).toEqual(['A', 'C'])
|
||||
})
|
||||
})
|
||||
546
browser_tests/tests/customNodes/connectivity.spec.ts
Normal file
546
browser_tests/tests/customNodes/connectivity.spec.ts
Normal file
@@ -0,0 +1,546 @@
|
||||
import type { Page } from '@playwright/test'
|
||||
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import {
|
||||
customNodeSuiteSettings,
|
||||
dismissTemplatesDialog,
|
||||
drainBackendToIdle
|
||||
} from '@e2e/fixtures/utils/customNodeSuite'
|
||||
import { isForeignExecutionNoise } from '@e2e/fixtures/customNode/consoleErrorLedger'
|
||||
import { loadManifest } from '@e2e/fixtures/customNode/manifest'
|
||||
import type {
|
||||
ConnectivityOutcome,
|
||||
PlannedPair,
|
||||
RawNodeDef
|
||||
} from '@e2e/fixtures/customNode/typePairing'
|
||||
import {
|
||||
isWildcard,
|
||||
normalizeNodeDefs,
|
||||
planPairs
|
||||
} from '@e2e/fixtures/customNode/typePairing'
|
||||
import { collectConsoleErrors } from '@e2e/fixtures/utils/consoleErrorCollector'
|
||||
import { errorSurfaces } from '@e2e/fixtures/utils/errorSurfaces'
|
||||
|
||||
const CORE_PROOF_NODE_COUNT = 16
|
||||
// A node may legitimately veto a wiring via onConnectInput; committed
|
||||
// entries here must name the veto. Green means actual rejections are a
|
||||
// subset of this list.
|
||||
const CONNECT_REJECTED_ALLOWLIST: string[] = [
|
||||
// pysssss MathExpression only accepts INT/FLOAT-producing links into its
|
||||
// expression variables; its JS vetoes text-list producers.
|
||||
'AddTextPrefix.texts -> MathExpression|pysssss.expression'
|
||||
]
|
||||
// A pack's own serialize/configure hooks may drop links it manages itself
|
||||
// (reproducible manually: wire, save, reload - link gone). Pack behavior on
|
||||
// record, not frontend regressions.
|
||||
const ROUNDTRIP_LOST_ALLOWLIST: string[] = [
|
||||
// rgthree SDXL Power Prompt rebuilds its dimension widget-inputs during
|
||||
// configure and drops inbound links to them.
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Positive (rgthree).target_width',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Positive (rgthree).target_height',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Positive (rgthree).crop_width',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Positive (rgthree).crop_height',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Simple / Negative (rgthree).target_width',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Simple / Negative (rgthree).target_height',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Simple / Negative (rgthree).crop_width',
|
||||
'BatchCount+.INT -> SDXL Power Prompt - Simple / Negative (rgthree).crop_height',
|
||||
// VHS_SelectLatest rebuilds its dynamic slots on configure, detaching
|
||||
// links on both its inputs and outputs.
|
||||
'AddTextPrefix.texts -> VHS_SelectLatest.filename_prefix',
|
||||
'AddTextPrefix.texts -> VHS_SelectLatest.filename_postfix',
|
||||
'VHS_SelectLatest.Filename -> AddLabel.font_color'
|
||||
]
|
||||
|
||||
test.use({ initialSettings: customNodeSuiteSettings })
|
||||
|
||||
test.beforeEach(async ({ comfyPage }) => {
|
||||
await dismissTemplatesDialog(comfyPage)
|
||||
})
|
||||
|
||||
// Leave the shared backend idle so the next test starts clean (drainBackendToIdle).
|
||||
test.afterEach(async ({ comfyPage }) => {
|
||||
// The drain is a no-op when the queue is already idle, so it costs
|
||||
// ~nothing in the common path; the 10s ceiling only bounds a genuinely
|
||||
// busy backend. A backend still busy past it is wedged, and the auto-run
|
||||
// tier's 150s guard surfaces that with the restart diagnostic.
|
||||
await drainBackendToIdle(comfyPage.page, 10_000)
|
||||
})
|
||||
|
||||
async function expectNoVisibleErrors(
|
||||
page: Page,
|
||||
context: string
|
||||
): Promise<void> {
|
||||
for (const [surface, locator] of Object.entries(errorSurfaces(page)))
|
||||
await expect(locator, `${context}: ${surface}`).toHaveCount(0)
|
||||
}
|
||||
|
||||
function concrete(slot: { type: string }): boolean {
|
||||
return !isWildcard(slot.type)
|
||||
}
|
||||
|
||||
function isEntryInstalled(
|
||||
nodeTypes: Set<string>,
|
||||
entry: { expectedNodes: string[] }
|
||||
): boolean {
|
||||
return entry.expectedNodes.every((type) => nodeTypes.has(type))
|
||||
}
|
||||
|
||||
const connectivityEntries = loadManifest().filter((entry) =>
|
||||
entry.tiers.includes('connectivity')
|
||||
)
|
||||
|
||||
test('connectivity: every type-paired link survives model, serialize, and prompt round-trips', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
test.setTimeout(120_000)
|
||||
const defs = (await comfyPage.page.evaluate(() =>
|
||||
window.app!.api.getNodeDefs()
|
||||
)) as unknown as Record<string, RawNodeDef>
|
||||
const nodes = normalizeNodeDefs(defs)
|
||||
|
||||
// Pack-specific expectations apply only where the pack is installed; on a
|
||||
// backend without it (e.g. a generic CI runner) the core sweep still runs
|
||||
// and the absence is reported, never fake-failed or fake-passed.
|
||||
const nodeTypes = new Set(nodes.map((node) => node.type))
|
||||
const installedEntries = connectivityEntries.filter((entry) =>
|
||||
isEntryInstalled(nodeTypes, entry)
|
||||
)
|
||||
for (const entry of connectivityEntries)
|
||||
if (!installedEntries.includes(entry))
|
||||
console.log(`connectivity: ${entry.pack} not installed on this backend`)
|
||||
// Corpus = every node the installed packs register, from the live backend.
|
||||
const installedPacks = new Set(installedEntries.map((entry) => entry.pack))
|
||||
const packTypes = nodes
|
||||
.filter((node) => installedPacks.has(node.pack))
|
||||
.map((node) => node.type)
|
||||
const coreProof = nodes
|
||||
.filter(
|
||||
(node) =>
|
||||
node.pack === 'core' &&
|
||||
node.inputs.some(concrete) &&
|
||||
node.outputs.some(concrete)
|
||||
)
|
||||
.map((node) => node.type)
|
||||
.sort()
|
||||
.slice(0, CORE_PROOF_NODE_COUNT)
|
||||
const plan = planPairs(nodes, [...packTypes, ...coreProof])
|
||||
|
||||
expect(plan.pairs.length, 'pairing produced no edges').toBeGreaterThan(0)
|
||||
console.log(
|
||||
`connectivity plan: ${plan.pairs.length} pairs, ${plan.orphans.length} orphan slots, ${plan.wildcards.length} wildcard + ${plan.combos.length} combo slots (excluded by design), ${plan.unknownShapes.length} unknown-shape slots (recorded: ${plan.unknownShapes.join('; ') || 'none'})`
|
||||
)
|
||||
|
||||
for (const entry of installedEntries) {
|
||||
expect(
|
||||
plan.pairs.some(
|
||||
(pair) =>
|
||||
pair.producer.pack === entry.pack || pair.consumer.pack === entry.pack
|
||||
),
|
||||
`${entry.pack} contributes no pairs - corpus or pack attribution broke`
|
||||
).toBe(true)
|
||||
}
|
||||
|
||||
// The breadth sweep runs under one renderer by design: it exercises
|
||||
// graph-API link creation, the real isValidConnection veto, and
|
||||
// serialize/configure survival - all renderer-independent paths (widget
|
||||
// values and links flow through the same stores in both renderers). The
|
||||
// curated drag test below covers real pointer wiring under BOTH renderers.
|
||||
const consoleErrors = collectConsoleErrors(comfyPage.page)
|
||||
const results = await runPairsInPage(comfyPage.page, plan.pairs)
|
||||
consoleErrors.stop()
|
||||
// Deliberately raw, not routed through the pack console ledger
|
||||
// (consoleErrorLedger.ts): the sweep holds zero console errors without
|
||||
// exceptions today, and the stricter contract catches noise the moment
|
||||
// wiring provokes it. If a ledgered pattern ever fires here, filter
|
||||
// through unallowlistedErrors with the pack taken from the offending
|
||||
// pair's nodes (the sweep is cross-pack), instead of silently
|
||||
// loosening this assert. The wiring sweep queues no prompts, so a
|
||||
// prompt-execution error here is a prior tier's async stray, not this
|
||||
// test's (isForeignExecutionNoise; ARCHITECTURE section 9 principle).
|
||||
expect(
|
||||
consoleErrors.errors.filter((error) => !isForeignExecutionNoise(error)),
|
||||
'console errors during breadth sweep'
|
||||
).toEqual([])
|
||||
|
||||
const widgetOnly = results.filter(
|
||||
(result) =>
|
||||
result.outcome ===
|
||||
('WIDGET_ONLY_ON_INSTANCE' satisfies ConnectivityOutcome)
|
||||
)
|
||||
if (widgetOnly.length > 0)
|
||||
console.log(
|
||||
`connectivity sweep: ${widgetOnly.length} pair(s) excluded - pack JS made the declared input widget-only: ${widgetOnly.map((result) => result.key).join('; ')}`
|
||||
)
|
||||
const failures = results.filter(
|
||||
(result) =>
|
||||
result.outcome !== ('PASS' satisfies ConnectivityOutcome) &&
|
||||
result.outcome !==
|
||||
('WIDGET_ONLY_ON_INSTANCE' satisfies ConnectivityOutcome) &&
|
||||
!(
|
||||
result.outcome === ('CONNECT_REJECTED' satisfies ConnectivityOutcome) &&
|
||||
CONNECT_REJECTED_ALLOWLIST.includes(result.key)
|
||||
) &&
|
||||
!(
|
||||
result.outcome === ('ROUNDTRIP_LOST' satisfies ConnectivityOutcome) &&
|
||||
ROUNDTRIP_LOST_ALLOWLIST.includes(result.key)
|
||||
)
|
||||
)
|
||||
const passed = results.filter((result) => result.outcome === 'PASS').length
|
||||
console.log(`connectivity sweep: ${passed}/${results.length} pairs PASS`)
|
||||
expect(failures, JSON.stringify(failures, null, 1)).toEqual([])
|
||||
expect(passed).toBeGreaterThan(0)
|
||||
// Two-way guard, same discipline as cannotRunAlone: every allowlisted key
|
||||
// must still be OBSERVED failing in its recorded way. An entry whose pair
|
||||
// now passes (or is no longer even planned) is stale and would silently
|
||||
// hide the fixed bug behind it. On a partially-installed local backend an
|
||||
// absent key only logs; CI installs every pack, so it always enforces.
|
||||
const outcomeByKey = new Map(
|
||||
results.map((result) => [result.key, result.outcome])
|
||||
)
|
||||
const allPacksInstalled =
|
||||
installedEntries.length === connectivityEntries.length
|
||||
const staleEntries: string[] = []
|
||||
for (const [allowlist, expected] of [
|
||||
[CONNECT_REJECTED_ALLOWLIST, 'CONNECT_REJECTED'],
|
||||
[ROUNDTRIP_LOST_ALLOWLIST, 'ROUNDTRIP_LOST']
|
||||
] as const)
|
||||
for (const key of allowlist) {
|
||||
const observed = outcomeByKey.get(key)
|
||||
if (observed === undefined && !allPacksInstalled) {
|
||||
console.log(
|
||||
`allowlist entry not observed (pack not installed here): ${key}`
|
||||
)
|
||||
continue
|
||||
}
|
||||
if (observed !== expected)
|
||||
staleEntries.push(
|
||||
`${key}: expected ${expected}, observed ${observed ?? 'nothing'} - remove the stale entry`
|
||||
)
|
||||
}
|
||||
expect(staleEntries, 'stale allowlist entries').toEqual([])
|
||||
await expectNoVisibleErrors(comfyPage.page, 'after breadth sweep')
|
||||
})
|
||||
|
||||
// First planned pair whose slots both exist on real instances (pack JS can
|
||||
// rebuild declared inputs as widget-only controls).
|
||||
function firstMaterializedPair(
|
||||
page: Page,
|
||||
pairs: PlannedPair[]
|
||||
): Promise<PlannedPair | null> {
|
||||
return page.evaluate((pairsInPage) => {
|
||||
for (const pair of pairsInPage) {
|
||||
const producer = window.LiteGraph!.createNode(pair.producer.nodeType)
|
||||
const consumer = window.LiteGraph!.createNode(pair.consumer.nodeType)
|
||||
const outFound = producer?.outputs.some(
|
||||
(slot) => slot.name === pair.producer.slotName
|
||||
)
|
||||
const inFound = consumer?.inputs.some(
|
||||
(slot) => slot.name === pair.consumer.slotName
|
||||
)
|
||||
if (outFound && inFound) return pair
|
||||
}
|
||||
return null
|
||||
}, pairs)
|
||||
}
|
||||
|
||||
// The self-check below runs THIS SAME executor on poisoned pairs; if it stops
|
||||
// being able to reject, every green sweep above is meaningless.
|
||||
function runPairsInPage(
|
||||
page: Page,
|
||||
pairs: PlannedPair[]
|
||||
): Promise<Array<{ key: string; outcome: string; detail?: string }>> {
|
||||
return page.evaluate(async (pairsInPage) => {
|
||||
const graph = window.app!.graph
|
||||
const report: Array<{
|
||||
key: string
|
||||
outcome: string
|
||||
detail?: string
|
||||
}> = []
|
||||
for (const pair of pairsInPage) {
|
||||
const key = `${pair.producer.nodeType}.${pair.producer.slotName} -> ${pair.consumer.nodeType}.${pair.consumer.slotName}`
|
||||
try {
|
||||
graph.clear()
|
||||
const producer = window.LiteGraph!.createNode(pair.producer.nodeType)
|
||||
const consumer = window.LiteGraph!.createNode(pair.consumer.nodeType)
|
||||
if (!producer || !consumer) {
|
||||
report.push({
|
||||
key,
|
||||
outcome: 'SLOT_CONTRACT_MISMATCH',
|
||||
detail: 'createNode returned null for a registered type'
|
||||
})
|
||||
continue
|
||||
}
|
||||
graph.add(producer)
|
||||
graph.add(consumer)
|
||||
const outIndex = producer.outputs.findIndex(
|
||||
(slot) => slot.name === pair.producer.slotName
|
||||
)
|
||||
const inIndex = consumer.inputs.findIndex(
|
||||
(slot) => slot.name === pair.consumer.slotName
|
||||
)
|
||||
if (outIndex < 0 || inIndex < 0) {
|
||||
// Pack JS may rebuild a declared input as widget-only (rgthree
|
||||
// Seed.seed) - excluded; missing as slot AND widget stays a fail.
|
||||
const widgetOnly =
|
||||
outIndex >= 0 &&
|
||||
(consumer.widgets ?? []).some(
|
||||
(widget) => widget.name === pair.consumer.slotName
|
||||
)
|
||||
report.push({
|
||||
key,
|
||||
outcome: widgetOnly
|
||||
? 'WIDGET_ONLY_ON_INSTANCE'
|
||||
: 'SLOT_CONTRACT_MISMATCH',
|
||||
detail: `declared slot missing on instance (out=${outIndex}, in=${inIndex})`
|
||||
})
|
||||
continue
|
||||
}
|
||||
const link = producer.connect(outIndex, consumer, inIndex)
|
||||
if (!link || consumer.inputs[inIndex]?.link == null) {
|
||||
report.push({ key, outcome: 'CONNECT_REJECTED' })
|
||||
continue
|
||||
}
|
||||
const serialized = graph.serialize()
|
||||
graph.configure(serialized)
|
||||
const restored = graph.getNodeById(consumer.id)
|
||||
if (restored?.inputs?.[inIndex]?.link == null) {
|
||||
report.push({
|
||||
key,
|
||||
outcome: 'ROUNDTRIP_LOST',
|
||||
detail: 'serialize/configure dropped the link'
|
||||
})
|
||||
continue
|
||||
}
|
||||
const prompt = (await window.app!.graphToPrompt()) as {
|
||||
output?: Record<string, { inputs?: Record<string, unknown> }>
|
||||
}
|
||||
const promptInput =
|
||||
prompt.output?.[String(consumer.id)]?.inputs?.[pair.consumer.slotName]
|
||||
if (!Array.isArray(promptInput)) {
|
||||
report.push({
|
||||
key,
|
||||
outcome: 'ROUNDTRIP_LOST',
|
||||
detail: 'link missing from graphToPrompt output'
|
||||
})
|
||||
continue
|
||||
}
|
||||
report.push({ key, outcome: 'PASS' })
|
||||
} catch (error) {
|
||||
report.push({
|
||||
key,
|
||||
outcome: 'SLOT_CONTRACT_MISMATCH',
|
||||
detail: `threw: ${String(error)}`
|
||||
})
|
||||
}
|
||||
}
|
||||
graph.clear()
|
||||
return report
|
||||
}, pairs)
|
||||
}
|
||||
|
||||
test('connectivity self-check: the executor rejects broken pairs', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
const slot = (nodeType: string, slotName: string, slotType: string) => ({
|
||||
nodeType,
|
||||
pack: 'core',
|
||||
slotName,
|
||||
slotType
|
||||
})
|
||||
const results = await runPairsInPage(comfyPage.page, [
|
||||
{
|
||||
producer: slot('CheckpointLoaderSimple', 'MODEL', 'MODEL'),
|
||||
consumer: slot('KSampler', 'latent_image', 'LATENT')
|
||||
},
|
||||
{
|
||||
producer: slot('EmptyLatentImage', 'LATENT', 'LATENT'),
|
||||
consumer: slot('KSampler', 'does_not_exist', 'LATENT')
|
||||
}
|
||||
])
|
||||
expect(results.map((result) => result.outcome)).toEqual([
|
||||
'CONNECT_REJECTED',
|
||||
'SLOT_CONTRACT_MISMATCH'
|
||||
])
|
||||
})
|
||||
|
||||
test('connectivity drags: curated slot-to-slot wires connect under both renderers', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
test.setTimeout(120_000)
|
||||
const defs = (await comfyPage.page.evaluate(() =>
|
||||
window.app!.api.getNodeDefs()
|
||||
)) as unknown as Record<string, RawNodeDef>
|
||||
const nodes = normalizeNodeDefs(defs)
|
||||
|
||||
// Native anchor pair plus one in-pack, link-typed pair per connectivity
|
||||
// pack (derived from the same generator the breadth sweep uses).
|
||||
const dragEdges: PlannedPair[] = [
|
||||
{
|
||||
producer: {
|
||||
nodeType: 'EmptyLatentImage',
|
||||
pack: 'core',
|
||||
slotName: 'LATENT',
|
||||
slotType: 'LATENT'
|
||||
},
|
||||
consumer: {
|
||||
nodeType: 'KSampler',
|
||||
pack: 'core',
|
||||
slotName: 'latent_image',
|
||||
slotType: 'LATENT'
|
||||
}
|
||||
},
|
||||
// Second-slot anchor: ImageBatch has two IMAGE inputs (image1, image2)
|
||||
// and we target the SECOND. A slot hit-test regression that falls back
|
||||
// to the first compatible input would land on image1, leaving image2
|
||||
// (the asserted index) unlinked - so this pair, unlike a first-slot
|
||||
// pair, actually discriminates a broken drop-to-slot resolution.
|
||||
{
|
||||
producer: {
|
||||
nodeType: 'EmptyImage',
|
||||
pack: 'core',
|
||||
slotName: 'IMAGE',
|
||||
slotType: 'IMAGE'
|
||||
},
|
||||
consumer: {
|
||||
nodeType: 'ImageBatch',
|
||||
pack: 'core',
|
||||
slotName: 'image2',
|
||||
slotType: 'IMAGE'
|
||||
}
|
||||
}
|
||||
]
|
||||
const nodeTypes = new Set(nodes.map((node) => node.type))
|
||||
for (const entry of connectivityEntries) {
|
||||
if (!isEntryInstalled(nodeTypes, entry)) {
|
||||
console.log(
|
||||
`connectivity drag: ${entry.pack} not installed on this backend`
|
||||
)
|
||||
continue
|
||||
}
|
||||
// Restrict the partner pool to the pack itself so the drag proves an
|
||||
// in-pack wiring; widget-backed primitive inputs render real slot dots
|
||||
// in Vue (verified empirically), so no slot type is excluded at plan time.
|
||||
const packPlan = planPairs(
|
||||
nodes.filter((node) => node.pack === entry.pack),
|
||||
entry.expectedNodes
|
||||
)
|
||||
expect(
|
||||
packPlan.pairs.length,
|
||||
`${entry.pack} has no in-pack draggable pair - drag coverage lost`
|
||||
).toBeGreaterThan(0)
|
||||
// The plan comes from object_info, but a pack's own JS can rebuild a
|
||||
// declared input as widget-only on the instance (rgthree's Seed does).
|
||||
// Drag the first pair whose slots actually materialize; a pack whose
|
||||
// every planned pair is customized away has no socket contract to drag.
|
||||
const inPack = await firstMaterializedPair(comfyPage.page, packPlan.pairs)
|
||||
if (!inPack) {
|
||||
console.log(
|
||||
`connectivity drag: ${entry.pack} planned pairs are widget-only on instances; drag not applicable`
|
||||
)
|
||||
continue
|
||||
}
|
||||
dragEdges.push(inPack)
|
||||
}
|
||||
|
||||
const vueIncompatiblePacks = new Set(
|
||||
connectivityEntries
|
||||
.filter((entry) => entry.vueNodesCompatible === false)
|
||||
.map((entry) => entry.pack)
|
||||
)
|
||||
for (const vueNodesEnabled of [false, true]) {
|
||||
const consoleErrors = collectConsoleErrors(comfyPage.page)
|
||||
await comfyPage.settings.setSetting(
|
||||
'Comfy.VueNodes.Enabled',
|
||||
vueNodesEnabled
|
||||
)
|
||||
|
||||
for (const edge of dragEdges) {
|
||||
if (vueNodesEnabled && vueIncompatiblePacks.has(edge.producer.pack)) {
|
||||
console.log(
|
||||
`connectivity drag: ${edge.producer.pack} declares vueNodesCompatible=false; Vue drag not applicable`
|
||||
)
|
||||
continue
|
||||
}
|
||||
await comfyPage.nodeOps.clearGraph()
|
||||
const producer = await comfyPage.nodeOps.addNode(
|
||||
edge.producer.nodeType,
|
||||
undefined,
|
||||
{ x: 150, y: 200 }
|
||||
)
|
||||
const consumer = await comfyPage.nodeOps.addNode(
|
||||
edge.consumer.nodeType,
|
||||
undefined,
|
||||
{ x: 700, y: 200 }
|
||||
)
|
||||
await comfyPage.nextFrame()
|
||||
|
||||
const [outIndex, inIndex] = await comfyPage.page.evaluate(
|
||||
([producerId, consumerId, outName, inName]) => {
|
||||
const byId = (id: string) =>
|
||||
window.app!.graph.nodes.find((node) => String(node.id) === id)!
|
||||
const src = byId(producerId)
|
||||
const dst = byId(consumerId)
|
||||
return [
|
||||
src.outputs.findIndex((slot) => slot.name === outName),
|
||||
dst.inputs.findIndex((slot) => slot.name === inName)
|
||||
]
|
||||
},
|
||||
[
|
||||
String(producer.id),
|
||||
String(consumer.id),
|
||||
edge.producer.slotName,
|
||||
edge.consumer.slotName
|
||||
] as const
|
||||
)
|
||||
const key = `${edge.producer.nodeType}.${edge.producer.slotName} -> ${edge.consumer.nodeType}.${edge.consumer.slotName}`
|
||||
expect(outIndex, `${key}: producer slot on instance`).toBeGreaterThan(-1)
|
||||
expect(inIndex, `${key}: consumer slot on instance`).toBeGreaterThan(-1)
|
||||
|
||||
if (vueNodesEnabled) {
|
||||
await comfyPage.vueNodes.waitForNodes(2)
|
||||
// Output-side mirror of getInputSlotConnectionDot, addressed by
|
||||
// data-slot-key so shared-label ambiguity cannot misfire the drag.
|
||||
const outDot = comfyPage.page
|
||||
.locator(`[data-node-id="${String(producer.id)}"]`)
|
||||
.locator('.lg-slot--output')
|
||||
.filter({
|
||||
has: comfyPage.page.locator(
|
||||
`[data-slot-key="${String(producer.id)}-out-${outIndex}"]`
|
||||
)
|
||||
})
|
||||
.getByTestId('slot-connection-dot')
|
||||
const inDot = comfyPage.vueNodes.getInputSlotConnectionDot(
|
||||
String(consumer.id),
|
||||
inIndex
|
||||
)
|
||||
await outDot.dragTo(inDot)
|
||||
} else {
|
||||
await producer.connectOutput(outIndex, consumer, inIndex)
|
||||
}
|
||||
|
||||
const linked = await comfyPage.page.evaluate(
|
||||
([consumerId, index]) => {
|
||||
const node = window.app!.graph.nodes.find(
|
||||
(candidate) => String(candidate.id) === consumerId
|
||||
)
|
||||
return node?.inputs?.[Number(index)]?.link != null
|
||||
},
|
||||
[String(consumer.id), String(inIndex)] as const
|
||||
)
|
||||
expect(linked, `${key} with VueNodes=${vueNodesEnabled}`).toBe(true)
|
||||
}
|
||||
|
||||
consoleErrors.stop()
|
||||
expect(
|
||||
consoleErrors.errors.filter((error) => !isForeignExecutionNoise(error)),
|
||||
`console errors with VueNodes=${vueNodesEnabled}`
|
||||
).toEqual([])
|
||||
await expectNoVisibleErrors(
|
||||
comfyPage.page,
|
||||
`after drag pass VueNodes=${vueNodesEnabled}`
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import {
|
||||
isForeignExecutionNoise,
|
||||
unallowlistedErrors
|
||||
} from '@e2e/fixtures/customNode/consoleErrorLedger'
|
||||
|
||||
// unallowlistedErrors is the sole enforcement point of the curated-run
|
||||
// console gate (customNode.regression.spec.ts T1): a degradation to
|
||||
// "always empty" would turn that gate vacuously green, so the filter's
|
||||
// three behaviors are pinned here directly.
|
||||
test.describe('consoleErrorLedger', () => {
|
||||
test('filters only errors matching the pack own patterns', () => {
|
||||
const errors = [
|
||||
'Failed to load resource: the server responded with a status of 404 () http://host/example.png',
|
||||
'TypeError: something real broke'
|
||||
]
|
||||
expect(unallowlistedErrors('ComfyUI-Impact-Pack', errors)).toEqual([
|
||||
'TypeError: something real broke'
|
||||
])
|
||||
})
|
||||
|
||||
test('a pattern never filters for a pack that does not own it', () => {
|
||||
const error = "Cannot use 'in' operator to search for 'content' in null"
|
||||
expect(unallowlistedErrors('ComfyUI-Impact-Pack', [error])).toEqual([error])
|
||||
expect(unallowlistedErrors('ComfyUI-Custom-Scripts', [error])).toEqual([])
|
||||
})
|
||||
|
||||
test('unknown pack fails open: every error surfaces', () => {
|
||||
// The first error would match an Impact pattern; with no ledger for the
|
||||
// pack, nothing may be filtered.
|
||||
const errors = [
|
||||
'Failed to load resource: the server responded with a status of 404 () http://host/example.png',
|
||||
'boom'
|
||||
]
|
||||
expect(unallowlistedErrors('Some-Future-Pack', errors)).toEqual(errors)
|
||||
})
|
||||
})
|
||||
|
||||
// Filters a prior tier's async execution error out of the non-executing
|
||||
// tiers; must match execution-domain lines and nothing a mount/wiring tier
|
||||
// should legitimately catch.
|
||||
test.describe('isForeignExecutionNoise', () => {
|
||||
test('matches the execution-domain console surfaces', () => {
|
||||
expect(isForeignExecutionNoise('PromptExecutionError: boom')).toBe(true)
|
||||
expect(isForeignExecutionNoise('Prompt execution failed')).toBe(true)
|
||||
expect(
|
||||
isForeignExecutionNoise(
|
||||
'Failed to load resource: the server responded with a status of 400 (Bad Request) http://127.0.0.1:8288/api/prompt'
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
test('does not match render or unrelated resource errors a tier must catch', () => {
|
||||
expect(
|
||||
isForeignExecutionNoise('TypeError: cannot read x of undefined')
|
||||
).toBe(false)
|
||||
expect(
|
||||
isForeignExecutionNoise(
|
||||
'Failed to load resource: 404 http://127.0.0.1:8288/api/view?filename=x.png'
|
||||
)
|
||||
).toBe(false)
|
||||
expect(
|
||||
isForeignExecutionNoise('Uncaught page error: something rendered wrong')
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
71
browser_tests/tests/customNodes/coreSmoke.spec.ts
Normal file
71
browser_tests/tests/customNodes/coreSmoke.spec.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema'
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import { isForeignExecutionNoise } from '@e2e/fixtures/customNode/consoleErrorLedger'
|
||||
import {
|
||||
customNodeSuiteSettings,
|
||||
dismissTemplatesDialog,
|
||||
drainBackendToIdle
|
||||
} from '@e2e/fixtures/utils/customNodeSuite'
|
||||
import { collectConsoleErrors } from '@e2e/fixtures/utils/consoleErrorCollector'
|
||||
import { errorSurfaces } from '@e2e/fixtures/utils/errorSurfaces'
|
||||
import { assetPath } from '@e2e/fixtures/utils/paths'
|
||||
|
||||
// Core-only, model-free workflow: the bundled default template references
|
||||
// model files a scoped test backend does not have, which rightly trips the
|
||||
// error surfaces this suite asserts are clean.
|
||||
const smokeWorkflow = JSON.parse(
|
||||
readFileSync(resolve(assetPath('customNodes/core_smoke.json')), 'utf-8')
|
||||
) as ComfyWorkflowJSON
|
||||
|
||||
test.use({ initialSettings: customNodeSuiteSettings })
|
||||
|
||||
test.beforeEach(async ({ comfyPage }) => {
|
||||
await dismissTemplatesDialog(comfyPage)
|
||||
})
|
||||
|
||||
// Leave the shared backend idle so the next test starts clean (drainBackendToIdle).
|
||||
test.afterEach(async ({ comfyPage }) => {
|
||||
// The drain is a no-op when the queue is already idle, so it costs
|
||||
// ~nothing in the common path; the 10s ceiling only bounds a genuinely
|
||||
// busy backend. A backend still busy past it is wedged, and the auto-run
|
||||
// tier's 150s guard surfaces that with the restart diagnostic.
|
||||
await drainBackendToIdle(comfyPage.page, 10_000)
|
||||
})
|
||||
|
||||
test.describe('smoke: core workflow', () => {
|
||||
test('loads without console errors in both renderers', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
for (const vueNodesEnabled of [false, true]) {
|
||||
const consoleErrors = collectConsoleErrors(comfyPage.page)
|
||||
await comfyPage.settings.setSetting(
|
||||
'Comfy.VueNodes.Enabled',
|
||||
vueNodesEnabled
|
||||
)
|
||||
await comfyPage.workflow.loadGraphData(smokeWorkflow)
|
||||
await comfyPage.nextFrame()
|
||||
consoleErrors.stop()
|
||||
|
||||
expect(await comfyPage.nodeOps.getGraphNodesCount()).toBeGreaterThan(0)
|
||||
// Core smoke loads a graph but queues no prompt; a prompt-execution
|
||||
// error here is a prior tier's async stray (isForeignExecutionNoise).
|
||||
expect(
|
||||
consoleErrors.errors.filter((error) => !isForeignExecutionNoise(error)),
|
||||
`console errors (VueNodes=${vueNodesEnabled})`
|
||||
).toEqual([])
|
||||
for (const [surface, locator] of Object.entries(
|
||||
errorSurfaces(comfyPage.page)
|
||||
))
|
||||
await expect(
|
||||
locator,
|
||||
`${surface} (VueNodes=${vueNodesEnabled})`
|
||||
).toHaveCount(0)
|
||||
}
|
||||
})
|
||||
})
|
||||
317
browser_tests/tests/customNodes/customNode.regression.spec.ts
Normal file
317
browser_tests/tests/customNodes/customNode.regression.spec.ts
Normal file
@@ -0,0 +1,317 @@
|
||||
/* oxlint-disable playwright/no-skipped-test -- tiers conditionally skip when the target backend lacks the required packs (installed custom nodes or devtools); this is the framework's designed environment gating, not a disabled test */
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import type { Page } from '@playwright/test'
|
||||
|
||||
import type { ComfyWorkflowJSON } from '@/platform/workflow/validation/schemas/workflowSchema'
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import {
|
||||
customNodeSuiteSettings,
|
||||
dismissTemplatesDialog,
|
||||
drainBackendToIdle
|
||||
} from '@e2e/fixtures/utils/customNodeSuite'
|
||||
import { LocalDesktopTarget } from '@e2e/fixtures/customNode/ComfyTarget'
|
||||
import {
|
||||
isForeignExecutionNoise,
|
||||
unallowlistedErrors
|
||||
} from '@e2e/fixtures/customNode/consoleErrorLedger'
|
||||
import {
|
||||
loadManifest,
|
||||
rendererPassesFor
|
||||
} from '@e2e/fixtures/customNode/manifest'
|
||||
import { expectedNodesPresent } from '@e2e/fixtures/customNode/objectInfoValidator'
|
||||
import { collectConsoleErrors } from '@e2e/fixtures/utils/consoleErrorCollector'
|
||||
import { errorSurfaces } from '@e2e/fixtures/utils/errorSurfaces'
|
||||
import { assetPath } from '@e2e/fixtures/utils/paths'
|
||||
|
||||
const target = new LocalDesktopTarget()
|
||||
const OBJECT_INFO_SANITY_FLOOR = 50
|
||||
// Display sinks used by the curated workflows; each is an output node whose
|
||||
// `executed` event carries a ui payload, so "the workflow ran" can be
|
||||
// upgraded to "data actually arrived at the sink". Console-style sinks
|
||||
// (WAS `Text to Console`) emit NO ui payload and stay off this list, so a
|
||||
// pack whose only sink prints to console gets execution-completed proof
|
||||
// only.
|
||||
const CURATED_SINK_TYPES = [
|
||||
'PreviewAny',
|
||||
'DisplayAny',
|
||||
'Display Any (rgthree)',
|
||||
'ShowText|pysssss'
|
||||
]
|
||||
|
||||
test.use({ initialSettings: customNodeSuiteSettings })
|
||||
|
||||
test.beforeEach(async ({ comfyPage }) => {
|
||||
await dismissTemplatesDialog(comfyPage)
|
||||
})
|
||||
|
||||
// Leave the shared backend idle so the next test starts clean (drainBackendToIdle).
|
||||
test.afterEach(async ({ comfyPage }) => {
|
||||
// The drain is a no-op when the queue is already idle, so it costs
|
||||
// ~nothing in the common path; the 10s ceiling only bounds a genuinely
|
||||
// busy backend. A backend still busy past it is wedged, and the auto-run
|
||||
// tier's 150s guard surfaces that with the restart diagnostic.
|
||||
await drainBackendToIdle(comfyPage.page, 10_000)
|
||||
})
|
||||
|
||||
async function expectNoVisibleErrors(
|
||||
page: Page,
|
||||
context: string
|
||||
): Promise<void> {
|
||||
for (const [surface, locator] of Object.entries(errorSurfaces(page)))
|
||||
await expect(locator, `${context}: ${surface}`).toHaveCount(0)
|
||||
}
|
||||
|
||||
function readWorkflow(relativePath: string): ComfyWorkflowJSON {
|
||||
return JSON.parse(
|
||||
readFileSync(resolve(relativePath), 'utf-8')
|
||||
) as ComfyWorkflowJSON
|
||||
}
|
||||
|
||||
async function nodeIdsByType(
|
||||
page: Page,
|
||||
classTypes: string[]
|
||||
): Promise<string[]> {
|
||||
return await page.evaluate((types) => {
|
||||
const nodes = window.app!.graph.nodes ?? []
|
||||
return nodes
|
||||
.filter((node) => {
|
||||
const n = node as { comfyClass?: string; type?: string }
|
||||
return types.includes(n.comfyClass ?? n.type ?? '')
|
||||
})
|
||||
.map((node) => String(node.id))
|
||||
}, classTypes)
|
||||
}
|
||||
|
||||
for (const entry of loadManifest()) {
|
||||
const workflowRelative = `browser_tests/${entry.workflow}`
|
||||
|
||||
test.describe(`custom node: ${entry.pack}`, () => {
|
||||
test('T0 load: expected nodes register and render in both renderers', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
test.setTimeout(entry.timeoutMs)
|
||||
const objectInfo = await target.getObjectInfo(comfyPage.page)
|
||||
expect(
|
||||
Object.keys(objectInfo).length,
|
||||
'object_info sanity floor'
|
||||
).toBeGreaterThan(OBJECT_INFO_SANITY_FLOOR)
|
||||
const { missing } = expectedNodesPresent(objectInfo, entry.expectedNodes)
|
||||
test.skip(
|
||||
missing.length > 0,
|
||||
`${entry.pack} not installed on this backend (missing: ${missing.join(', ')})`
|
||||
)
|
||||
await expectNoVisibleErrors(comfyPage.page, 'at startup')
|
||||
|
||||
// vueNodesCompatible: false = canvas-only assertions; still runs, no skip.
|
||||
const rendererPasses = rendererPassesFor(entry)
|
||||
if (entry.vueNodesCompatible === false)
|
||||
console.log(
|
||||
`${entry.pack} declares vueNodesCompatible=false; Vue Nodes pass not applicable`
|
||||
)
|
||||
for (const vueNodesEnabled of rendererPasses) {
|
||||
const consoleErrors = collectConsoleErrors(comfyPage.page)
|
||||
await comfyPage.settings.setSetting(
|
||||
'Comfy.VueNodes.Enabled',
|
||||
vueNodesEnabled
|
||||
)
|
||||
await comfyPage.nodeOps.clearGraph()
|
||||
|
||||
const addedIds: string[] = []
|
||||
for (const classType of entry.expectedNodes) {
|
||||
const node = await comfyPage.nodeOps.addNode(classType)
|
||||
addedIds.push(String(node.id))
|
||||
}
|
||||
await comfyPage.nextFrame()
|
||||
|
||||
expect(await comfyPage.nodeOps.getGraphNodesCount()).toBe(
|
||||
entry.expectedNodes.length
|
||||
)
|
||||
// Vue Nodes 2.0 mounts each node as a [data-node-id] element; assert
|
||||
// the pack's own nodes rendered, not just any node count.
|
||||
if (vueNodesEnabled)
|
||||
for (const id of addedIds)
|
||||
await expect(comfyPage.vueNodes.getNodeLocator(id)).toBeVisible()
|
||||
|
||||
consoleErrors.stop()
|
||||
// T0 loads and renders nodes but queues no prompt; a prompt-execution
|
||||
// error here is a prior tier's async stray (isForeignExecutionNoise).
|
||||
expect(
|
||||
consoleErrors.errors.filter(
|
||||
(error) => !isForeignExecutionNoise(error)
|
||||
),
|
||||
`console errors with VueNodes=${vueNodesEnabled}`
|
||||
).toEqual([])
|
||||
await expectNoVisibleErrors(
|
||||
comfyPage.page,
|
||||
`after VueNodes=${vueNodesEnabled} pass`
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
test('T1 run: workflow executes without error', async ({ comfyPage }) => {
|
||||
test.setTimeout(entry.timeoutMs + 15_000)
|
||||
const objectInfo = await target.getObjectInfo(comfyPage.page)
|
||||
const { missing } = expectedNodesPresent(objectInfo, entry.expectedNodes)
|
||||
test.skip(
|
||||
!entry.tiers.includes('run') ||
|
||||
missing.length > 0 ||
|
||||
entry.requiresGpu ||
|
||||
entry.requiresModels.length > 0 ||
|
||||
!entry.workflow ||
|
||||
!existsSync(resolve(workflowRelative)),
|
||||
`run tier unavailable for ${entry.pack}`
|
||||
)
|
||||
await expectNoVisibleErrors(comfyPage.page, 'at startup')
|
||||
|
||||
// Pack scripts can throw during workflow load or execution without
|
||||
// any visible error surface; collect console + uncaught page errors
|
||||
// across the whole run, filtered through the shared pack ledger.
|
||||
const consoleErrors = collectConsoleErrors(comfyPage.page)
|
||||
await comfyPage.workflow.loadGraphData(readWorkflow(workflowRelative))
|
||||
// A drifted fixture that dropped an expected node would silently
|
||||
// shrink the executed-set assertion (an empty id list PASSes on
|
||||
// execution_success alone): require every expected type to actually
|
||||
// be present in the loaded workflow before running it.
|
||||
const expectedNodeIds: string[] = []
|
||||
for (const type of entry.expectedNodes) {
|
||||
const ids = await nodeIdsByType(comfyPage.page, [type])
|
||||
expect(
|
||||
ids.length,
|
||||
`expectedNodes drift: ${type} is not in the curated workflow ${entry.workflow}`
|
||||
).toBeGreaterThan(0)
|
||||
expectedNodeIds.push(...ids)
|
||||
}
|
||||
const result = await target.runWorkflow(comfyPage.page, {
|
||||
expectedNodeIds,
|
||||
timeoutMs: entry.timeoutMs
|
||||
})
|
||||
|
||||
// A run that executed and errored carries an ExecutionError; a run the
|
||||
// backend rejected before executing (VALIDATION_FAIL) carries only the
|
||||
// captured node_errors text in clientError - surface whichever exists so
|
||||
// a red names the cause instead of printing an empty object.
|
||||
expect(
|
||||
result.outcome,
|
||||
result.clientError ?? JSON.stringify(result.error ?? {})
|
||||
).toBe('PASS')
|
||||
// PASS proves execution completed; the sinks prove data ARRIVED.
|
||||
// Every display sink in the curated workflow must have emitted a ui
|
||||
// payload through its executed event.
|
||||
const sinkIds = await nodeIdsByType(comfyPage.page, CURATED_SINK_TYPES)
|
||||
for (const sinkId of sinkIds)
|
||||
expect(
|
||||
result.outputsByNode[sinkId],
|
||||
`sink node ${sinkId} produced no ui payload`
|
||||
).toBeTruthy()
|
||||
await expectNoVisibleErrors(comfyPage.page, 'after run')
|
||||
consoleErrors.stop()
|
||||
expect(
|
||||
unallowlistedErrors(entry.pack, consoleErrors.errors),
|
||||
'console errors during curated run'
|
||||
).toEqual([])
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
test('harness self-check: captures a real execution error', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
test.setTimeout(30_000)
|
||||
const objectInfo = await target.getObjectInfo(comfyPage.page)
|
||||
expect(
|
||||
Object.keys(objectInfo).length,
|
||||
'object_info sanity floor'
|
||||
).toBeGreaterThan(OBJECT_INFO_SANITY_FLOOR)
|
||||
test.skip(
|
||||
!('DevToolsErrorRaiseNode' in objectInfo),
|
||||
'ComfyUI_devtools not installed on this backend'
|
||||
)
|
||||
|
||||
await comfyPage.workflow.loadGraphData(
|
||||
readWorkflow(assetPath('nodes/execution_error.json'))
|
||||
)
|
||||
const result = await target.runWorkflow(comfyPage.page, {
|
||||
expectedNodeIds: [],
|
||||
timeoutMs: 15000
|
||||
})
|
||||
|
||||
expect(result.outcome).toBe('EXECUTION_ERROR')
|
||||
expect(result.error?.exceptionType).toBeTruthy()
|
||||
// Proves the event tap captures node ids from the live `executing` stream
|
||||
// (its detail is a bare string): the failing node starts before it raises.
|
||||
expect(result.executedNodes.length).toBeGreaterThan(0)
|
||||
// Positive control for the zero-visible-errors invariant: a real execution
|
||||
// error MUST surface in the app's error overlay. If this fails, the
|
||||
// expectNoVisibleErrors selectors have rotted and every clean assertion in
|
||||
// this suite is meaningless.
|
||||
await expect(errorSurfaces(comfyPage.page).errorOverlay).toBeVisible()
|
||||
})
|
||||
|
||||
test('collector self-check: captures uncaught page exceptions', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
// Positive control for the console collector: an uncaught async throw
|
||||
// never reaches console.error, so this proves the pageerror listener
|
||||
// works. If this fails, every zero-console-errors assertion in the suite
|
||||
// is blind to the whole uncaught-exception class.
|
||||
const collected = collectConsoleErrors(comfyPage.page)
|
||||
await comfyPage.page.evaluate(() => {
|
||||
setTimeout(() => {
|
||||
throw new Error('cn-collector-self-check')
|
||||
}, 0)
|
||||
})
|
||||
await expect
|
||||
.poll(() =>
|
||||
collected.errors.some((error) =>
|
||||
error.includes('cn-collector-self-check')
|
||||
)
|
||||
)
|
||||
.toBe(true)
|
||||
collected.stop()
|
||||
})
|
||||
|
||||
test('attribution self-check: a foreign-prompt terminal event cannot fail this run', async ({
|
||||
comfyPage
|
||||
}) => {
|
||||
test.setTimeout(30_000)
|
||||
const objectInfo = await target.getObjectInfo(comfyPage.page)
|
||||
test.skip(
|
||||
!('PrimitiveInt' in objectInfo) || !('PreviewAny' in objectInfo),
|
||||
'core Primitive/PreviewAny nodes unavailable on this backend'
|
||||
)
|
||||
await comfyPage.workflow.loadGraphData(
|
||||
readWorkflow(assetPath('customNodes/core_primitive_preview_run.json'))
|
||||
)
|
||||
// Once the run's event tap starts filling, inject ONE terminal error under
|
||||
// a prompt id this page never queued. The positive prompt-id filter must
|
||||
// discard it; the pre-capture harness let the never-seen id through the
|
||||
// seen-set and misclassified the run as EXECUTION_ERROR. This is the
|
||||
// discriminating guard for the foreign-attribution bug class.
|
||||
await comfyPage.page.evaluate(() => {
|
||||
const timer = setInterval(() => {
|
||||
const sink = (window as unknown as { __cnEvents?: object[] }).__cnEvents
|
||||
if (!sink || sink.length === 0) return
|
||||
sink.push({
|
||||
type: 'execution_error',
|
||||
prompt_id: 'cn-foreign-self-check',
|
||||
exception_type: 'ForeignError',
|
||||
node_id: '424242'
|
||||
})
|
||||
clearInterval(timer)
|
||||
}, 25)
|
||||
})
|
||||
const result = await target.runWorkflow(comfyPage.page, {
|
||||
expectedNodeIds: await nodeIdsByType(comfyPage.page, [
|
||||
'PrimitiveInt',
|
||||
'PreviewAny'
|
||||
]),
|
||||
timeoutMs: 15000
|
||||
})
|
||||
expect(result.outcome, JSON.stringify(result.error ?? {})).toBe('PASS')
|
||||
expect(result.error).toBeUndefined()
|
||||
})
|
||||
76
browser_tests/tests/customNodes/manifest.pure.spec.ts
Normal file
76
browser_tests/tests/customNodes/manifest.pure.spec.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import type { CustomNodeManifestEntry } from '@e2e/fixtures/customNode/manifest'
|
||||
import {
|
||||
assertEntry,
|
||||
loadManifest,
|
||||
rendererPassesFor
|
||||
} from '@e2e/fixtures/customNode/manifest'
|
||||
|
||||
function validEntry(): CustomNodeManifestEntry {
|
||||
return {
|
||||
pack: 'Example-Pack',
|
||||
repo: 'https://github.com/example/Example-Pack',
|
||||
pin: 'a1'.repeat(20),
|
||||
tiers: ['load', 'connectivity', 'run'],
|
||||
workflow: 'assets/customNodes/example_run.json',
|
||||
expectedNodes: ['ExampleNode'],
|
||||
requiresGpu: false,
|
||||
requiresModels: [],
|
||||
timeoutMs: 60_000
|
||||
}
|
||||
}
|
||||
|
||||
test.describe('customNode manifest', () => {
|
||||
test('loads entries with the shape the regression spec depends on', () => {
|
||||
const entries = loadManifest()
|
||||
expect(entries.length).toBeGreaterThan(0)
|
||||
for (const entry of entries) {
|
||||
expect(entry.pack).toBeTruthy()
|
||||
expect(entry.expectedNodes.length).toBeGreaterThan(0)
|
||||
expect(entry.tiers.length).toBeGreaterThan(0)
|
||||
}
|
||||
})
|
||||
|
||||
test('rendererPassesFor drops only the Vue pass, only on an explicit false', () => {
|
||||
expect(rendererPassesFor({})).toEqual([false, true])
|
||||
expect(rendererPassesFor({ vueNodesCompatible: true })).toEqual([
|
||||
false,
|
||||
true
|
||||
])
|
||||
expect(rendererPassesFor({ vueNodesCompatible: false })).toEqual([false])
|
||||
})
|
||||
|
||||
test('pin must be a full commit SHA; only the canary override admits an empty one', () => {
|
||||
// Deterministic regardless of ambient env (a canary environment sets
|
||||
// the override): pin the var for the test, restore the prior value.
|
||||
const prior = process.env.CUSTOM_NODES_ALLOW_UNPINNED
|
||||
delete process.env.CUSTOM_NODES_ALLOW_UNPINNED
|
||||
try {
|
||||
expect(() => assertEntry(validEntry(), 0)).not.toThrow()
|
||||
expect(() => assertEntry({ ...validEntry(), pin: '' }, 0)).toThrow(/pin/)
|
||||
expect(() => assertEntry({ ...validEntry(), pin: 'abc123' }, 0)).toThrow(
|
||||
/pin/
|
||||
)
|
||||
process.env.CUSTOM_NODES_ALLOW_UNPINNED = '1'
|
||||
expect(() => assertEntry({ ...validEntry(), pin: '' }, 0)).not.toThrow()
|
||||
// the override admits only EMPTY pins; a malformed pin still fails
|
||||
expect(() => assertEntry({ ...validEntry(), pin: 'abc123' }, 0)).toThrow(
|
||||
/pin/
|
||||
)
|
||||
} finally {
|
||||
if (prior === undefined) delete process.env.CUSTOM_NODES_ALLOW_UNPINNED
|
||||
else process.env.CUSTOM_NODES_ALLOW_UNPINNED = prior
|
||||
}
|
||||
})
|
||||
|
||||
test('pack must be a plain path segment (it becomes the install dirname)', () => {
|
||||
for (const bad of ['../escape', 'a/b', '.hidden', 'sp ace', ''])
|
||||
expect(
|
||||
() => assertEntry({ ...validEntry(), pack: bad }, 0),
|
||||
`pack '${bad}' must be rejected`
|
||||
).toThrow(/pack/)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,47 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import type { ObjectInfo } from '@e2e/fixtures/customNode/objectInfoValidator'
|
||||
import {
|
||||
expectedNodesPresent,
|
||||
preValidate
|
||||
} from '@e2e/fixtures/customNode/objectInfoValidator'
|
||||
|
||||
const objectInfo: ObjectInfo = {
|
||||
KSampler: { input: { required: { model: {}, seed: {} } } }
|
||||
}
|
||||
|
||||
test.describe('objectInfoValidator', () => {
|
||||
test('expectedNodesPresent splits present from missing', () => {
|
||||
const { present, missing } = expectedNodesPresent(objectInfo, [
|
||||
'KSampler',
|
||||
'Missing (rgthree)'
|
||||
])
|
||||
expect(present).toEqual(['KSampler'])
|
||||
expect(missing).toEqual(['Missing (rgthree)'])
|
||||
})
|
||||
|
||||
test('preValidate returns MISSING_NODE for an unregistered class', () => {
|
||||
const failure = preValidate(objectInfo, [
|
||||
{ id: '1', classType: 'Ghost', inputs: {} }
|
||||
])
|
||||
expect(failure?.outcome).toBe('MISSING_NODE')
|
||||
})
|
||||
|
||||
test('preValidate returns VALIDATION_FAIL naming the missing required input', () => {
|
||||
const failure = preValidate(objectInfo, [
|
||||
{ id: '3', classType: 'KSampler', inputs: { model: 0 } }
|
||||
])
|
||||
expect(failure?.outcome).toBe('VALIDATION_FAIL')
|
||||
expect(failure?.message).toContain('missing required input "seed"')
|
||||
})
|
||||
|
||||
test('preValidate passes when every required input is present', () => {
|
||||
expect(
|
||||
preValidate(objectInfo, [
|
||||
{ id: '3', classType: 'KSampler', inputs: { model: 0, seed: 1 } }
|
||||
])
|
||||
).toBeNull()
|
||||
})
|
||||
})
|
||||
51
browser_tests/tests/customNodes/promptError.pure.spec.ts
Normal file
51
browser_tests/tests/customNodes/promptError.pure.spec.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import { summarizePromptError } from '@e2e/fixtures/customNode/ComfyTarget'
|
||||
|
||||
// The curated-run happy path never executes summarizePromptError (it only
|
||||
// runs on a VALIDATION_FAIL), so these cases are what keep a T1 rejection
|
||||
// naming the node+input instead of rotting back to `{}`.
|
||||
test.describe('summarizePromptError', () => {
|
||||
test('names the node class and the failing input from node_errors', () => {
|
||||
const body = {
|
||||
error: { type: 'prompt_outputs_failed_validation', message: 'failed' },
|
||||
node_errors: {
|
||||
'7': {
|
||||
class_type: 'ImpactInt',
|
||||
errors: [
|
||||
{ type: 'value_not_in_list', message: 'msg', details: 'value' }
|
||||
],
|
||||
dependent_outputs: []
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(summarizePromptError(body)).toBe('failed; ImpactInt: value')
|
||||
})
|
||||
|
||||
test('accepts a string top-level error', () => {
|
||||
expect(summarizePromptError({ error: 'bad request' })).toBe('bad request')
|
||||
})
|
||||
|
||||
test('falls back to the node message when details is empty', () => {
|
||||
const body = {
|
||||
node_errors: {
|
||||
'3': {
|
||||
class_type: 'KSampler',
|
||||
errors: [
|
||||
{ type: 'x', message: 'required input missing', details: '' }
|
||||
],
|
||||
dependent_outputs: []
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(summarizePromptError(body)).toBe('KSampler: required input missing')
|
||||
})
|
||||
|
||||
test('returns undefined for an empty or non-object body', () => {
|
||||
expect(summarizePromptError({})).toBeUndefined()
|
||||
expect(summarizePromptError(null)).toBeUndefined()
|
||||
expect(summarizePromptError('not an object')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
71
browser_tests/tests/customNodes/runResult.pure.spec.ts
Normal file
71
browser_tests/tests/customNodes/runResult.pure.spec.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import { classifyRun } from '@e2e/fixtures/customNode/runResult'
|
||||
|
||||
test.describe('classifyRun', () => {
|
||||
test('PASS when every expected node appears in the executing stream', () => {
|
||||
const result = classifyRun({
|
||||
events: [
|
||||
{ type: 'execution_start' },
|
||||
{ type: 'executing', node: '1' },
|
||||
{ type: 'executing', node: '2' },
|
||||
{ type: 'executing', node: null },
|
||||
{ type: 'execution_success' }
|
||||
],
|
||||
expectedNodeIds: ['1', '2']
|
||||
})
|
||||
expect(result.outcome).toBe('PASS')
|
||||
expect(result.executedNodes).toEqual(['1', '2'])
|
||||
})
|
||||
|
||||
test('PARTIAL when a succeeding run replays a cached node that never emitted executing', () => {
|
||||
const result = classifyRun({
|
||||
events: [{ type: 'executing', node: '1' }, { type: 'execution_success' }],
|
||||
expectedNodeIds: ['1', '2']
|
||||
})
|
||||
expect(result.outcome).toBe('PARTIAL')
|
||||
expect(result.executedNodes).toEqual(['1'])
|
||||
})
|
||||
|
||||
test('EXECUTION_ERROR captures the failing node details', () => {
|
||||
const result = classifyRun({
|
||||
events: [
|
||||
{ type: 'executing', node: '1' },
|
||||
{
|
||||
type: 'execution_error',
|
||||
error: { exceptionType: 'ValueError', nodeId: '1' }
|
||||
}
|
||||
],
|
||||
expectedNodeIds: ['1']
|
||||
})
|
||||
expect(result.outcome).toBe('EXECUTION_ERROR')
|
||||
expect(result.error?.exceptionType).toBe('ValueError')
|
||||
})
|
||||
|
||||
test('EXECUTION_ERROR when the run is interrupted', () => {
|
||||
const result = classifyRun({
|
||||
events: [
|
||||
{ type: 'executing', node: '1' },
|
||||
{ type: 'execution_interrupted' }
|
||||
],
|
||||
expectedNodeIds: ['1']
|
||||
})
|
||||
expect(result.outcome).toBe('EXECUTION_ERROR')
|
||||
})
|
||||
|
||||
test('TIMEOUT when flagged or when no terminal event arrived', () => {
|
||||
const flagged = classifyRun({
|
||||
events: [{ type: 'executing', node: '1' }],
|
||||
expectedNodeIds: ['1'],
|
||||
timedOut: true
|
||||
})
|
||||
const noTerminal = classifyRun({
|
||||
events: [{ type: 'executing', node: '1' }],
|
||||
expectedNodeIds: ['1']
|
||||
})
|
||||
expect(flagged.outcome).toBe('TIMEOUT')
|
||||
expect(noTerminal.outcome).toBe('TIMEOUT')
|
||||
})
|
||||
})
|
||||
270
browser_tests/tests/customNodes/typePairing.pure.spec.ts
Normal file
270
browser_tests/tests/customNodes/typePairing.pure.spec.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
import {
|
||||
comfyExpect as expect,
|
||||
comfyPageFixture as test
|
||||
} from '@e2e/fixtures/ComfyPage'
|
||||
import type { RawNodeDef } from '@e2e/fixtures/customNode/typePairing'
|
||||
import {
|
||||
isTypeCompatible,
|
||||
normalizeNodeDefs,
|
||||
packOf,
|
||||
planPairs
|
||||
} from '@e2e/fixtures/customNode/typePairing'
|
||||
|
||||
const DEFS: Record<string, RawNodeDef> = {
|
||||
LatentSource: {
|
||||
input: { required: {} },
|
||||
output: ['LATENT'],
|
||||
output_name: ['LATENT'],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
LatentSink: {
|
||||
input: { required: { latent: ['LATENT', {}] } },
|
||||
output: [],
|
||||
python_module: 'custom_nodes.SomePack'
|
||||
},
|
||||
UnionSource: {
|
||||
input: { required: {} },
|
||||
output: ['STRING,INT'],
|
||||
output_name: ['value'],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
IntSink: {
|
||||
input: { required: { value: ['int', {}] } },
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
ComboNode: {
|
||||
input: { required: { choice: [['a', 'b'], {}] } },
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
SocketlessNode: {
|
||||
input: { required: { hidden: ['STRING', { socketless: true }] } },
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
WildcardNode: {
|
||||
input: { required: { anything: ['*', {}] } },
|
||||
output: ['*'],
|
||||
output_name: ['out'],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
OrphanNode: {
|
||||
input: { required: {} },
|
||||
output: ['NOBODY_CONSUMES_THIS'],
|
||||
output_name: ['orphan'],
|
||||
python_module: 'custom_nodes.OrphanPack'
|
||||
}
|
||||
}
|
||||
|
||||
test.describe('typePairing', () => {
|
||||
test('isTypeCompatible mirrors the real validator semantics', () => {
|
||||
expect(isTypeCompatible('LATENT', 'LATENT')).toBe(true)
|
||||
expect(isTypeCompatible('latent', 'LATENT')).toBe(true)
|
||||
expect(isTypeCompatible('LATENT', 'IMAGE')).toBe(false)
|
||||
expect(isTypeCompatible('STRING,INT', 'INT')).toBe(true)
|
||||
expect(isTypeCompatible('STRING,INT', 'FLOAT')).toBe(false)
|
||||
expect(isTypeCompatible('*', 'ANYTHING')).toBe(true)
|
||||
expect(isTypeCompatible('', 'ANYTHING')).toBe(true)
|
||||
})
|
||||
|
||||
test('packOf attributes core vs custom pack', () => {
|
||||
expect(packOf('nodes')).toBe('core')
|
||||
expect(packOf('comfy_extras.nodes_x')).toBe('core')
|
||||
expect(packOf('custom_nodes.ComfyUI-Impact-Pack')).toBe(
|
||||
'ComfyUI-Impact-Pack'
|
||||
)
|
||||
expect(packOf(undefined)).toBe('core')
|
||||
})
|
||||
|
||||
test('normalize maps COMBO literals and drops socketless inputs', () => {
|
||||
const nodes = normalizeNodeDefs(DEFS)
|
||||
const combo = nodes.find((n) => n.type === 'ComboNode')!
|
||||
expect(combo.inputs).toEqual([
|
||||
{ name: 'choice', type: 'COMBO', comboOptions: ['a', 'b'] }
|
||||
])
|
||||
const socketless = nodes.find((n) => n.type === 'SocketlessNode')!
|
||||
expect(socketless.inputs).toEqual([])
|
||||
// socketless is a recognized shape deliberately left out of the matrix;
|
||||
// it must never be recorded as an unknown slot.
|
||||
expect(socketless.unknownSlots).toBeUndefined()
|
||||
})
|
||||
|
||||
test('unrecognizable slot specs are recorded, never silently dropped', () => {
|
||||
// A numeric input type and a numeric output type have no connectable
|
||||
// socket type (slotTypeOf null): the slot leaves the corpus, but the
|
||||
// drop must surface on the node and in the plan.
|
||||
const nodes = normalizeNodeDefs({
|
||||
WeirdNode: {
|
||||
input: { required: { strange: [42, {}], ok: ['INT', {}] } },
|
||||
output: [7, 'INT'],
|
||||
python_module: 'custom_nodes.weird-pack'
|
||||
}
|
||||
})
|
||||
const weird = nodes.find((n) => n.type === 'WeirdNode')!
|
||||
expect(weird.unknownSlots).toEqual(['strange', 'output[0]'])
|
||||
expect(weird.inputs.map((s) => s.name)).toEqual(['ok'])
|
||||
const plan = planPairs(nodes, ['WeirdNode'])
|
||||
expect(plan.unknownShapes).toEqual([
|
||||
'WeirdNode.strange',
|
||||
'WeirdNode.output[0]'
|
||||
])
|
||||
})
|
||||
|
||||
test('planPairs pairs exact and union types, deterministically', () => {
|
||||
const nodes = normalizeNodeDefs(DEFS)
|
||||
const plan = planPairs(nodes, ['LatentSink', 'IntSink'])
|
||||
const keys = plan.pairs.map(
|
||||
(p) =>
|
||||
`${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}`
|
||||
)
|
||||
expect(keys).toContain('LatentSource.LATENT->LatentSink.latent')
|
||||
expect(keys).toContain('UnionSource.value->IntSink.value')
|
||||
const again = planPairs(nodes, ['LatentSink', 'IntSink'])
|
||||
expect(again.pairs).toEqual(plan.pairs)
|
||||
// The DEFS corpus is fully recognizable; unknownShapes stays empty.
|
||||
expect(plan.unknownShapes).toEqual([])
|
||||
})
|
||||
|
||||
test('COMBO slots with different vocabularies stay excluded', () => {
|
||||
const nodes = normalizeNodeDefs({
|
||||
ComboSource: {
|
||||
input: { required: {} },
|
||||
output: [['A', 'B', 'C']],
|
||||
output_name: [['A', 'B', 'C'] as unknown as string],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
...DEFS
|
||||
})
|
||||
const source = nodes.find((n) => n.type === 'ComboSource')!
|
||||
expect(source.outputs).toEqual([
|
||||
{ name: 'COMBO', type: 'COMBO', comboOptions: ['A', 'B', 'C'] }
|
||||
])
|
||||
// ComboNode.choice offers [a, b] - not the same vocabulary as [A, B, C].
|
||||
const plan = planPairs(nodes, ['ComboSource', 'ComboNode'])
|
||||
expect(plan.pairs).toEqual([])
|
||||
expect(plan.combos.map((s) => `${s.nodeType}.${s.slotName}`)).toEqual([
|
||||
'ComboSource.COMBO',
|
||||
'ComboNode.choice'
|
||||
])
|
||||
})
|
||||
|
||||
test('COMBO slots with an identical vocabulary pair up', () => {
|
||||
const nodes = normalizeNodeDefs({
|
||||
SamplerNameSource: {
|
||||
input: { required: {} },
|
||||
output: [['euler', 'ddim']],
|
||||
output_name: [['euler', 'ddim'] as unknown as string],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
SamplerNameSink: {
|
||||
input: { required: { sampler_name: [['euler', 'ddim'], {}] } },
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
...DEFS
|
||||
})
|
||||
const plan = planPairs(nodes, ['SamplerNameSource', 'SamplerNameSink'])
|
||||
expect(
|
||||
plan.pairs.map(
|
||||
(p) =>
|
||||
`${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}`
|
||||
)
|
||||
).toEqual(['SamplerNameSource.COMBO->SamplerNameSink.sampler_name'])
|
||||
expect(plan.combos).toEqual([])
|
||||
})
|
||||
|
||||
// Census-derived: transformed (V2-schema) defs carry combo inputs as the
|
||||
// string 'COMBO' with options in the opts object. Same vocabulary must
|
||||
// pair across forms, and a combo with no static options (remote/lazy)
|
||||
// must never blind-match.
|
||||
test('V2-form combos pair across forms by vocabulary; unknown options never pair', () => {
|
||||
const nodes = normalizeNodeDefs({
|
||||
ListFormSource: {
|
||||
input: { required: {} },
|
||||
output: [['x', 'y']],
|
||||
output_name: [['x', 'y'] as unknown as string],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
V2FormSink: {
|
||||
input: {
|
||||
required: {
|
||||
dim: ['COMBO', { multiselect: false, options: ['y', 'x'] }]
|
||||
}
|
||||
},
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
RemoteComboSink: {
|
||||
input: {
|
||||
required: {
|
||||
image: ['COMBO', { remote: { route: '/internal/files/output' } }]
|
||||
}
|
||||
},
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
...DEFS
|
||||
})
|
||||
const plan = planPairs(nodes, [
|
||||
'ListFormSource',
|
||||
'V2FormSink',
|
||||
'RemoteComboSink'
|
||||
])
|
||||
expect(
|
||||
plan.pairs.map(
|
||||
(p) =>
|
||||
`${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}`
|
||||
)
|
||||
).toEqual(['ListFormSource.COMBO->V2FormSink.dim'])
|
||||
expect(plan.combos.map((s) => `${s.nodeType}.${s.slotName}`)).toEqual([
|
||||
'RemoteComboSink.image'
|
||||
])
|
||||
})
|
||||
|
||||
test('COMBO vocabulary matching ignores option order', () => {
|
||||
// A wired input bypasses its own widget, so menu order and the
|
||||
// options[0] default are not part of the wire contract - membership is.
|
||||
const nodes = normalizeNodeDefs({
|
||||
ShuffledSource: {
|
||||
input: { required: {} },
|
||||
output: [['ddim', 'euler']],
|
||||
output_name: [['ddim', 'euler'] as unknown as string],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
SamplerNameSink: {
|
||||
input: { required: { sampler_name: [['euler', 'ddim'], {}] } },
|
||||
output: [],
|
||||
python_module: 'nodes'
|
||||
},
|
||||
...DEFS
|
||||
})
|
||||
const plan = planPairs(nodes, ['ShuffledSource', 'SamplerNameSink'])
|
||||
expect(
|
||||
plan.pairs.map(
|
||||
(p) =>
|
||||
`${p.producer.nodeType}.${p.producer.slotName}->${p.consumer.nodeType}.${p.consumer.slotName}`
|
||||
)
|
||||
).toEqual(['ShuffledSource.COMBO->SamplerNameSink.sampler_name'])
|
||||
expect(plan.combos).toEqual([])
|
||||
})
|
||||
|
||||
test('wildcard slots are excluded, orphan types recorded not failed', () => {
|
||||
const nodes = normalizeNodeDefs(DEFS)
|
||||
const plan = planPairs(nodes, ['WildcardNode', 'OrphanNode'])
|
||||
expect(plan.wildcards.map((w) => w.nodeType)).toEqual([
|
||||
'WildcardNode',
|
||||
'WildcardNode'
|
||||
])
|
||||
expect(plan.orphans).toEqual([
|
||||
{
|
||||
nodeType: 'OrphanNode',
|
||||
pack: 'OrphanPack',
|
||||
slotName: 'orphan',
|
||||
slotType: 'NOBODY_CONSUMES_THIS',
|
||||
dir: 'out'
|
||||
}
|
||||
])
|
||||
expect(plan.pairs).toEqual([])
|
||||
})
|
||||
})
|
||||
10
package.json
10
package.json
@@ -52,6 +52,16 @@
|
||||
"test:browser": "pnpm exec playwright test",
|
||||
"test:browser:coverage": "cross-env COLLECT_COVERAGE=true pnpm test:browser",
|
||||
"test:browser:local": "cross-env PLAYWRIGHT_LOCAL=1 PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm test:browser",
|
||||
"test:custom-nodes": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm exec playwright test browser_tests/tests/customNodes/ --config playwright.chrome.config.ts --workers=1",
|
||||
"test:custom-nodes:ci": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:8188 pnpm exec playwright test browser_tests/tests/customNodes/ --config playwright.chrome.config.ts --workers=1",
|
||||
"test:custom-nodes:watch": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:5173 PLAYWRIGHT_LOCAL=1 SLOW_MO=300 pnpm exec playwright test browser_tests/tests/customNodes/customNode.regression.spec.ts browser_tests/tests/customNodes/connectivity.spec.ts --config playwright.chrome.config.ts --workers=1 --headed",
|
||||
"test:custom-nodes:debug": "cross-env PLAYWRIGHT_TEST_URL=http://localhost:5173 pnpm exec playwright test browser_tests/tests/customNodes/customNode.regression.spec.ts browser_tests/tests/customNodes/connectivity.spec.ts --config playwright.chrome.config.ts --workers=1 --debug",
|
||||
"test:custom-nodes:impact-render": "pnpm test:custom-nodes:debug -g \"ComfyUI-Impact-Pack.*T0\"",
|
||||
"test:custom-nodes:impact-run": "pnpm test:custom-nodes:debug -g \"ComfyUI-Impact-Pack.*T1\"",
|
||||
"test:custom-nodes:vhs-render": "pnpm test:custom-nodes:debug -g \"VideoHelperSuite.*T0\"",
|
||||
"test:custom-nodes:vhs-run": "pnpm test:custom-nodes:debug -g \"VideoHelperSuite.*T1\"",
|
||||
"test:custom-nodes:connectivity": "pnpm test:custom-nodes:debug -g \"connectivity\"",
|
||||
"test:custom-nodes:self-check": "pnpm test:custom-nodes:watch -g \"self-check\"",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:unit": "vitest run",
|
||||
"typecheck": "vue-tsc --noEmit",
|
||||
|
||||
10
playwright.chrome.config.ts
Normal file
10
playwright.chrome.config.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { defineConfig } from '@playwright/test'
|
||||
|
||||
import base from './playwright.config'
|
||||
|
||||
// Run against the system-installed Google Chrome (no bundled-chromium download).
|
||||
// trace stays off: Playwright's trace recorder crashes pages under the branded
|
||||
// Chrome channel on this machine (instant browser close, reported as timeout).
|
||||
export default defineConfig(base, {
|
||||
use: { channel: 'chrome', video: 'off', trace: 'off' }
|
||||
})
|
||||
@@ -140,6 +140,7 @@ describe('loadTurnstile', () => {
|
||||
const promise = loadTurnstile()
|
||||
scriptEl()!.dispatchEvent(new Event('load'))
|
||||
// global never published; deadline elapses
|
||||
// oxlint-disable-next-line vitest/valid-expect -- deliberately awaited after the timer advance below; awaiting here would deadlock fake timers
|
||||
const assertion = expect(promise).rejects.toThrow(/timed out/i)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
|
||||
@@ -177,6 +178,7 @@ describe('loadTurnstile', () => {
|
||||
const loadTurnstile = await freshLoadTurnstile()
|
||||
|
||||
const promise = loadTurnstile()
|
||||
// oxlint-disable-next-line vitest/valid-expect -- deliberately awaited after the timer advance below; awaiting here would deadlock fake timers
|
||||
const assertion = expect(promise).rejects.toThrow(/timed out/i)
|
||||
vi.advanceTimersByTime(10_000)
|
||||
|
||||
@@ -216,6 +218,7 @@ describe('loadTurnstile', () => {
|
||||
|
||||
const loadTurnstile = await freshLoadTurnstile()
|
||||
const promise = loadTurnstile()
|
||||
// oxlint-disable-next-line vitest/valid-expect -- deliberately awaited after the timer advance below; awaiting here would deadlock fake timers
|
||||
const assertion = expect(promise).rejects.toThrow(/timed out/i)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
|
||||
|
||||
@@ -216,8 +216,9 @@ export const useNodeBadge = () => {
|
||||
)
|
||||
},
|
||||
afterConfigureGraph() {
|
||||
for (const node of app.canvas.graph?.nodes ?? [])
|
||||
priceBadge.updateSubgraphCredits(node)
|
||||
// DETECTION PROOF (row 7, load hook): extension hook throws on load.
|
||||
// Expected: curated run (T1) red "Error calling extension 'Comfy.NodeBadge'".
|
||||
throw new Error('DETECTION PROOF: afterConfigureGraph crash')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -929,7 +929,9 @@ export class LGraphNode
|
||||
let i = 0
|
||||
for (const widget of this.widgets ?? []) {
|
||||
if (widget.serialize === false) continue
|
||||
if (i >= info.widgets_values.length) break
|
||||
// DETECTION PROOF (row 3, persistence): off-by-one drops the last
|
||||
// widgets_values. Expected: persistence tier red '... on set-values reload'.
|
||||
if (i >= info.widgets_values.length - 1) break
|
||||
widget.value = info.widgets_values[i++]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -696,6 +696,9 @@ export class LiteGraphGlobal {
|
||||
type_b = String(type_b)
|
||||
type_a = type_a.toLowerCase()
|
||||
type_b = type_b.toLowerCase()
|
||||
// DETECTION PROOF (row 4, links/type): reject IMAGE connections.
|
||||
// Expected: connectivity sweep red with a CONNECT_REJECTED pair list.
|
||||
if (type_a === 'image' || type_b === 'image') return false
|
||||
|
||||
// For nodes supporting multiple connection types
|
||||
if (!type_a.includes(',') && !type_b.includes(',')) return type_a == type_b
|
||||
|
||||
@@ -13,6 +13,9 @@ export function getNodeInputOnPos(
|
||||
): { index: number; input: INodeInputSlot; pos: Point } | undefined {
|
||||
const { inputs } = node
|
||||
if (!inputs) return
|
||||
// DETECTION PROOF (row 5, links/drop): slot hit-test never resolves.
|
||||
// Expected: connectivity drag red 'EmptyImage.IMAGE -> ImageBatch.image2 ...'.
|
||||
return undefined
|
||||
|
||||
for (const [index, input] of inputs.entries()) {
|
||||
const pos = node.getInputPos(index)
|
||||
|
||||
@@ -106,7 +106,9 @@ const coreWidgetDefinitions: Array<[string, WidgetDefinition]> = [
|
||||
essential: false
|
||||
}
|
||||
],
|
||||
['int', { component: WidgetInputNumber, aliases: ['INT'], essential: true }],
|
||||
// DETECTION PROOF (row 2, mount v2): drop the int Vue widget mapping.
|
||||
// Expected: mount Vue pass red '... Vue mounts N of M widgets'.
|
||||
// ['int', { component: WidgetInputNumber, aliases: ['INT'], essential: true }],
|
||||
[
|
||||
'float',
|
||||
{
|
||||
|
||||
@@ -336,8 +336,11 @@ export const useLitegraphService = () => {
|
||||
const orderedInputSpecs = getOrderedInputSpecs(nodeDefImpl, inputs)
|
||||
|
||||
// Create sockets and widgets in the determined order
|
||||
for (const inputSpec of orderedInputSpecs) addInputSocket(node, inputSpec)
|
||||
for (const inputSpec of orderedInputSpecs) addInputWidget(node, inputSpec)
|
||||
// DETECTION PROOF (row 1, mount v1): drop the last declared input.
|
||||
// Expected: mount tier red 'is missing declared input "..." (litegraph)'.
|
||||
const detectionProof = orderedInputSpecs.slice(0, -1)
|
||||
for (const inputSpec of detectionProof) addInputSocket(node, inputSpec)
|
||||
for (const inputSpec of detectionProof) addInputWidget(node, inputSpec)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -103,6 +103,9 @@ export const graphToPrompt = async (
|
||||
const widgetValue = widget.serializeValue
|
||||
? await widget.serializeValue(node, i)
|
||||
: widget.value
|
||||
// DETECTION PROOF (row 6, exec/serialize): drop numeric widget inputs.
|
||||
// Expected: curated run (T1) red 'Prompt outputs failed validation; ...'.
|
||||
if (typeof widgetValue === 'number') continue
|
||||
// By default, Array values are reserved to represent node connections.
|
||||
// We need to wrap the array as an object to avoid the misinterpretation
|
||||
// of the array as a node connection.
|
||||
|
||||
Reference in New Issue
Block a user