kvcache-ai/sglang
1
0
Fork 0
mirror of https://github.com/kvcache-ai/sglang.git synced 2026-07-09 00:37:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[diffusion] CI: improve diffusion CI (#13562)

Browse Source 
Co-authored-by: Adarsh Shirawalmath <114558126+adarshxs@users.noreply.github.com>
This commit is contained in:
Mick
2025-11-20 10:54:13 +08:00
committed by GitHub
parent af6bcadcf7
commit 127d59cd2c
19 changed files with 782 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
python/sglang/multimodal_gen/envs.py
View File

@@ -15,26 +15,26 @@ from packaging import version
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60
SGL_DIFFUSION_NCCL_SO_PATH: str | None = None
SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60
SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None
LD_LIBRARY_PATH: str | None = None
LOCAL_RANK: int = 0
CUDA_VISIBLE_DEVICES: str | None = None
SGL_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion")
SGL_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion")
SGL_DIFFUSION_CONFIGURE_LOGGING: int = 1
SGL_DIFFUSION_LOGGING_LEVEL: str = "INFO"
SGL_DIFFUSION_LOGGING_PREFIX: str = ""
SGL_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None
SGL_DIFFUSION_TRACE_FUNCTION: int = 0
SGL_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork"
SGL_DIFFUSION_TARGET_DEVICE: str = "cuda"
SGLANG_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion")
SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion")
SGLANG_DIFFUSION_CONFIGURE_LOGGING: int = 1
SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO"
SGLANG_DIFFUSION_LOGGING_PREFIX: str = ""
SGLANG_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None
SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0
SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork"
SGLANG_DIFFUSION_TARGET_DEVICE: str = "cuda"
MAX_JOBS: str | None = None
NVCC_THREADS: str | None = None
CMAKE_BUILD_TYPE: str | None = None
VERBOSE: bool = False
SGL_DIFFUSION_SERVER_DEV_MODE: bool = False
SGL_DIFFUSION_STAGE_LOGGING: bool = False
SGLANG_DIFFUSION_SERVER_DEV_MODE: bool = False
SGLANG_DIFFUSION_STAGE_LOGGING: bool = False
def _is_hip():
@@ -165,8 +165,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
# ================== Installation Time Env Vars ==================
# Target device of sgl-diffusion, supporting [cuda (by default),
# rocm, neuron, cpu, openvino]
"SGL_DIFFUSION_TARGET_DEVICE": lambda: os.getenv(
"SGL_DIFFUSION_TARGET_DEVICE", "cuda"
"SGLANG_DIFFUSION_TARGET_DEVICE": lambda: os.getenv(
"SGLANG_DIFFUSION_TARGET_DEVICE", "cuda"
),
# Maximum number of compilation jobs to run in parallel.
# By default this is the number of CPUs
@@ -176,10 +176,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
# If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU.
"NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None),
# If set, sgl_diffusion will use precompiled binaries (*.so)
"SGL_DIFFUSION_USE_PRECOMPILED": lambda: bool(
os.environ.get("SGL_DIFFUSION_USE_PRECOMPILED")
"SGLANG_DIFFUSION_USE_PRECOMPILED": lambda: bool(
os.environ.get("SGLANG_DIFFUSION_USE_PRECOMPILED")
)
or bool(os.environ.get("SGL_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")),
or bool(os.environ.get("SGLANG_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")),
# CMake build type
# If not set, defaults to "Debug" or "RelWithDebInfo"
# Available options: "Debug", "Release", "RelWithDebInfo"
@@ -191,36 +191,36 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Note that this not only affects how sgl_diffusion finds its configuration files
# during runtime, but also affects how sgl_diffusion installs its configuration
# files during **installation**.
"SGL_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser(
"SGLANG_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser(
os.getenv(
"SGL_DIFFUSION_CONFIG_ROOT",
"SGLANG_DIFFUSION_CONFIG_ROOT",
os.path.join(get_default_config_root(), "sgl_diffusion"),
)
),
# ================== Runtime Env Vars ==================
# Root directory for FASTVIDEO cache files
# Defaults to `~/.cache/sgl_diffusion` unless `XDG_CACHE_HOME` is set
"SGL_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser(
"SGLANG_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser(
os.getenv(
"SGL_DIFFUSION_CACHE_ROOT",
"SGLANG_DIFFUSION_CACHE_ROOT",
os.path.join(get_default_cache_root(), "sgl_diffusion"),
)
),
# Interval in seconds to log a warning message when the ring buffer is full
"SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int(
os.environ.get("SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60")
"SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int(
os.environ.get("SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60")
),
# Path to the NCCL library file. It is needed because nccl>=2.19 brought
# by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234
"SGL_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get(
"SGL_DIFFUSION_NCCL_SO_PATH", None
"SGLANG_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get(
"SGLANG_DIFFUSION_NCCL_SO_PATH", None
),
# when `SGL_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl
# when `SGLANG_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl
# library file in the locations specified by `LD_LIBRARY_PATH`
"LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None),
# Internal flag to enable Dynamo fullgraph capture
"SGL_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool(
os.environ.get("SGL_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"
"SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool(
os.environ.get("SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"
),
# local rank of the process in the distributed setting, used to determine
# the GPU device id
@@ -228,62 +228,62 @@ environment_variables: dict[str, Callable[[], Any]] = {
# used to control the visible devices in the distributed setting
"CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
# timeout for each iteration in the engine
"SGL_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int(
os.environ.get("SGL_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60")
"SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int(
os.environ.get("SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60")
),
# Logging configuration
# If set to 0, sgl_diffusion will not configure logging
# If set to 1, sgl_diffusion will configure logging using the default configuration
# or the configuration file specified by SGL_DIFFUSION_LOGGING_CONFIG_PATH
"SGL_DIFFUSION_CONFIGURE_LOGGING": lambda: int(
os.getenv("SGL_DIFFUSION_CONFIGURE_LOGGING", "1")
# or the configuration file specified by SGLANG_DIFFUSION_LOGGING_CONFIG_PATH
"SGLANG_DIFFUSION_CONFIGURE_LOGGING": lambda: int(
os.getenv("SGLANG_DIFFUSION_CONFIGURE_LOGGING", "1")
),
"SGL_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv(
"SGL_DIFFUSION_LOGGING_CONFIG_PATH"
"SGLANG_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv(
"SGLANG_DIFFUSION_LOGGING_CONFIG_PATH"
),
# this is used for configuring the default logging level
"SGL_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv(
"SGL_DIFFUSION_LOGGING_LEVEL", "INFO"
"SGLANG_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv(
"SGLANG_DIFFUSION_LOGGING_LEVEL", "INFO"
),
# if set, SGL_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages
"SGL_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv(
"SGL_DIFFUSION_LOGGING_PREFIX", ""
# if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages
"SGLANG_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv(
"SGLANG_DIFFUSION_LOGGING_PREFIX", ""
),
# Trace function calls
# If set to 1, sgl_diffusion will trace function calls
# Useful for debugging
"SGL_DIFFUSION_TRACE_FUNCTION": lambda: int(
os.getenv("SGL_DIFFUSION_TRACE_FUNCTION", "0")
"SGLANG_DIFFUSION_TRACE_FUNCTION": lambda: int(
os.getenv("SGLANG_DIFFUSION_TRACE_FUNCTION", "0")
),
# Path to the attention configuration file. Only used for sliding tile
# attention for now.
"SGL_DIFFUSION_ATTENTION_CONFIG": lambda: (
"SGLANG_DIFFUSION_ATTENTION_CONFIG": lambda: (
None
if os.getenv("SGL_DIFFUSION_ATTENTION_CONFIG", None) is None
else os.path.expanduser(os.getenv("SGL_DIFFUSION_ATTENTION_CONFIG", "."))
if os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", None) is None
else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", "."))
),
# Use dedicated multiprocess context for workers.
# Both spawn and fork work
"SGL_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv(
"SGL_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork"
"SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv(
"SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork"
),
# Enables torch profiler if set. Path to the directory where torch profiler
# traces are saved. Note that it must be an absolute path.
"SGL_DIFFUSION_TORCH_PROFILER_DIR": lambda: (
"SGLANG_DIFFUSION_TORCH_PROFILER_DIR": lambda: (
None
if os.getenv("SGL_DIFFUSION_TORCH_PROFILER_DIR", None) is None
else os.path.expanduser(os.getenv("SGL_DIFFUSION_TORCH_PROFILER_DIR", "."))
if os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", None) is None
else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", "."))
),
# If set, sgl_diffusion will run in development mode, which will enable
# some additional endpoints for developing and debugging,
# e.g. `/reset_prefix_cache`
"SGL_DIFFUSION_SERVER_DEV_MODE": lambda: bool(
int(os.getenv("SGL_DIFFUSION_SERVER_DEV_MODE", "0"))
"SGLANG_DIFFUSION_SERVER_DEV_MODE": lambda: bool(
int(os.getenv("SGLANG_DIFFUSION_SERVER_DEV_MODE", "0"))
),
# If set, sgl_diffusion will enable stage logging, which will print the time
# taken for each stage
"SGL_DIFFUSION_STAGE_LOGGING": lambda: bool(
int(os.getenv("SGL_DIFFUSION_STAGE_LOGGING", "0"))
"SGLANG_DIFFUSION_STAGE_LOGGING": lambda: bool(
int(os.getenv("SGLANG_DIFFUSION_STAGE_LOGGING", "0"))
),
}

6
python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py
View File

@@ -21,10 +21,10 @@
# recompilation of the code every time we want to switch between different
# versions. This current implementation, with a **pure** Python wrapper, is
# more flexible. We can easily switch between different versions of NCCL by
# changing the environment variable `SGL_DIFFUSION_NCCL_SO_PATH`, or the `so_file`
# changing the environment variable `SGLANG_DIFFUSION_NCCL_SO_PATH`, or the `so_file`
# variable in the code.
# TODO(will): support SGL_DIFFUSION_NCCL_SO_PATH
# TODO(will): support SGLANG_DIFFUSION_NCCL_SO_PATH
import ctypes
import platform
@@ -281,7 +281,7 @@ class NCCLLibrary:
"Otherwise, the nccl library might not exist, be corrupted "
"or it does not support the current platform %s."
"If you already have the library, please set the "
"environment variable SGL_DIFFUSION_NCCL_SO_PATH"
"environment variable SGLANG_DIFFUSION_NCCL_SO_PATH"
" to point to the correct nccl library path.",
so_file,
platform.platform(),

4
python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py
View File

@@ -119,9 +119,9 @@ class SlidingTileAttentionImpl(AttentionImpl):
raise ValueError("st attn not supported")
# TODO(will-refactor): for now this is the mask strategy, but maybe we should
# have a more general config for STA?
config_file = envs.SGL_DIFFUSION_ATTENTION_CONFIG
config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG
if config_file is None:
raise ValueError("SGL_DIFFUSION_ATTENTION_CONFIG is not set")
raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set")
# TODO(kevin): get mask strategy for different STA modes
with open(config_file) as f:

2
python/sglang/multimodal_gen/runtime/layers/attention/selector.py
View File

@@ -110,7 +110,7 @@ def _cached_get_attn_backend(
# Check whether a particular choice of backend was
# previously forced.
#
# THIS SELECTION OVERRIDES THE SGL_DIFFUSION_ATTENTION_BACKEND
# THIS SELECTION OVERRIDES THE SGLANG_DIFFUSION_ATTENTION_BACKEND
# ENVIRONMENT VARIABLE.
from sglang.multimodal_gen.runtime.platforms import current_platform

4
python/sglang/multimodal_gen/runtime/managers/forward_context.py
View File

@@ -19,11 +19,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
# TODO(will): check if this is needed
# track_batchsize: bool = envs.SGL_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0
# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0
track_batchsize: bool = False
last_logging_time: float = 0
forward_start_time: float = 0
# batchsize_logging_interval: float = envs.SGL_DIFFUSION_LOG_BATCHSIZE_INTERVAL
# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL
batchsize_logging_interval: float = 1000
batchsize_forward_time: defaultdict = defaultdict(list)

4
python/sglang/multimodal_gen/runtime/models/registry.py
View File

@@ -102,7 +102,7 @@ def _discover_and_register_models() -> dict[str, tuple[str, str, str]]:
return discovered_models
_SGL_DIFFUSION_MODELS = _discover_and_register_models()
_SGLANG_DIFFUSION_MODELS = _discover_and_register_models()
_SUBPROCESS_COMMAND = [
sys.executable,
@@ -361,6 +361,6 @@ ModelRegistry = _ModelRegistry(
component_name,
mod_relname,
cls_name,
) in _SGL_DIFFUSION_MODELS.items()
) in _SGLANG_DIFFUSION_MODELS.items()
}
)

2
python/sglang/multimodal_gen/runtime/pipelines/stages/base.py
View File

@@ -187,7 +187,7 @@ class PipelineStage(ABC):
# Execute the actual stage logic
logging_info = getattr(batch, "logging_info", None)
if envs.SGL_DIFFUSION_STAGE_LOGGING:
if envs.SGLANG_DIFFUSION_STAGE_LOGGING:
logger.info("[%s] Starting execution", stage_name)
start_time = time.perf_counter()

4
python/sglang/multimodal_gen/runtime/pipelines/stages/denoising.py
View File

@@ -1284,9 +1284,9 @@ class DenoisingStage(PipelineStage):
elif STA_mode == STA_Mode.STA_INFERENCE:
import sglang.multimodal_gen.envs as envs
config_file = envs.SGL_DIFFUSION_ATTENTION_CONFIG
config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG
if config_file is None:
raise ValueError("SGL_DIFFUSION_ATTENTION_CONFIG is not set")
raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set")
STA_param = configure_sta(
mode=STA_Mode.STA_INFERENCE,
layer_num=layer_num,

4
python/sglang/multimodal_gen/runtime/platforms/rocm.py
View File

@@ -69,8 +69,8 @@ class RocmPlatform(Platform):
dtype: torch.dtype,
) -> str:
logger.info(
"Trying SGL_DIFFUSION_ATTENTION_BACKEND=%s",
envs.SGL_DIFFUSION_ATTENTION_BACKEND,
"Trying SGLANG_DIFFUSION_ATTENTION_BACKEND=%s",
envs.SGLANG_DIFFUSION_ATTENTION_BACKEND,
)
if selected_backend == AttentionBackendEnum.TORCH_SDPA:

14
python/sglang/multimodal_gen/runtime/utils/logging_utils.py
View File

@@ -17,10 +17,10 @@ from typing import Any, cast
import sglang.multimodal_gen.envs as envs
SGL_DIFFUSION_CONFIGURE_LOGGING = envs.SGL_DIFFUSION_CONFIGURE_LOGGING
SGL_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGL_DIFFUSION_LOGGING_CONFIG_PATH
SGL_DIFFUSION_LOGGING_LEVEL = envs.SGL_DIFFUSION_LOGGING_LEVEL
SGL_DIFFUSION_LOGGING_PREFIX = envs.SGL_DIFFUSION_LOGGING_PREFIX
SGLANG_DIFFUSION_CONFIGURE_LOGGING = envs.SGLANG_DIFFUSION_CONFIGURE_LOGGING
SGLANG_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGLANG_DIFFUSION_LOGGING_CONFIG_PATH
SGLANG_DIFFUSION_LOGGING_LEVEL = envs.SGLANG_DIFFUSION_LOGGING_LEVEL
SGLANG_DIFFUSION_LOGGING_PREFIX = envs.SGLANG_DIFFUSION_LOGGING_PREFIX
RED = "\033[91m"
GREEN = "\033[92m"
@@ -28,7 +28,7 @@ YELLOW = "\033[93m"
RESET = "\033[0;0m"
_FORMAT = (
f"{SGL_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s "
f"{SGLANG_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s "
"[%(filename)s: %(lineno)d] %(message)s"
)
@@ -47,7 +47,7 @@ DEFAULT_LOGGING_CONFIG = {
"sgl_diffusion": {
"class": "logging.StreamHandler",
"formatter": "sgl_diffusion",
"level": SGL_DIFFUSION_LOGGING_LEVEL,
"level": SGLANG_DIFFUSION_LOGGING_LEVEL,
"stream": "ext://sys.stdout",
},
},
@@ -340,7 +340,7 @@ def enable_trace_function_call(log_file_path: str, root_dir: str | None = None):
will have the trace enabled. Other threads will not be affected.
"""
logger.warning(
"SGL_DIFFUSION_TRACE_FUNCTION is enabled. It will record every"
"SGLANG_DIFFUSION_TRACE_FUNCTION is enabled. It will record every"
" function executed by Python. This will slow down the code. It "
"is suggested to be used for debugging hang or crashes only."
)

8
python/sglang/multimodal_gen/runtime/utils/performance_logger.py
View File

@@ -6,10 +6,13 @@ import os
import subprocess
import time
from datetime import datetime
from pathlib import Path
from typing import Any
from dateutil.tz import UTC
import sglang
def get_diffusion_perf_log_dir() -> str:
"""
@@ -26,7 +29,10 @@ def get_diffusion_perf_log_dir() -> str:
return os.path.abspath(log_dir)
if log_dir is None:
# Not set, use default
return os.path.join(os.path.expanduser("~/.cache/sglang"), "logs")
sglang_path = Path(sglang.__file__).resolve()
# .gitignore
target_path = (sglang_path.parent / "../../.cache/logs").resolve()
return str(target_path)
# Is set, but is an empty string
return ""

490
python/sglang/multimodal_gen/test/server/perf_baselines.json
View File

@@ -1,155 +1,351 @@
{
"metadata": {
"model": "Diffusion Server",
"hardware": "CI H100 80GB pool",
"description": "Reference numbers captured from the CI diffusion server baseline run"
},
"tolerances": {
"e2e": 0.25,
"stage": 0.3,
"denoise_step": 0.2,
"denoise_agg": 0.1
},
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
],
"warmup_requests": {
"text": 1,
"image_edit": 0
}
},
"scenarios": {
"text_to_image": {
"notes": "Single-image generation using the default prompt",
"expected_e2e_ms": 74500.0,
"expected_avg_denoise_ms": 422.42,
"expected_median_denoise_ms": 410.62,
"stages_ms": {
"InputValidationStage": 0.1,
"TextEncodingStage": 834.2,
"ConditioningStage": 0.1,
"TimestepPreparationStage": 10.6,
"LatentPreparationStage": 9.0,
"DenoisingStage": 21202.6,
"DecodingStage": 476.12
},
"denoise_step_ms": {
"0": 1077.77, "1": 345.13, "2": 413.8, "3": 405.49, "4": 408.14, "5": 409.06,
"6": 408.85, "7": 410.53, "8": 407.51, "9": 409.44, "10": 408.65, "11": 410.14,
"12": 411.74, "13": 409.59, "14": 409.17, "15": 410.78, "16": 410.66, "17": 410.58,
"18": 411.27, "19": 410.51, "20": 409.03, "21": 410.16, "22": 409.42, "23": 411.03,
"24": 410.18, "25": 409.72, "26": 410.26, "27": 410.21, "28": 410.71, "29": 410.76,
"30": 411.06, "31": 410.1, "32": 410.55, "33": 410.77, "34": 410.74, "35": 411.75,
"36": 410.78, "37": 411.56, "38": 410.85, "39": 411.08, "40": 411.12, "41": 411.1,
"42": 411.09, "43": 410.87, "44": 411.37, "45": 411.68, "46": 411.0, "47": 410.09,
"48": 412.72, "49": 410.42
}
"metadata": {
"model": "Diffusion Server",
"hardware": "CI H100 80GB pool",
"description": "Reference numbers captured from the CI diffusion server baseline run"
},
"image_edit": {
"notes": "single uploaded reference image, Qwen/Qwen-Image-Edit",
"expected_e2e_ms": 138500.0,
"expected_avg_denoise_ms": 720.0,
"expected_median_denoise_ms": 718.0,
"stages_ms": {
"InputValidationStage": 23,
"ImageEncodingStage": 1350.0,
"ImageVAEEncodingStage": 340.0,
"ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78,
"LatentPreparationStage": 10.0,
"DenoisingStage": 36000.0,
"DecodingStage": 850.0
},
"denoise_step_ms": {
"0": 720.0, "1": 720.0, "2": 720.0, "3": 720.0, "4": 720.0, "5": 720.0,
"6": 720.0, "7": 720.0, "8": 720.0, "9": 720.0, "10": 720.0, "11": 720.0,
"12": 720.0, "13": 720.0, "14": 720.0, "15": 720.0, "16": 720.0, "17": 720.0,
"18": 720.0, "19": 720.0, "20": 720.0, "21": 720.0, "22": 720.0, "23": 720.0,
"24": 720.0, "25": 720.0, "26": 720.0, "27": 720.0, "28": 720.0, "29": 720.0,
"30": 720.0, "31": 720.0, "32": 720.0, "33": 720.0, "34": 720.0, "35": 720.0,
"36": 720.0, "37": 720.0, "38": 720.0, "39": 720.0, "40": 720.0, "41": 720.0,
"42": 720.0, "43": 720.0, "44": 720.0, "45": 720.0, "46": 720.0, "47": 720.0,
"48": 720.0, "49": 720.0
}
"tolerances": {
"e2e": 0.05,
"stage": 0.05,
"denoise_step": 0.15,
"denoise_agg": 0.08
},
"text_to_video": {
"notes": "Single-video generation using the default prompt",
"expected_e2e_ms": 95616.59,
"expected_avg_denoise_ms": 1798.77,
"expected_median_denoise_ms": 1786.78,
"stages_ms": {
"InputValidationStage": 1.03,
"TextEncodingStage": 3450.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 15.0,
"DenoisingStage": 90100.0,
"DecodingStage": 3650.0
},
"denoise_step_ms": {
"0": 3500.0, "10": 1800.0, "20": 1800.0, "29": 1800.0, "39": 1800.0, "49": 1800.0
},
"frames_per_second": 0.51,
"total_frames": 49,
"avg_frame_time_ms": 1951.36
"improvement_reporting": {
"threshold": 0.2
},
"image_to_video": {
"notes": "Wan-AI/Wan2.2-I2V-A14B",
"expected_e2e_ms": 282500.0,
"expected_avg_denoise_ms": 7000.0,
"expected_median_denoise_ms": 7000.19,
"stages_ms": {
"InputValidationStage": 20.0,
"TextEncodingStage": 2100.0,
"ConditioningStage": 2.0,
"TimestepPreparationStage": 2.0,
"LatentPreparationStage": 10.0,
"ImageVAEEncodingStage": 1800.0,
"DenoisingStage": 278000.0,
"DecodingStage": 2700.0
},
"denoise_step_ms": {
"0": 24000.0,
"8": 7000.0,
"16": 7000.0,
"23": 7000.0,
"31": 7000.0,
"39": 7000.0
"sampling": {
"step_fractions": [
0.0,
0.2,
0.4,
0.6,
0.8,
1.0
],
"warmup_requests": {
"text": 1,
"image_edit": 0
}
},
"text_image_to_video": {
"notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B",
"expected_e2e_ms": 178300.0,
"expected_avg_denoise_ms": 3250.0,
"expected_median_denoise_ms": 3260.0,
"stages_ms": {
"InputValidationStage": 80.0,
"TextEncodingStage": 3000.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 30.0,
"DenoisingStage": 162900.0,
"DecodingStage": 13500.0
},
"denoise_step_ms": {
"0": 3700.0,
"10": 3300.0,
"20": 3300.0,
"29": 3300.0,
"39": 3300.0,
"49": 3300.0
},
"frames_per_second": null,
"total_frames": null,
"avg_frame_time_ms": null
"scenarios": {
"qwen_image_t2i": {
"notes": "Single-image generation using the default prompt",
"expected_e2e_ms": 74500.0,
"expected_avg_denoise_ms": 422.42,
"expected_median_denoise_ms": 410.62,
"stages_ms": {
"InputValidationStage": 0.1,
"TextEncodingStage": 834.2,
"ConditioningStage": 0.1,
"TimestepPreparationStage": 10.6,
"LatentPreparationStage": 11.8,
"DenoisingStage": 21202.6,
"DecodingStage": 476.12
},
"denoise_step_ms": {
"0": 1077.77,
"1": 345.13,
"2": 413.8,
"3": 405.49,
"4": 408.14,
"5": 409.06,
"6": 408.85,
"7": 410.53,
"8": 407.51,
"9": 409.44,
"10": 408.65,
"11": 410.14,
"12": 411.74,
"13": 409.59,
"14": 409.17,
"15": 410.78,
"16": 410.66,
"17": 410.58,
"18": 411.27,
"19": 410.51,
"20": 409.03,
"21": 410.16,
"22": 409.42,
"23": 411.03,
"24": 410.18,
"25": 409.72,
"26": 410.26,
"27": 410.21,
"28": 410.71,
"29": 410.76,
"30": 411.06,
"31": 410.1,
"32": 410.55,
"33": 410.77,
"34": 410.74,
"35": 411.75,
"36": 410.78,
"37": 411.56,
"38": 410.85,
"39": 411.08,
"40": 411.12,
"41": 411.1,
"42": 411.09,
"43": 410.87,
"44": 411.37,
"45": 411.68,
"46": 411.0,
"47": 410.09,
"48": 412.72,
"49": 410.42
}
},
"flux_image_t2i": {
"stages_ms": {
"InputValidationStage": 0.08,
"TextEncodingStage": 87.36,
"ConditioningStage": 0.024,
"TimestepPreparationStage": 2.53,
"LatentPreparationStage": 6.21,
"DenoisingStage": 8161.97,
"DecodingStage": 378.21
},
"denoise_step_ms": {
"0": 56.16,
"10": 163.54,
"20": 165.94,
"29": 165.26,
"39": 165.16,
"49": 165.36
},
"expected_e2e_ms": 8764.73,
"expected_avg_denoise_ms": 160.79,
"expected_median_denoise_ms": 165.14
},
"qwen_image_edit_ti2i": {
"notes": "single uploaded reference image, Qwen/Qwen-Image-Edit",
"expected_e2e_ms": 138500.0,
"expected_avg_denoise_ms": 720.0,
"expected_median_denoise_ms": 718.0,
"stages_ms": {
"InputValidationStage": 23,
"ImageEncodingStage": 1350.0,
"ImageVAEEncodingStage": 340.0,
"ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78,
"LatentPreparationStage": 15.0,
"DenoisingStage": 36000.0,
"DecodingStage": 850.0
},
"denoise_step_ms": {
"0": 720.0,
"1": 720.0,
"2": 720.0,
"3": 720.0,
"4": 720.0,
"5": 720.0,
"6": 720.0,
"7": 720.0,
"8": 720.0,
"9": 720.0,
"10": 720.0,
"11": 720.0,
"12": 720.0,
"13": 720.0,
"14": 720.0,
"15": 720.0,
"16": 720.0,
"17": 720.0,
"18": 720.0,
"19": 720.0,
"20": 720.0,
"21": 720.0,
"22": 720.0,
"23": 720.0,
"24": 720.0,
"25": 720.0,
"26": 720.0,
"27": 720.0,
"28": 720.0,
"29": 720.0,
"30": 720.0,
"31": 720.0,
"32": 720.0,
"33": 720.0,
"34": 720.0,
"35": 720.0,
"36": 720.0,
"37": 720.0,
"38": 720.0,
"39": 720.0,
"40": 720.0,
"41": 720.0,
"42": 720.0,
"43": 720.0,
"44": 720.0,
"45": 720.0,
"46": 720.0,
"47": 720.0,
"48": 720.0,
"49": 720.0
}
},
"fastwan2_1_t2v": {
"notes": "Single-video generation using the default prompt",
"expected_e2e_ms": 95616.59,
"expected_avg_denoise_ms": 1798.77,
"expected_median_denoise_ms": 1786.78,
"stages_ms": {
"InputValidationStage": 1.03,
"TextEncodingStage": 3450.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 15.0,
"DenoisingStage": 90100.0,
"DecodingStage": 3650.0
},
"denoise_step_ms": {
"0": 3500.0,
"10": 1800.0,
"20": 1800.0,
"29": 1800.0,
"39": 1800.0,
"49": 1800.0
},
"frames_per_second": 0.51,
"total_frames": 49,
"avg_frame_time_ms": 1951.36
},
"wan2_2_i2v_a14b": {
"notes": "Wan-AI/Wan2.2-I2V-A14B",
"expected_e2e_ms": 282500.0,
"expected_avg_denoise_ms": 7000.0,
"expected_median_denoise_ms": 7000.19,
"stages_ms": {
"InputValidationStage": 26.26,
"TextEncodingStage": 2749.6,
"ConditioningStage": 2.0,
"TimestepPreparationStage": 2.0,
"LatentPreparationStage": 10.0,
"ImageVAEEncodingStage": 2031.0,
"DenoisingStage": 278000.0,
"DecodingStage": 2849.6
},
"denoise_step_ms": {
"0": 24000.0,
"8": 7000.0,
"16": 7000.0,
"23": 7000.0,
"31": 7000.0,
"39": 7000.0
}
},
"wan2_1_i2v_14b_480P": {
"stages_ms": {
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 5432.37,
"1": 4861.66,
"2": 4890.23,
"3": 4919.9,
"4": 4929.49,
"5": 4927.35,
"6": 4927.6,
"7": 4932.02,
"8": 4897.83,
"9": 4897.16,
"10": 4899.59,
"11": 4901.51,
"12": 4896.38,
"13": 4914.3,
"14": 4901.91,
"15": 4898.58,
"16": 4903.18,
"17": 4899.54,
"18": 4910.5,
"19": 4898.18,
"20": 4901.56,
"21": 4903.12,
"22": 4896.82,
"23": 4901.01,
"24": 4897.42,
"25": 4911.94,
"26": 4901.29,
"27": 4894.79,
"28": 4899.57,
"29": 4900.63,
"30": 4904.78,
"31": 4899.79,
"32": 4898.01,
"33": 4910.81,
"34": 4901.16,
"35": 4898.48,
"36": 4894.97,
"37": 4905.9,
"38": 4901.52,
"39": 4898.56,
"40": 4900.19,
"41": 4896.5,
"42": 4899.89,
"43": 4898.45,
"44": 4897.27,
"45": 4895.47,
"46": 4889.26,
"47": 4887.44,
"48": 4894.07,
"49": 4888.59
},
"expected_e2e_ms": 254552.63,
"expected_avg_denoise_ms": 4912.17,
"expected_median_denoise_ms": 4899.69
},
"wan2_2_i2v_14b_720P": {
"stages_ms": {
"InputValidationStage": 47.04,
"TextEncodingStage": 2321.27,
"ImageEncodingStage": 3244.34,
"ConditioningStage": 0.0234,
"TimestepPreparationStage": 2.88,
"LatentPreparationStage": 5.24,
"ImageVAEEncodingStage": 1887.64,
"DenoisingStage": 245826.78,
"DecodingStage": 2882.45,
"per_frame_generation": null
},
"denoise_step_ms": {
"0": 5429.38,
"10": 4901.39,
"20": 4912.89,
"29": 4900.75,
"39": 4906.23,
"49": 4892.55
},
"expected_e2e_ms": 254850.94,
"expected_avg_denoise_ms": 4914.73,
"expected_median_denoise_ms": 4903.27
},
"wan2_2_ti2v_5b": {
"notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B",
"expected_e2e_ms": 178300.0,
"expected_avg_denoise_ms": 3250.0,
"expected_median_denoise_ms": 3260.0,
"stages_ms": {
"InputValidationStage": 150.0,
"TextEncodingStage": 3000.0,
"ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0,
"LatentPreparationStage": 30.0,
"DenoisingStage": 162900.0,
"DecodingStage": 14767.0
},
"denoise_step_ms": {
"0": 3700.0,
"10": 3300.0,
"20": 3300.0,
"29": 3300.0,
"39": 3300.0,
"49": 3300.0
},
"frames_per_second": null,
"total_frames": null,
"avg_frame_time_ms": null
}
}
}
}

169
python/sglang/multimodal_gen/test/server/test_server_performance.py
View File

@@ -1,6 +1,5 @@
"""
Config-driven diffusion performance test with pytest parametrization.
Adding a new model/scenario = adding one DiffusionCase entry in diffusion_config.py.
"""
from __future__ import annotations
@@ -15,20 +14,20 @@ from openai import OpenAI
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.conftest import _GLOBAL_PERF_RESULTS
from sglang.multimodal_gen.test.server.diffusion_config import (
BASELINE_CONFIG,
DIFFUSION_CASES,
DiffusionCase,
)
from sglang.multimodal_gen.test.server.diffusion_server import (
from sglang.multimodal_gen.test.server.test_server_utils import (
VALIDATOR_REGISTRY,
PerformanceValidator,
ServerContext,
ServerManager,
VideoPerformanceValidator,
WarmupRunner,
download_image_from_url,
)
from sglang.multimodal_gen.test.server.testcase_configs import (
BASELINE_CONFIG,
DIFFUSION_CASES,
DiffusionTestCase,
PerformanceSummary,
)
from sglang.multimodal_gen.test.test_utils import (
get_dynamic_server_port,
read_perf_records,
@@ -42,13 +41,13 @@ logger = init_logger(__name__)
@pytest.fixture(params=DIFFUSION_CASES, ids=lambda c: c.id)
def case(request) -> DiffusionCase:
"""Provide a DiffusionCase for each test."""
def case(request) -> DiffusionTestCase:
"""Provide a DiffusionTestCase for each test."""
return request.param
@pytest.fixture
def diffusion_server(case: DiffusionCase) -> ServerContext:
def diffusion_server(case: DiffusionTestCase) -> ServerContext:
"""Start a diffusion server for a single case and tear it down afterwards."""
default_port = get_dynamic_server_port()
port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port))
@@ -79,17 +78,17 @@ def diffusion_server(case: DiffusionCase) -> ServerContext:
)
warmup.run_text_warmups(case.warmup_text)
if case.warmup_edit > 0 and case.image_edit_prompt and case.image_edit_path:
if case.warmup_edit > 0 and case.edit_prompt and case.image_path:
# Handle URL or local path
image_path = case.image_edit_path
image_path = case.image_path
if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path))
image_path = download_image_from_url(str(case.image_path))
else:
image_path = Path(case.image_edit_path)
image_path = Path(case.image_path)
warmup.run_edit_warmups(
count=case.warmup_edit,
edit_prompt=case.image_edit_prompt,
edit_prompt=case.edit_prompt,
image_path=image_path,
)
except Exception as exc:
@@ -132,12 +131,13 @@ class TestDiffusionPerformance:
def _run_and_collect(
self,
ctx: ServerContext,
case: DiffusionCase,
case: DiffusionTestCase,
generate_fn: Callable[[], None],
) -> tuple[dict, dict]:
"""Run generation and collect performance records."""
log_path = ctx.perf_log_path
prev_len = len(read_perf_records(log_path))
log_wait_timeout = 1200
generate_fn()
@@ -145,22 +145,25 @@ class TestDiffusionPerformance:
"total_inference_time",
prev_len,
log_path,
timeout=log_wait_timeout,
)
scenario = BASELINE_CONFIG.scenarios[case.scenario_name]
stage_metrics, _ = wait_for_stage_metrics(
perf_record.get("request_id", ""),
prev_len,
len(scenario.stages_ms),
log_path,
)
stage_metrics = {}
if perf_record:
stage_metrics, _ = wait_for_stage_metrics(
perf_record.get("request_id", ""),
prev_len,
log_path,
timeout=log_wait_timeout,
)
return perf_record, stage_metrics
def _generate_for_case(
self,
ctx: ServerContext,
case: DiffusionCase,
case: DiffusionTestCase,
) -> Callable[[], None]:
"""Return appropriate generation function for the case."""
client = self._client(ctx)
@@ -192,21 +195,35 @@ class TestDiffusionPerformance:
job = client.videos.create(**create_kwargs) # type: ignore[attr-defined]
video_id = job.id
deadline = time.time() + 600
job_completed = False
is_baseline_generation_mode = (
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
)
timeout = 3600.0 if is_baseline_generation_mode else 600.0
deadline = time.time() + timeout
while True:
page = client.videos.list() # type: ignore[attr-defined]
item = next((v for v in page.data if v.id == video_id), None)
if item and getattr(item, "status", None) == "completed":
job_completed = True
break
if time.time() > deadline:
pytest.fail(
f"{case.id}: video job {video_id} did not complete in time"
)
break
time.sleep(5)
if not job_completed:
if is_baseline_generation_mode:
logger.warning(
f"{case.id}: video job {video_id} timed out during baseline generation. "
"Attempting to collect performance data anyway."
)
return b""
pytest.fail(f"{case.id}: video job {video_id} did not complete in time")
# download video
resp = client.videos.download_content(video_id=video_id) # type: ignore[attr-defined]
content = resp.read()
@@ -235,14 +252,14 @@ class TestDiffusionPerformance:
def generate_image_edit():
"""TI2I: Text + Image ? Image edit."""
if not case.image_edit_prompt or not case.image_edit_path:
if not case.edit_prompt or not case.image_path:
pytest.skip(f"{case.id}: no edit config")
# Handle URL or local path
if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path))
image_path = download_image_from_url(str(case.image_path))
else:
image_path = Path(case.image_edit_path)
image_path = Path(case.image_path)
if not image_path.exists():
pytest.skip(f"{case.id}: file missing: {image_path}")
@@ -250,7 +267,7 @@ class TestDiffusionPerformance:
result = client.images.edit(
model=case.model_path,
image=fh,
prompt=case.image_edit_prompt,
prompt=case.edit_prompt,
n=1,
size=case.output_size,
response_format="b64_json",
@@ -275,21 +292,21 @@ class TestDiffusionPerformance:
def generate_image_to_video():
"""I2V: Image ? Video (optional prompt)."""
if not case.image_edit_path:
if not case.image_path:
pytest.skip(f"{case.id}: no input image configured")
# Handle URL or local path
if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path))
image_path = download_image_from_url(str(case.image_path))
else:
image_path = Path(case.image_edit_path)
image_path = Path(case.image_path)
if not image_path.exists():
pytest.skip(f"{case.id}: file missing: {image_path}")
with image_path.open("rb") as fh:
_create_and_download_video(
model=case.model_path,
prompt=case.image_edit_prompt,
prompt=case.edit_prompt,
size=case.output_size,
seconds=video_seconds,
input_reference=fh,
@@ -297,49 +314,55 @@ class TestDiffusionPerformance:
def generate_text_image_to_video():
"""TI2V: Text + Image ? Video."""
if not case.image_edit_prompt or not case.image_edit_path:
if not case.edit_prompt or not case.image_path:
pytest.skip(f"{case.id}: no edit config")
# Handle URL or local path
if case.is_image_url():
image_path = download_image_from_url(str(case.image_edit_path))
image_path = download_image_from_url(str(case.image_path))
else:
image_path = Path(case.image_edit_path)
image_path = Path(case.image_path)
if not image_path.exists():
pytest.skip(f"{case.id}: file missing: {image_path}")
with image_path.open("rb") as fh:
_create_and_download_video(
model=case.model_path,
prompt=case.image_edit_prompt,
prompt=case.edit_prompt,
size=case.output_size,
seconds=video_seconds,
input_reference=fh,
)
if case.modality == "video":
if case.image_edit_path and case.image_edit_prompt:
if case.image_path and case.edit_prompt:
return generate_text_image_to_video
elif case.image_edit_path:
elif case.image_path:
return generate_image_to_video
else:
return generate_video
# Image modality
if case.image_edit_prompt and case.image_edit_path:
if case.edit_prompt and case.image_path:
return generate_image_edit
return generate_image
def _validate_and_record(
self,
case: DiffusionCase,
case: DiffusionTestCase,
perf_record: dict,
stage_metrics: dict,
) -> None:
"""Validate metrics and record results."""
scenario = BASELINE_CONFIG.scenarios[case.scenario_name]
is_baseline_generation_mode = os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
scenario = BASELINE_CONFIG.scenarios.get(case.id)
if scenario is None:
if is_baseline_generation_mode:
scenario = {} # Dummy scenario
else:
pytest.fail(f"Testcase '{case.id}' not in perf_baselines.json")
validator_name = case.custom_validator or "default"
validator_class = VALIDATOR_REGISTRY.get(validator_name, PerformanceValidator)
@@ -349,10 +372,17 @@ class TestDiffusionPerformance:
step_fractions=BASELINE_CONFIG.step_fractions,
)
if isinstance(validator, VideoPerformanceValidator):
summary = validator.validate(perf_record, stage_metrics, case.num_frames)
else:
summary = validator.validate(perf_record, stage_metrics)
summary = validator.collect_metrics(perf_record, stage_metrics)
if is_baseline_generation_mode:
self._dump_baseline_scenario(case, summary)
return
try:
validator.validate(perf_record, stage_metrics, case.num_frames)
except AssertionError:
self._dump_baseline_scenario(case, summary)
raise
if case.modality == "video" and summary.frames_per_second:
logger.info(
@@ -428,9 +458,46 @@ class TestDiffusionPerformance:
summary.avg_frame_time_ms,
)
def _dump_baseline_scenario(
self, case: DiffusionTestCase, summary: "PerformanceSummary"
) -> None:
"""Dump performance metrics as a JSON scenario for baselines."""
import json
denoise_steps_formatted = {
str(k): round(v, 2) for k, v in summary.sampled_steps.items()
}
stages_formatted = {k: round(v, 2) for k, v in summary.stage_metrics.items()}
baseline = {
"stages_ms": stages_formatted,
"denoise_step_ms": denoise_steps_formatted,
"expected_e2e_ms": round(summary.e2e_ms, 2),
"expected_avg_denoise_ms": round(summary.avg_denoise_ms, 2),
"expected_median_denoise_ms": round(summary.median_denoise_ms, 2),
}
# Video-specific metrics
if case.modality == "video":
if "per_frame_generation" not in baseline["stages_ms"]:
baseline["stages_ms"]["per_frame_generation"] = (
round(summary.avg_frame_time_ms, 2)
if summary.avg_frame_time_ms
else None
)
output = f"""
To add this baseline, copy the following JSON snippet into
the "scenarios" section of perf_baselines.json:
"{case.id}": {json.dumps(baseline, indent=4)}
"""
print(output)
def test_diffusion_perf(
self,
case: DiffusionCase,
case: DiffusionTestCase,
diffusion_server: ServerContext,
):
"""Single parametrized test that runs for all cases.

218
python/sglang/multimodal_gen/test/server/diffusion_server.py → python/sglang/multimodal_gen/test/server/test_server_utils.py
View File

@@ -7,7 +7,9 @@ from __future__ import annotations
import os
import statistics
import subprocess
import sys
import tempfile
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
@@ -18,7 +20,7 @@ from openai import OpenAI
from sglang.multimodal_gen.runtime.utils.common import kill_process_tree
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.test.server.diffusion_config import (
from sglang.multimodal_gen.test.server.testcase_configs import (
PerformanceSummary,
ScenarioConfig,
ToleranceConfig,
@@ -74,6 +76,7 @@ class ServerContext:
perf_log_path: Path
log_dir: Path
_stdout_fh: Any = field(repr=False)
_log_thread: threading.Thread | None = field(default=None, repr=False)
def cleanup(self) -> None:
"""Clean up server resources."""
@@ -127,19 +130,42 @@ class ServerManager:
command.extend(self.extra_args.strip().split())
env = os.environ.copy()
env["SGL_DIFFUSION_STAGE_LOGGING"] = "1"
env["SGLANG_DIFFUSION_STAGE_LOGGING"] = "1"
env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix()
stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1)
process = subprocess.Popen(
command,
stdout=stdout_fh,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
bufsize=1,
env=env,
)
log_thread = None
if process.stdout:
def _log_pipe(pipe: Any, file: Any) -> None:
"""Read from pipe and write to file and stdout."""
try:
with pipe:
for line in iter(pipe.readline, ""):
sys.stdout.write(line)
file.write(line)
file.flush()
except Exception as e:
logger.error("Log pipe thread error: %s", e)
finally:
file.close()
logger.debug("Log pipe thread finished.")
log_thread = threading.Thread(
target=_log_pipe, args=(process.stdout, stdout_fh)
)
log_thread.daemon = True
log_thread.start()
logger.info(
"[server-test] Starting server pid=%s, model=%s, log=%s",
process.pid,
@@ -157,6 +183,7 @@ class ServerManager:
perf_log_path=perf_log_path,
log_dir=log_dir,
_stdout_fh=stdout_fh,
_log_thread=log_thread,
)
def _wait_for_ready(self, process: subprocess.Popen, stdout_path: Path) -> None:
@@ -264,6 +291,8 @@ class WarmupRunner:
class PerformanceValidator:
"""Validates performance metrics against expectations."""
is_video_gen: bool = False
def __init__(
self,
scenario: ScenarioConfig,
@@ -273,104 +302,134 @@ class PerformanceValidator:
self.scenario = scenario
self.tolerances = tolerances
self.step_fractions = step_fractions
self.is_baseline_generation_mode = (
os.environ.get("SGLANG_GEN_BASELINE", "0") == "1"
)
def _assert_le(self, name: str, actual: float, expected: float, tolerance: float):
"""Assert that actual is less than or equal to expected within a tolerance."""
upper_bound = expected * (1 + tolerance)
assert actual <= upper_bound, (
f"Validation failed for '{name}'.\n"
f" - Actual: {actual:.4f}ms\n"
f" - Expected: {expected:.4f}ms\n"
f" - Limit: {upper_bound:.4f}ms (tolerance: {tolerance:.1%})"
)
def validate(
self, perf_record: dict, stage_metrics: dict, *args, **kwargs
) -> PerformanceSummary:
"""Validate all performance metrics and return summary."""
summary = self.collect_metrics(perf_record, stage_metrics)
if self.is_baseline_generation_mode:
return summary
self._validate_e2e(summary)
self._validate_denoise_agg(summary)
self._validate_denoise_steps(summary)
self._validate_stages(summary)
return summary
def collect_metrics(
self,
perf_record: dict,
stage_metrics: dict,
) -> PerformanceSummary:
"""Validate all performance metrics and return summary."""
self._validate_e2e(perf_record)
avg_denoise, median_denoise = self._validate_denoise_agg(perf_record)
sampled_steps = self._validate_denoise_steps(perf_record)
self._validate_stages(stage_metrics)
return PerformanceSummary(
e2e_ms=float(perf_record["total_duration_ms"]),
avg_denoise_ms=avg_denoise,
median_denoise_ms=median_denoise,
stage_metrics=stage_metrics,
sampled_steps=sampled_steps,
)
def _validate_e2e(self, perf_record: dict) -> None:
"""Validate end-to-end performance."""
"""Collect all performance metrics into a summary without validation."""
e2e_ms = float(perf_record.get("total_duration_ms", 0.0))
assert e2e_ms > 0, "E2E duration missing"
upper = self.scenario.expected_e2e_ms * (1 + self.tolerances.e2e)
assert e2e_ms <= upper, f"E2E {e2e_ms:.2f}ms exceeds {upper:.2f}ms"
def _validate_denoise_agg(self, perf_record: dict) -> tuple[float, float]:
"""Validate aggregate denoising metrics."""
steps = [
s
for s in perf_record.get("steps", []) or []
if s.get("name") == "denoising_step_guided" and "duration_ms" in s
]
assert steps, "Denoising step timings missing"
durations = [float(s["duration_ms"]) for s in steps]
avg = sum(durations) / len(durations)
median = statistics.median(durations)
avg_upper = self.scenario.expected_avg_denoise_ms * (
1 + self.tolerances.denoise_agg
)
med_upper = self.scenario.expected_median_denoise_ms * (
1 + self.tolerances.denoise_agg
)
assert avg <= avg_upper, f"Avg denoise {avg:.2f}ms exceeds {avg_upper:.2f}ms"
assert (
median <= med_upper
), f"Median denoise {median:.2f}ms exceeds {med_upper:.2f}ms"
return avg, median
def _validate_denoise_steps(self, perf_record: dict) -> dict[int, float]:
"""Validate individual denoising steps."""
steps = [
s
for s in perf_record.get("steps", []) or []
if s.get("name") == "denoising_step_guided" and "duration_ms" in s
]
avg_denoise = 0.0
median_denoise = 0.0
if steps:
durations = [float(s["duration_ms"]) for s in steps]
avg_denoise = sum(durations) / len(durations)
median_denoise = statistics.median(durations)
per_step = {
int(s["index"]): float(s["duration_ms"])
for s in steps
if s.get("index") is not None
}
sample_indices = sample_step_indices(per_step, self.step_fractions)
sampled = {idx: per_step[idx] for idx in sample_indices}
sampled_steps = {idx: per_step[idx] for idx in sample_indices}
for idx in sample_indices:
return PerformanceSummary(
e2e_ms=e2e_ms,
avg_denoise_ms=avg_denoise,
median_denoise_ms=median_denoise,
stage_metrics=stage_metrics,
sampled_steps=sampled_steps,
)
def _validate_e2e(self, summary: PerformanceSummary) -> None:
"""Validate end-to-end performance."""
assert summary.e2e_ms > 0, "E2E duration missing"
self._assert_le(
"E2E Latency",
summary.e2e_ms,
self.scenario.expected_e2e_ms,
self.tolerances.e2e,
)
def _validate_denoise_agg(self, summary: PerformanceSummary) -> None:
"""Validate aggregate denoising metrics."""
assert summary.avg_denoise_ms > 0, "Denoising step timings missing"
self._assert_le(
"Average Denoise Step",
summary.avg_denoise_ms,
self.scenario.expected_avg_denoise_ms,
self.tolerances.denoise_agg,
)
self._assert_le(
"Median Denoise Step",
summary.median_denoise_ms,
self.scenario.expected_median_denoise_ms,
self.tolerances.denoise_agg,
)
def _validate_denoise_steps(self, summary: PerformanceSummary) -> None:
"""Validate individual denoising steps."""
for idx, actual in summary.sampled_steps.items():
expected = self.scenario.denoise_step_ms.get(idx)
if expected is None:
continue
self._assert_le(
f"Denoise Step {idx}",
actual,
expected,
self.tolerances.denoise_step,
)
actual = per_step[idx]
upper = expected * (1 + self.tolerances.denoise_step)
assert actual <= upper, f"Step {idx}: {actual:.2f}ms > {upper:.2f}ms"
return sampled
def _validate_stages(self, stage_metrics: dict) -> None:
def _validate_stages(self, summary: PerformanceSummary) -> None:
"""Validate stage-level metrics."""
assert stage_metrics, "Stage metrics missing"
assert summary.stage_metrics, "Stage metrics missing"
for stage, expected in self.scenario.stages_ms.items():
actual = stage_metrics.get(stage)
if stage == "per_frame_generation" and self.is_video_gen:
continue
actual = summary.stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing"
upper = expected * (1 + self.tolerances.stage)
assert actual <= upper, f"Stage {stage}: {actual:.2f}ms > {upper:.2f}ms"
self._assert_le(
f"Stage '{stage}'",
actual,
expected,
self.tolerances.stage,
)
class VideoPerformanceValidator(PerformanceValidator):
"""Extended validator for video diffusion with frame-level metrics."""
is_video_gen = True
def validate(
self,
perf_record: dict,
@@ -385,7 +444,8 @@ class VideoPerformanceValidator(PerformanceValidator):
summary.avg_frame_time_ms = summary.e2e_ms / num_frames
summary.frames_per_second = 1000.0 / summary.avg_frame_time_ms
self._validate_frame_rate(summary)
if not self.is_baseline_generation_mode:
self._validate_frame_rate(summary)
return summary
@@ -393,24 +453,12 @@ class VideoPerformanceValidator(PerformanceValidator):
"""Validate frame generation performance."""
expected_frame_time = self.scenario.stages_ms.get("per_frame_generation")
if expected_frame_time and summary.avg_frame_time_ms:
upper = expected_frame_time * (1 + self.tolerances.stage)
assert (
summary.avg_frame_time_ms <= upper
), f"Avg frame time {summary.avg_frame_time_ms:.2f}ms exceeds {upper:.2f}ms"
def _validate_stages(self, stage_metrics: dict) -> None:
"""Validate video-specific stages."""
assert stage_metrics, "Stage metrics missing"
for stage, expected in self.scenario.stages_ms.items():
if stage == "per_frame_generation":
continue
actual = stage_metrics.get(stage)
assert actual is not None, f"Stage {stage} timing missing"
upper = expected * (1 + self.tolerances.stage)
assert actual <= upper, f"Stage {stage}: {actual:.2f}ms > {upper:.2f}ms"
self._assert_le(
"Average Frame Time",
summary.avg_frame_time_ms,
expected_frame_time,
self.tolerances.stage,
)
# Registry of validators by name

112
python/sglang/multimodal_gen/test/server/diffusion_config.py → python/sglang/multimodal_gen/test/server/testcase_configs.py
View File

@@ -1,5 +1,19 @@
"""
Configuration and data structures for diffusion performance tests.
Usage:
pytest python/sglang/multimodal_gen/test/server/test_server_performance.py
# for a single testcase, look for the name of the testcases in DIFFUSION_CASES
pytest python/sglang/multimodal_gen/test/server/test_server_performance.py -k qwen_image_t2i
To add a new testcase:
1. add your testcase with case-id: `my_new_test_case_id` to DIFFUSION_CASES
2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/test_server_performance.py -k my_new_test_case_id`
3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2
"""
from __future__ import annotations
@@ -78,34 +92,36 @@ class BaselineConfig:
@dataclass(frozen=True)
class DiffusionCase:
class DiffusionTestCase:
"""Configuration for a single model/scenario test case."""
id: str # pytest test id
id: str # pytest test id and scenario name
model_path: str # HF repo or local path
scenario_name: str # key into BASELINE_CONFIG.scenarios
modality: str = "image" # "image" or "video" or "3d"
prompt: str | None = None # text prompt for generation
output_size: str = "1024x1024" # output image dimensions (or video resolution)
# inputs and conditioning
prompt: str | None = None # text prompt for generation
edit_prompt: str | None = None # prompt for editing
image_path: Path | str | None = None # input image/video for editing (Path or URL)
# duration
seconds: int = 4 # for video: duration in seconds
num_frames: int | None = None # for video: number of frames
fps: int | None = None # for video: frames per second
warmup_text: int = 1 # number of text-to-image/video warmups
warmup_edit: int = 0 # number of image/video-edit warmups
image_edit_prompt: str | None = None # prompt for editing
image_edit_path: Path | str | None = (
None # input image/video for editing (Path or URL)
)
startup_grace_seconds: float = 0.0 # wait time after server starts
custom_validator: str | None = None # optional custom validator name
seconds: int = 4 # for video: duration in seconds
def is_image_url(self) -> bool:
"""Check if image_edit_path is a URL."""
if self.image_edit_path is None:
if self.image_path is None:
return False
return isinstance(self.image_edit_path, str) and (
self.image_edit_path.startswith("http://")
or self.image_edit_path.startswith("https://")
return isinstance(self.image_path, str) and (
self.image_path.startswith("http://")
or self.image_path.startswith("https://")
)
@@ -128,12 +144,11 @@ IMAGE_INPUT_FILE = Path(__file__).resolve().parents[1] / "test_files" / "girl.jp
# All test cases with clean default values
# To test different models, simply add more DiffusionCase entries
DIFFUSION_CASES: list[DiffusionCase] = [
DIFFUSION_CASES: list[DiffusionTestCase] = [
# === Text to Image (T2I) ===
DiffusionCase(
DiffusionTestCase(
id="qwen_image_t2i",
model_path="Qwen/Qwen-Image",
scenario_name="text_to_image",
modality="image",
prompt="A futuristic cityscape at sunset with flying cars",
output_size="1024x1024",
@@ -141,10 +156,9 @@ DIFFUSION_CASES: list[DiffusionCase] = [
warmup_edit=0,
startup_grace_seconds=30.0,
),
DiffusionCase(
DiffusionTestCase(
id="flux_image_t2i",
model_path="black-forest-labs/FLUX.1-dev",
scenario_name="text_to_image",
modality="image",
prompt="A futuristic cityscape at sunset with flying cars",
output_size="1024x1024",
@@ -153,24 +167,23 @@ DIFFUSION_CASES: list[DiffusionCase] = [
startup_grace_seconds=30.0,
),
# === Text and Image to Image (TI2I) ===
DiffusionCase(
DiffusionTestCase(
id="qwen_image_edit_ti2i",
model_path="Qwen/Qwen-Image-Edit",
scenario_name="image_edit",
modality="image",
prompt=None, # not used for editing
output_size="1024x1536",
warmup_text=0,
warmup_edit=1,
image_edit_prompt="Convert 2D style to 3D style",
image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
edit_prompt="Convert 2D style to 3D style",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
startup_grace_seconds=30.0,
),
# === Text to Video (T2V) ===
DiffusionCase(
# TODO: FastWan2.1, FastWan2.2
DiffusionTestCase(
id="fastwan2_1_t2v",
model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
scenario_name="text_to_video",
modality="video",
prompt="A curious raccoon",
output_size="848x480",
@@ -181,39 +194,64 @@ DIFFUSION_CASES: list[DiffusionCase] = [
custom_validator="video",
),
# === Image to Video (I2V) ===
DiffusionCase(
id="wan2_2_i2v",
DiffusionTestCase(
id="wan2_2_i2v_a14b",
model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
scenario_name="image_to_video",
modality="video",
prompt="generate", # passing in something since failing if no prompt is passed
warmup_text=0, # warmups only for image gen models
warmup_edit=0,
output_size="832x1104",
image_edit_prompt="generate",
image_edit_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
edit_prompt="generate",
image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
startup_grace_seconds=30.0,
custom_validator="video",
seconds=1,
),
# === Text and Image to Video (TI2V) ===
DiffusionCase(
id="wan2_2_ti2v_5b",
model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
scenario_name="text_image_to_video",
DiffusionTestCase(
id="wan2_1_i2v_14b_480P",
model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
output_size="832x1104",
modality="video",
prompt="Animate this image",
edit_prompt="Add dynamic motion to the scene",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
warmup_text=0, # warmups only for image gen models
warmup_edit=0,
startup_grace_seconds=30.0,
custom_validator="video",
seconds=1,
),
DiffusionTestCase(
id="wan2_2_i2v_14b_720P",
model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
modality="video",
prompt="Animate this image",
edit_prompt="Add dynamic motion to the scene",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
output_size="832x1104",
warmup_text=0, # warmups only for image gen models
warmup_edit=0,
image_edit_prompt="Add dynamic motion to the scene",
image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
startup_grace_seconds=30.0,
custom_validator="video",
seconds=4,
seconds=1,
),
DiffusionTestCase(
id="wan2_2_ti2v_5b",
model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
modality="video",
output_size="832x1104",
prompt="Animate this image",
edit_prompt="Add dynamic motion to the scene",
image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg",
warmup_text=0, # warmups only for image gen models
warmup_edit=0,
startup_grace_seconds=30.0,
custom_validator="video",
seconds=1,
),
]
# Load global configuration
BASELINE_CONFIG = BaselineConfig.load(Path(__file__).with_name("perf_baselines.json"))

26
python/sglang/multimodal_gen/test/test_utils.py
View File

@@ -172,6 +172,11 @@ def wait_for_perf_record(
if rec.get("tag") == tag:
return rec, len(records)
time.sleep(0.5)
if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1":
records = read_perf_records(log_path)
return {}, len(records)
raise AssertionError(
f"Timeout waiting for perf log entry '{tag}' (start_len={prev_len})"
)
@@ -180,15 +185,21 @@ def wait_for_perf_record(
def wait_for_stage_metrics(
request_id: str,
prev_len: int,
expected_count: int,
log_path: Path,
timeout: float = 120.0,
timeout: float = 300.0,
) -> tuple[dict[str, float], int]:
deadline = time.time() + timeout
metrics: dict[str, float] = {}
while time.time() < deadline:
records = read_perf_records(log_path)
for rec in records[prev_len:]:
# Check if the request is completed
if (
rec.get("tag") == "total_inference_time"
and rec.get("request_id") == request_id
):
return metrics, len(records)
if (
rec.get("tag") == "pipeline_stage_metric"
and rec.get("request_id") == request_id
@@ -197,13 +208,12 @@ def wait_for_stage_metrics(
duration = rec.get("duration_ms")
if stage is not None and duration is not None:
metrics[str(stage)] = float(duration)
if len(metrics) >= expected_count:
return metrics, len(records)
time.sleep(0.5)
raise AssertionError(
f"Timeout waiting for stage metrics for request {request_id} "
f"(collected={len(metrics)} expected={expected_count})"
)
if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1":
records = read_perf_records(log_path)
return {}, len(records)
raise AssertionError(f"Timeout waiting for stage metrics for request {request_id} ")
def sample_step_indices(

8
python/sglang/multimodal_gen/utils.py
View File

@@ -48,8 +48,8 @@ PRECISION_TO_TYPE = {
"bf16": torch.bfloat16,
}
STR_BACKEND_ENV_VAR: str = "SGL_DIFFUSION_ATTENTION_BACKEND"
STR_ATTN_CONFIG_ENV_VAR: str = "SGL_DIFFUSION_ATTENTION_CONFIG"
STR_BACKEND_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_BACKEND"
STR_ATTN_CONFIG_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_CONFIG"
def find_nccl_library() -> str:
@@ -59,12 +59,12 @@ def find_nccl_library() -> str:
After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be
found by `ctypes` automatically.
"""
so_file = envs.SGL_DIFFUSION_NCCL_SO_PATH
so_file = envs.SGLANG_DIFFUSION_NCCL_SO_PATH
# manually load the nccl library
if so_file:
logger.info(
"Found nccl from environment variable SGL_DIFFUSION_NCCL_SO_PATH=%s",
"Found nccl from environment variable SGLANG_DIFFUSION_NCCL_SO_PATH=%s",
so_file,
)
else: