diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index f314ae193..d378a9aea 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -244,8 +244,6 @@ jobs: echo "All benchmark tests completed!" - # =============================================== multimodal_gen ==================================================== - # Adding a single CUDA13 smoke test to verify that the kernel builds and runs # TODO: Add back this test when it can pass on CI # cuda13-kernel-smoke-test: diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index 387ecde2b..17a0a14c3 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -15,26 +15,26 @@ from packaging import version logger = logging.getLogger(__name__) if TYPE_CHECKING: - SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60 - SGL_DIFFUSION_NCCL_SO_PATH: str | None = None + SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL: int = 60 + SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None LD_LIBRARY_PATH: str | None = None LOCAL_RANK: int = 0 CUDA_VISIBLE_DEVICES: str | None = None - SGL_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion") - SGL_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion") - SGL_DIFFUSION_CONFIGURE_LOGGING: int = 1 - SGL_DIFFUSION_LOGGING_LEVEL: str = "INFO" - SGL_DIFFUSION_LOGGING_PREFIX: str = "" - SGL_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None - SGL_DIFFUSION_TRACE_FUNCTION: int = 0 - SGL_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork" - SGL_DIFFUSION_TARGET_DEVICE: str = "cuda" + SGLANG_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion") + SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion") + SGLANG_DIFFUSION_CONFIGURE_LOGGING: int = 1 + SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO" + SGLANG_DIFFUSION_LOGGING_PREFIX: str = "" + SGLANG_DIFFUSION_LOGGING_CONFIG_PATH: str | None = None + SGLANG_DIFFUSION_TRACE_FUNCTION: int = 0 + SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD: str = "fork" + SGLANG_DIFFUSION_TARGET_DEVICE: str = "cuda" MAX_JOBS: str | None = None NVCC_THREADS: str | None = None CMAKE_BUILD_TYPE: str | None = None VERBOSE: bool = False - SGL_DIFFUSION_SERVER_DEV_MODE: bool = False - SGL_DIFFUSION_STAGE_LOGGING: bool = False + SGLANG_DIFFUSION_SERVER_DEV_MODE: bool = False + SGLANG_DIFFUSION_STAGE_LOGGING: bool = False def _is_hip(): @@ -165,8 +165,8 @@ environment_variables: dict[str, Callable[[], Any]] = { # ================== Installation Time Env Vars ================== # Target device of sgl-diffusion, supporting [cuda (by default), # rocm, neuron, cpu, openvino] - "SGL_DIFFUSION_TARGET_DEVICE": lambda: os.getenv( - "SGL_DIFFUSION_TARGET_DEVICE", "cuda" + "SGLANG_DIFFUSION_TARGET_DEVICE": lambda: os.getenv( + "SGLANG_DIFFUSION_TARGET_DEVICE", "cuda" ), # Maximum number of compilation jobs to run in parallel. # By default this is the number of CPUs @@ -176,10 +176,10 @@ environment_variables: dict[str, Callable[[], Any]] = { # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU. "NVCC_THREADS": lambda: os.getenv("NVCC_THREADS", None), # If set, sgl_diffusion will use precompiled binaries (*.so) - "SGL_DIFFUSION_USE_PRECOMPILED": lambda: bool( - os.environ.get("SGL_DIFFUSION_USE_PRECOMPILED") + "SGLANG_DIFFUSION_USE_PRECOMPILED": lambda: bool( + os.environ.get("SGLANG_DIFFUSION_USE_PRECOMPILED") ) - or bool(os.environ.get("SGL_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")), + or bool(os.environ.get("SGLANG_DIFFUSION_PRECOMPILED_WHEEL_LOCATION")), # CMake build type # If not set, defaults to "Debug" or "RelWithDebInfo" # Available options: "Debug", "Release", "RelWithDebInfo" @@ -191,36 +191,36 @@ environment_variables: dict[str, Callable[[], Any]] = { # Note that this not only affects how sgl_diffusion finds its configuration files # during runtime, but also affects how sgl_diffusion installs its configuration # files during **installation**. - "SGL_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser( + "SGLANG_DIFFUSION_CONFIG_ROOT": lambda: os.path.expanduser( os.getenv( - "SGL_DIFFUSION_CONFIG_ROOT", + "SGLANG_DIFFUSION_CONFIG_ROOT", os.path.join(get_default_config_root(), "sgl_diffusion"), ) ), # ================== Runtime Env Vars ================== # Root directory for FASTVIDEO cache files # Defaults to `~/.cache/sgl_diffusion` unless `XDG_CACHE_HOME` is set - "SGL_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser( + "SGLANG_DIFFUSION_CACHE_ROOT": lambda: os.path.expanduser( os.getenv( - "SGL_DIFFUSION_CACHE_ROOT", + "SGLANG_DIFFUSION_CACHE_ROOT", os.path.join(get_default_cache_root(), "sgl_diffusion"), ) ), # Interval in seconds to log a warning message when the ring buffer is full - "SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int( - os.environ.get("SGL_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60") + "SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL": lambda: int( + os.environ.get("SGLANG_DIFFUSION_RINGBUFFER_WARNING_INTERVAL", "60") ), # Path to the NCCL library file. It is needed because nccl>=2.19 brought # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234 - "SGL_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get( - "SGL_DIFFUSION_NCCL_SO_PATH", None + "SGLANG_DIFFUSION_NCCL_SO_PATH": lambda: os.environ.get( + "SGLANG_DIFFUSION_NCCL_SO_PATH", None ), - # when `SGL_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl + # when `SGLANG_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl # library file in the locations specified by `LD_LIBRARY_PATH` "LD_LIBRARY_PATH": lambda: os.environ.get("LD_LIBRARY_PATH", None), # Internal flag to enable Dynamo fullgraph capture - "SGL_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool( - os.environ.get("SGL_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0" + "SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE": lambda: bool( + os.environ.get("SGLANG_DIFFUSION_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0" ), # local rank of the process in the distributed setting, used to determine # the GPU device id @@ -228,62 +228,62 @@ environment_variables: dict[str, Callable[[], Any]] = { # used to control the visible devices in the distributed setting "CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None), # timeout for each iteration in the engine - "SGL_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int( - os.environ.get("SGL_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60") + "SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S": lambda: int( + os.environ.get("SGLANG_DIFFUSION_ENGINE_ITERATION_TIMEOUT_S", "60") ), # Logging configuration # If set to 0, sgl_diffusion will not configure logging # If set to 1, sgl_diffusion will configure logging using the default configuration - # or the configuration file specified by SGL_DIFFUSION_LOGGING_CONFIG_PATH - "SGL_DIFFUSION_CONFIGURE_LOGGING": lambda: int( - os.getenv("SGL_DIFFUSION_CONFIGURE_LOGGING", "1") + # or the configuration file specified by SGLANG_DIFFUSION_LOGGING_CONFIG_PATH + "SGLANG_DIFFUSION_CONFIGURE_LOGGING": lambda: int( + os.getenv("SGLANG_DIFFUSION_CONFIGURE_LOGGING", "1") ), - "SGL_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv( - "SGL_DIFFUSION_LOGGING_CONFIG_PATH" + "SGLANG_DIFFUSION_LOGGING_CONFIG_PATH": lambda: os.getenv( + "SGLANG_DIFFUSION_LOGGING_CONFIG_PATH" ), # this is used for configuring the default logging level - "SGL_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv( - "SGL_DIFFUSION_LOGGING_LEVEL", "INFO" + "SGLANG_DIFFUSION_LOGGING_LEVEL": lambda: os.getenv( + "SGLANG_DIFFUSION_LOGGING_LEVEL", "INFO" ), - # if set, SGL_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages - "SGL_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv( - "SGL_DIFFUSION_LOGGING_PREFIX", "" + # if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages + "SGLANG_DIFFUSION_LOGGING_PREFIX": lambda: os.getenv( + "SGLANG_DIFFUSION_LOGGING_PREFIX", "" ), # Trace function calls # If set to 1, sgl_diffusion will trace function calls # Useful for debugging - "SGL_DIFFUSION_TRACE_FUNCTION": lambda: int( - os.getenv("SGL_DIFFUSION_TRACE_FUNCTION", "0") + "SGLANG_DIFFUSION_TRACE_FUNCTION": lambda: int( + os.getenv("SGLANG_DIFFUSION_TRACE_FUNCTION", "0") ), # Path to the attention configuration file. Only used for sliding tile # attention for now. - "SGL_DIFFUSION_ATTENTION_CONFIG": lambda: ( + "SGLANG_DIFFUSION_ATTENTION_CONFIG": lambda: ( None - if os.getenv("SGL_DIFFUSION_ATTENTION_CONFIG", None) is None - else os.path.expanduser(os.getenv("SGL_DIFFUSION_ATTENTION_CONFIG", ".")) + if os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", None) is None + else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_ATTENTION_CONFIG", ".")) ), # Use dedicated multiprocess context for workers. # Both spawn and fork work - "SGL_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv( - "SGL_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork" + "SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD": lambda: os.getenv( + "SGLANG_DIFFUSION_WORKER_MULTIPROC_METHOD", "fork" ), # Enables torch profiler if set. Path to the directory where torch profiler # traces are saved. Note that it must be an absolute path. - "SGL_DIFFUSION_TORCH_PROFILER_DIR": lambda: ( + "SGLANG_DIFFUSION_TORCH_PROFILER_DIR": lambda: ( None - if os.getenv("SGL_DIFFUSION_TORCH_PROFILER_DIR", None) is None - else os.path.expanduser(os.getenv("SGL_DIFFUSION_TORCH_PROFILER_DIR", ".")) + if os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", None) is None + else os.path.expanduser(os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR", ".")) ), # If set, sgl_diffusion will run in development mode, which will enable # some additional endpoints for developing and debugging, # e.g. `/reset_prefix_cache` - "SGL_DIFFUSION_SERVER_DEV_MODE": lambda: bool( - int(os.getenv("SGL_DIFFUSION_SERVER_DEV_MODE", "0")) + "SGLANG_DIFFUSION_SERVER_DEV_MODE": lambda: bool( + int(os.getenv("SGLANG_DIFFUSION_SERVER_DEV_MODE", "0")) ), # If set, sgl_diffusion will enable stage logging, which will print the time # taken for each stage - "SGL_DIFFUSION_STAGE_LOGGING": lambda: bool( - int(os.getenv("SGL_DIFFUSION_STAGE_LOGGING", "0")) + "SGLANG_DIFFUSION_STAGE_LOGGING": lambda: bool( + int(os.getenv("SGLANG_DIFFUSION_STAGE_LOGGING", "0")) ), } diff --git a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py index 40da43f49..598e7be9b 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py +++ b/python/sglang/multimodal_gen/runtime/distributed/device_communicators/pynccl_wrapper.py @@ -21,10 +21,10 @@ # recompilation of the code every time we want to switch between different # versions. This current implementation, with a **pure** Python wrapper, is # more flexible. We can easily switch between different versions of NCCL by -# changing the environment variable `SGL_DIFFUSION_NCCL_SO_PATH`, or the `so_file` +# changing the environment variable `SGLANG_DIFFUSION_NCCL_SO_PATH`, or the `so_file` # variable in the code. -# TODO(will): support SGL_DIFFUSION_NCCL_SO_PATH +# TODO(will): support SGLANG_DIFFUSION_NCCL_SO_PATH import ctypes import platform @@ -281,7 +281,7 @@ class NCCLLibrary: "Otherwise, the nccl library might not exist, be corrupted " "or it does not support the current platform %s." "If you already have the library, please set the " - "environment variable SGL_DIFFUSION_NCCL_SO_PATH" + "environment variable SGLANG_DIFFUSION_NCCL_SO_PATH" " to point to the correct nccl library path.", so_file, platform.platform(), diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py index f7917c520..6db3785ff 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/backends/sliding_tile_attn.py @@ -119,9 +119,9 @@ class SlidingTileAttentionImpl(AttentionImpl): raise ValueError("st attn not supported") # TODO(will-refactor): for now this is the mask strategy, but maybe we should # have a more general config for STA? - config_file = envs.SGL_DIFFUSION_ATTENTION_CONFIG + config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG if config_file is None: - raise ValueError("SGL_DIFFUSION_ATTENTION_CONFIG is not set") + raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set") # TODO(kevin): get mask strategy for different STA modes with open(config_file) as f: diff --git a/python/sglang/multimodal_gen/runtime/layers/attention/selector.py b/python/sglang/multimodal_gen/runtime/layers/attention/selector.py index 7d9d18463..19c407663 100644 --- a/python/sglang/multimodal_gen/runtime/layers/attention/selector.py +++ b/python/sglang/multimodal_gen/runtime/layers/attention/selector.py @@ -110,7 +110,7 @@ def _cached_get_attn_backend( # Check whether a particular choice of backend was # previously forced. # - # THIS SELECTION OVERRIDES THE SGL_DIFFUSION_ATTENTION_BACKEND + # THIS SELECTION OVERRIDES THE SGLANG_DIFFUSION_ATTENTION_BACKEND # ENVIRONMENT VARIABLE. from sglang.multimodal_gen.runtime.platforms import current_platform diff --git a/python/sglang/multimodal_gen/runtime/managers/forward_context.py b/python/sglang/multimodal_gen/runtime/managers/forward_context.py index d9d107e69..0fcf747f4 100644 --- a/python/sglang/multimodal_gen/runtime/managers/forward_context.py +++ b/python/sglang/multimodal_gen/runtime/managers/forward_context.py @@ -19,11 +19,11 @@ if TYPE_CHECKING: logger = init_logger(__name__) # TODO(will): check if this is needed -# track_batchsize: bool = envs.SGL_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0 +# track_batchsize: bool = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL >= 0 track_batchsize: bool = False last_logging_time: float = 0 forward_start_time: float = 0 -# batchsize_logging_interval: float = envs.SGL_DIFFUSION_LOG_BATCHSIZE_INTERVAL +# batchsize_logging_interval: float = envs.SGLANG_DIFFUSION_LOG_BATCHSIZE_INTERVAL batchsize_logging_interval: float = 1000 batchsize_forward_time: defaultdict = defaultdict(list) diff --git a/python/sglang/multimodal_gen/runtime/models/registry.py b/python/sglang/multimodal_gen/runtime/models/registry.py index a3cb0934e..ea81be77b 100644 --- a/python/sglang/multimodal_gen/runtime/models/registry.py +++ b/python/sglang/multimodal_gen/runtime/models/registry.py @@ -102,7 +102,7 @@ def _discover_and_register_models() -> dict[str, tuple[str, str, str]]: return discovered_models -_SGL_DIFFUSION_MODELS = _discover_and_register_models() +_SGLANG_DIFFUSION_MODELS = _discover_and_register_models() _SUBPROCESS_COMMAND = [ sys.executable, @@ -361,6 +361,6 @@ ModelRegistry = _ModelRegistry( component_name, mod_relname, cls_name, - ) in _SGL_DIFFUSION_MODELS.items() + ) in _SGLANG_DIFFUSION_MODELS.items() } ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines/stages/base.py index 1e6671270..8601eb1e1 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/stages/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/stages/base.py @@ -187,7 +187,7 @@ class PipelineStage(ABC): # Execute the actual stage logic logging_info = getattr(batch, "logging_info", None) - if envs.SGL_DIFFUSION_STAGE_LOGGING: + if envs.SGLANG_DIFFUSION_STAGE_LOGGING: logger.info("[%s] Starting execution", stage_name) start_time = time.perf_counter() diff --git a/python/sglang/multimodal_gen/runtime/pipelines/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines/stages/denoising.py index 6b7c6f5f3..1c99da75e 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/stages/denoising.py @@ -1284,9 +1284,9 @@ class DenoisingStage(PipelineStage): elif STA_mode == STA_Mode.STA_INFERENCE: import sglang.multimodal_gen.envs as envs - config_file = envs.SGL_DIFFUSION_ATTENTION_CONFIG + config_file = envs.SGLANG_DIFFUSION_ATTENTION_CONFIG if config_file is None: - raise ValueError("SGL_DIFFUSION_ATTENTION_CONFIG is not set") + raise ValueError("SGLANG_DIFFUSION_ATTENTION_CONFIG is not set") STA_param = configure_sta( mode=STA_Mode.STA_INFERENCE, layer_num=layer_num, diff --git a/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/python/sglang/multimodal_gen/runtime/platforms/rocm.py index cadf98000..1e5c370dd 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/rocm.py +++ b/python/sglang/multimodal_gen/runtime/platforms/rocm.py @@ -69,8 +69,8 @@ class RocmPlatform(Platform): dtype: torch.dtype, ) -> str: logger.info( - "Trying SGL_DIFFUSION_ATTENTION_BACKEND=%s", - envs.SGL_DIFFUSION_ATTENTION_BACKEND, + "Trying SGLANG_DIFFUSION_ATTENTION_BACKEND=%s", + envs.SGLANG_DIFFUSION_ATTENTION_BACKEND, ) if selected_backend == AttentionBackendEnum.TORCH_SDPA: diff --git a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py index 669bb4b46..7b752d5bb 100644 --- a/python/sglang/multimodal_gen/runtime/utils/logging_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/logging_utils.py @@ -17,10 +17,10 @@ from typing import Any, cast import sglang.multimodal_gen.envs as envs -SGL_DIFFUSION_CONFIGURE_LOGGING = envs.SGL_DIFFUSION_CONFIGURE_LOGGING -SGL_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGL_DIFFUSION_LOGGING_CONFIG_PATH -SGL_DIFFUSION_LOGGING_LEVEL = envs.SGL_DIFFUSION_LOGGING_LEVEL -SGL_DIFFUSION_LOGGING_PREFIX = envs.SGL_DIFFUSION_LOGGING_PREFIX +SGLANG_DIFFUSION_CONFIGURE_LOGGING = envs.SGLANG_DIFFUSION_CONFIGURE_LOGGING +SGLANG_DIFFUSION_LOGGING_CONFIG_PATH = envs.SGLANG_DIFFUSION_LOGGING_CONFIG_PATH +SGLANG_DIFFUSION_LOGGING_LEVEL = envs.SGLANG_DIFFUSION_LOGGING_LEVEL +SGLANG_DIFFUSION_LOGGING_PREFIX = envs.SGLANG_DIFFUSION_LOGGING_PREFIX RED = "\033[91m" GREEN = "\033[92m" @@ -28,7 +28,7 @@ YELLOW = "\033[93m" RESET = "\033[0;0m" _FORMAT = ( - f"{SGL_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s " + f"{SGLANG_DIFFUSION_LOGGING_PREFIX}%(levelname)s %(asctime)s " "[%(filename)s: %(lineno)d] %(message)s" ) @@ -47,7 +47,7 @@ DEFAULT_LOGGING_CONFIG = { "sgl_diffusion": { "class": "logging.StreamHandler", "formatter": "sgl_diffusion", - "level": SGL_DIFFUSION_LOGGING_LEVEL, + "level": SGLANG_DIFFUSION_LOGGING_LEVEL, "stream": "ext://sys.stdout", }, }, @@ -340,7 +340,7 @@ def enable_trace_function_call(log_file_path: str, root_dir: str | None = None): will have the trace enabled. Other threads will not be affected. """ logger.warning( - "SGL_DIFFUSION_TRACE_FUNCTION is enabled. It will record every" + "SGLANG_DIFFUSION_TRACE_FUNCTION is enabled. It will record every" " function executed by Python. This will slow down the code. It " "is suggested to be used for debugging hang or crashes only." ) diff --git a/python/sglang/multimodal_gen/runtime/utils/performance_logger.py b/python/sglang/multimodal_gen/runtime/utils/performance_logger.py index 95d93ca2b..c5c1d99bf 100644 --- a/python/sglang/multimodal_gen/runtime/utils/performance_logger.py +++ b/python/sglang/multimodal_gen/runtime/utils/performance_logger.py @@ -6,10 +6,13 @@ import os import subprocess import time from datetime import datetime +from pathlib import Path from typing import Any from dateutil.tz import UTC +import sglang + def get_diffusion_perf_log_dir() -> str: """ @@ -26,7 +29,10 @@ def get_diffusion_perf_log_dir() -> str: return os.path.abspath(log_dir) if log_dir is None: # Not set, use default - return os.path.join(os.path.expanduser("~/.cache/sglang"), "logs") + sglang_path = Path(sglang.__file__).resolve() + # .gitignore + target_path = (sglang_path.parent / "../../.cache/logs").resolve() + return str(target_path) # Is set, but is an empty string return "" diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines.json b/python/sglang/multimodal_gen/test/server/perf_baselines.json index cfb0c5ccb..4b06fcb08 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines.json @@ -1,155 +1,351 @@ { - "metadata": { - "model": "Diffusion Server", - "hardware": "CI H100 80GB pool", - "description": "Reference numbers captured from the CI diffusion server baseline run" - }, - "tolerances": { - "e2e": 0.25, - "stage": 0.3, - "denoise_step": 0.2, - "denoise_agg": 0.1 - }, - "sampling": { - "step_fractions": [ - 0.0, - 0.2, - 0.4, - 0.6, - 0.8, - 1.0 - ], - "warmup_requests": { - "text": 1, - "image_edit": 0 - } - }, - "scenarios": { - "text_to_image": { - "notes": "Single-image generation using the default prompt", - "expected_e2e_ms": 74500.0, - "expected_avg_denoise_ms": 422.42, - "expected_median_denoise_ms": 410.62, - "stages_ms": { - "InputValidationStage": 0.1, - "TextEncodingStage": 834.2, - "ConditioningStage": 0.1, - "TimestepPreparationStage": 10.6, - "LatentPreparationStage": 9.0, - "DenoisingStage": 21202.6, - "DecodingStage": 476.12 - }, - "denoise_step_ms": { - "0": 1077.77, "1": 345.13, "2": 413.8, "3": 405.49, "4": 408.14, "5": 409.06, - "6": 408.85, "7": 410.53, "8": 407.51, "9": 409.44, "10": 408.65, "11": 410.14, - "12": 411.74, "13": 409.59, "14": 409.17, "15": 410.78, "16": 410.66, "17": 410.58, - "18": 411.27, "19": 410.51, "20": 409.03, "21": 410.16, "22": 409.42, "23": 411.03, - "24": 410.18, "25": 409.72, "26": 410.26, "27": 410.21, "28": 410.71, "29": 410.76, - "30": 411.06, "31": 410.1, "32": 410.55, "33": 410.77, "34": 410.74, "35": 411.75, - "36": 410.78, "37": 411.56, "38": 410.85, "39": 411.08, "40": 411.12, "41": 411.1, - "42": 411.09, "43": 410.87, "44": 411.37, "45": 411.68, "46": 411.0, "47": 410.09, - "48": 412.72, "49": 410.42 - } + "metadata": { + "model": "Diffusion Server", + "hardware": "CI H100 80GB pool", + "description": "Reference numbers captured from the CI diffusion server baseline run" }, - "image_edit": { - "notes": "single uploaded reference image, Qwen/Qwen-Image-Edit", - "expected_e2e_ms": 138500.0, - "expected_avg_denoise_ms": 720.0, - "expected_median_denoise_ms": 718.0, - "stages_ms": { - "InputValidationStage": 23, - "ImageEncodingStage": 1350.0, - "ImageVAEEncodingStage": 340.0, - "ConditioningStage": 0.13, - "TimestepPreparationStage": 13.78, - "LatentPreparationStage": 10.0, - "DenoisingStage": 36000.0, - "DecodingStage": 850.0 - }, - "denoise_step_ms": { - "0": 720.0, "1": 720.0, "2": 720.0, "3": 720.0, "4": 720.0, "5": 720.0, - "6": 720.0, "7": 720.0, "8": 720.0, "9": 720.0, "10": 720.0, "11": 720.0, - "12": 720.0, "13": 720.0, "14": 720.0, "15": 720.0, "16": 720.0, "17": 720.0, - "18": 720.0, "19": 720.0, "20": 720.0, "21": 720.0, "22": 720.0, "23": 720.0, - "24": 720.0, "25": 720.0, "26": 720.0, "27": 720.0, "28": 720.0, "29": 720.0, - "30": 720.0, "31": 720.0, "32": 720.0, "33": 720.0, "34": 720.0, "35": 720.0, - "36": 720.0, "37": 720.0, "38": 720.0, "39": 720.0, "40": 720.0, "41": 720.0, - "42": 720.0, "43": 720.0, "44": 720.0, "45": 720.0, "46": 720.0, "47": 720.0, - "48": 720.0, "49": 720.0 - } + "tolerances": { + "e2e": 0.05, + "stage": 0.05, + "denoise_step": 0.15, + "denoise_agg": 0.08 }, - "text_to_video": { - "notes": "Single-video generation using the default prompt", - "expected_e2e_ms": 95616.59, - "expected_avg_denoise_ms": 1798.77, - "expected_median_denoise_ms": 1786.78, - - "stages_ms": { - "InputValidationStage": 1.03, - "TextEncodingStage": 3450.0, - "ConditioningStage": 1.0, - "TimestepPreparationStage": 6.0, - "LatentPreparationStage": 15.0, - "DenoisingStage": 90100.0, - "DecodingStage": 3650.0 - }, - - "denoise_step_ms": { - "0": 3500.0, "10": 1800.0, "20": 1800.0, "29": 1800.0, "39": 1800.0, "49": 1800.0 - }, - "frames_per_second": 0.51, - "total_frames": 49, - "avg_frame_time_ms": 1951.36 + "improvement_reporting": { + "threshold": 0.2 }, - "image_to_video": { - "notes": "Wan-AI/Wan2.2-I2V-A14B", - "expected_e2e_ms": 282500.0, - "expected_avg_denoise_ms": 7000.0, - "expected_median_denoise_ms": 7000.19, - "stages_ms": { - "InputValidationStage": 20.0, - "TextEncodingStage": 2100.0, - "ConditioningStage": 2.0, - "TimestepPreparationStage": 2.0, - "LatentPreparationStage": 10.0, - "ImageVAEEncodingStage": 1800.0, - "DenoisingStage": 278000.0, - "DecodingStage": 2700.0 - }, - "denoise_step_ms": { - "0": 24000.0, - "8": 7000.0, - "16": 7000.0, - "23": 7000.0, - "31": 7000.0, - "39": 7000.0 + "sampling": { + "step_fractions": [ + 0.0, + 0.2, + 0.4, + 0.6, + 0.8, + 1.0 + ], + "warmup_requests": { + "text": 1, + "image_edit": 0 } }, - "text_image_to_video": { - "notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B", - "expected_e2e_ms": 178300.0, - "expected_avg_denoise_ms": 3250.0, - "expected_median_denoise_ms": 3260.0, - "stages_ms": { - "InputValidationStage": 80.0, - "TextEncodingStage": 3000.0, - "ConditioningStage": 1.0, - "TimestepPreparationStage": 6.0, - "LatentPreparationStage": 30.0, - "DenoisingStage": 162900.0, - "DecodingStage": 13500.0 - }, - "denoise_step_ms": { - "0": 3700.0, - "10": 3300.0, - "20": 3300.0, - "29": 3300.0, - "39": 3300.0, - "49": 3300.0 - }, - "frames_per_second": null, - "total_frames": null, - "avg_frame_time_ms": null + "scenarios": { + "qwen_image_t2i": { + "notes": "Single-image generation using the default prompt", + "expected_e2e_ms": 74500.0, + "expected_avg_denoise_ms": 422.42, + "expected_median_denoise_ms": 410.62, + "stages_ms": { + "InputValidationStage": 0.1, + "TextEncodingStage": 834.2, + "ConditioningStage": 0.1, + "TimestepPreparationStage": 10.6, + "LatentPreparationStage": 11.8, + "DenoisingStage": 21202.6, + "DecodingStage": 476.12 + }, + "denoise_step_ms": { + "0": 1077.77, + "1": 345.13, + "2": 413.8, + "3": 405.49, + "4": 408.14, + "5": 409.06, + "6": 408.85, + "7": 410.53, + "8": 407.51, + "9": 409.44, + "10": 408.65, + "11": 410.14, + "12": 411.74, + "13": 409.59, + "14": 409.17, + "15": 410.78, + "16": 410.66, + "17": 410.58, + "18": 411.27, + "19": 410.51, + "20": 409.03, + "21": 410.16, + "22": 409.42, + "23": 411.03, + "24": 410.18, + "25": 409.72, + "26": 410.26, + "27": 410.21, + "28": 410.71, + "29": 410.76, + "30": 411.06, + "31": 410.1, + "32": 410.55, + "33": 410.77, + "34": 410.74, + "35": 411.75, + "36": 410.78, + "37": 411.56, + "38": 410.85, + "39": 411.08, + "40": 411.12, + "41": 411.1, + "42": 411.09, + "43": 410.87, + "44": 411.37, + "45": 411.68, + "46": 411.0, + "47": 410.09, + "48": 412.72, + "49": 410.42 + } + }, + "flux_image_t2i": { + "stages_ms": { + "InputValidationStage": 0.08, + "TextEncodingStage": 87.36, + "ConditioningStage": 0.024, + "TimestepPreparationStage": 2.53, + "LatentPreparationStage": 6.21, + "DenoisingStage": 8161.97, + "DecodingStage": 378.21 + }, + "denoise_step_ms": { + "0": 56.16, + "10": 163.54, + "20": 165.94, + "29": 165.26, + "39": 165.16, + "49": 165.36 + }, + "expected_e2e_ms": 8764.73, + "expected_avg_denoise_ms": 160.79, + "expected_median_denoise_ms": 165.14 + }, + "qwen_image_edit_ti2i": { + "notes": "single uploaded reference image, Qwen/Qwen-Image-Edit", + "expected_e2e_ms": 138500.0, + "expected_avg_denoise_ms": 720.0, + "expected_median_denoise_ms": 718.0, + "stages_ms": { + "InputValidationStage": 23, + "ImageEncodingStage": 1350.0, + "ImageVAEEncodingStage": 340.0, + "ConditioningStage": 0.13, + "TimestepPreparationStage": 13.78, + "LatentPreparationStage": 15.0, + "DenoisingStage": 36000.0, + "DecodingStage": 850.0 + }, + "denoise_step_ms": { + "0": 720.0, + "1": 720.0, + "2": 720.0, + "3": 720.0, + "4": 720.0, + "5": 720.0, + "6": 720.0, + "7": 720.0, + "8": 720.0, + "9": 720.0, + "10": 720.0, + "11": 720.0, + "12": 720.0, + "13": 720.0, + "14": 720.0, + "15": 720.0, + "16": 720.0, + "17": 720.0, + "18": 720.0, + "19": 720.0, + "20": 720.0, + "21": 720.0, + "22": 720.0, + "23": 720.0, + "24": 720.0, + "25": 720.0, + "26": 720.0, + "27": 720.0, + "28": 720.0, + "29": 720.0, + "30": 720.0, + "31": 720.0, + "32": 720.0, + "33": 720.0, + "34": 720.0, + "35": 720.0, + "36": 720.0, + "37": 720.0, + "38": 720.0, + "39": 720.0, + "40": 720.0, + "41": 720.0, + "42": 720.0, + "43": 720.0, + "44": 720.0, + "45": 720.0, + "46": 720.0, + "47": 720.0, + "48": 720.0, + "49": 720.0 + } + }, + "fastwan2_1_t2v": { + "notes": "Single-video generation using the default prompt", + "expected_e2e_ms": 95616.59, + "expected_avg_denoise_ms": 1798.77, + "expected_median_denoise_ms": 1786.78, + "stages_ms": { + "InputValidationStage": 1.03, + "TextEncodingStage": 3450.0, + "ConditioningStage": 1.0, + "TimestepPreparationStage": 6.0, + "LatentPreparationStage": 15.0, + "DenoisingStage": 90100.0, + "DecodingStage": 3650.0 + }, + "denoise_step_ms": { + "0": 3500.0, + "10": 1800.0, + "20": 1800.0, + "29": 1800.0, + "39": 1800.0, + "49": 1800.0 + }, + "frames_per_second": 0.51, + "total_frames": 49, + "avg_frame_time_ms": 1951.36 + }, + "wan2_2_i2v_a14b": { + "notes": "Wan-AI/Wan2.2-I2V-A14B", + "expected_e2e_ms": 282500.0, + "expected_avg_denoise_ms": 7000.0, + "expected_median_denoise_ms": 7000.19, + "stages_ms": { + "InputValidationStage": 26.26, + "TextEncodingStage": 2749.6, + "ConditioningStage": 2.0, + "TimestepPreparationStage": 2.0, + "LatentPreparationStage": 10.0, + "ImageVAEEncodingStage": 2031.0, + "DenoisingStage": 278000.0, + "DecodingStage": 2849.6 + }, + "denoise_step_ms": { + "0": 24000.0, + "8": 7000.0, + "16": 7000.0, + "23": 7000.0, + "31": 7000.0, + "39": 7000.0 + } + }, + + "wan2_1_i2v_14b_480P": { + "stages_ms": { + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 5432.37, + "1": 4861.66, + "2": 4890.23, + "3": 4919.9, + "4": 4929.49, + "5": 4927.35, + "6": 4927.6, + "7": 4932.02, + "8": 4897.83, + "9": 4897.16, + "10": 4899.59, + "11": 4901.51, + "12": 4896.38, + "13": 4914.3, + "14": 4901.91, + "15": 4898.58, + "16": 4903.18, + "17": 4899.54, + "18": 4910.5, + "19": 4898.18, + "20": 4901.56, + "21": 4903.12, + "22": 4896.82, + "23": 4901.01, + "24": 4897.42, + "25": 4911.94, + "26": 4901.29, + "27": 4894.79, + "28": 4899.57, + "29": 4900.63, + "30": 4904.78, + "31": 4899.79, + "32": 4898.01, + "33": 4910.81, + "34": 4901.16, + "35": 4898.48, + "36": 4894.97, + "37": 4905.9, + "38": 4901.52, + "39": 4898.56, + "40": 4900.19, + "41": 4896.5, + "42": 4899.89, + "43": 4898.45, + "44": 4897.27, + "45": 4895.47, + "46": 4889.26, + "47": 4887.44, + "48": 4894.07, + "49": 4888.59 + }, + "expected_e2e_ms": 254552.63, + "expected_avg_denoise_ms": 4912.17, + "expected_median_denoise_ms": 4899.69 + }, + "wan2_2_i2v_14b_720P": { + "stages_ms": { + "InputValidationStage": 47.04, + "TextEncodingStage": 2321.27, + "ImageEncodingStage": 3244.34, + "ConditioningStage": 0.0234, + "TimestepPreparationStage": 2.88, + "LatentPreparationStage": 5.24, + "ImageVAEEncodingStage": 1887.64, + "DenoisingStage": 245826.78, + "DecodingStage": 2882.45, + "per_frame_generation": null + }, + "denoise_step_ms": { + "0": 5429.38, + "10": 4901.39, + "20": 4912.89, + "29": 4900.75, + "39": 4906.23, + "49": 4892.55 + }, + "expected_e2e_ms": 254850.94, + "expected_avg_denoise_ms": 4914.73, + "expected_median_denoise_ms": 4903.27 + }, + "wan2_2_ti2v_5b": { + "notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B", + "expected_e2e_ms": 178300.0, + "expected_avg_denoise_ms": 3250.0, + "expected_median_denoise_ms": 3260.0, + "stages_ms": { + "InputValidationStage": 150.0, + "TextEncodingStage": 3000.0, + "ConditioningStage": 1.0, + "TimestepPreparationStage": 6.0, + "LatentPreparationStage": 30.0, + "DenoisingStage": 162900.0, + "DecodingStage": 14767.0 + }, + "denoise_step_ms": { + "0": 3700.0, + "10": 3300.0, + "20": 3300.0, + "29": 3300.0, + "39": 3300.0, + "49": 3300.0 + }, + "frames_per_second": null, + "total_frames": null, + "avg_frame_time_ms": null + } } - } } diff --git a/python/sglang/multimodal_gen/test/server/test_server_performance.py b/python/sglang/multimodal_gen/test/server/test_server_performance.py index d9ac0a7ae..45c62433f 100644 --- a/python/sglang/multimodal_gen/test/server/test_server_performance.py +++ b/python/sglang/multimodal_gen/test/server/test_server_performance.py @@ -1,6 +1,5 @@ """ Config-driven diffusion performance test with pytest parametrization. -Adding a new model/scenario = adding one DiffusionCase entry in diffusion_config.py. """ from __future__ import annotations @@ -15,20 +14,20 @@ from openai import OpenAI from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.test.server.conftest import _GLOBAL_PERF_RESULTS -from sglang.multimodal_gen.test.server.diffusion_config import ( - BASELINE_CONFIG, - DIFFUSION_CASES, - DiffusionCase, -) -from sglang.multimodal_gen.test.server.diffusion_server import ( +from sglang.multimodal_gen.test.server.test_server_utils import ( VALIDATOR_REGISTRY, PerformanceValidator, ServerContext, ServerManager, - VideoPerformanceValidator, WarmupRunner, download_image_from_url, ) +from sglang.multimodal_gen.test.server.testcase_configs import ( + BASELINE_CONFIG, + DIFFUSION_CASES, + DiffusionTestCase, + PerformanceSummary, +) from sglang.multimodal_gen.test.test_utils import ( get_dynamic_server_port, read_perf_records, @@ -42,13 +41,13 @@ logger = init_logger(__name__) @pytest.fixture(params=DIFFUSION_CASES, ids=lambda c: c.id) -def case(request) -> DiffusionCase: - """Provide a DiffusionCase for each test.""" +def case(request) -> DiffusionTestCase: + """Provide a DiffusionTestCase for each test.""" return request.param @pytest.fixture -def diffusion_server(case: DiffusionCase) -> ServerContext: +def diffusion_server(case: DiffusionTestCase) -> ServerContext: """Start a diffusion server for a single case and tear it down afterwards.""" default_port = get_dynamic_server_port() port = int(os.environ.get("SGLANG_TEST_SERVER_PORT", default_port)) @@ -79,17 +78,17 @@ def diffusion_server(case: DiffusionCase) -> ServerContext: ) warmup.run_text_warmups(case.warmup_text) - if case.warmup_edit > 0 and case.image_edit_prompt and case.image_edit_path: + if case.warmup_edit > 0 and case.edit_prompt and case.image_path: # Handle URL or local path - image_path = case.image_edit_path + image_path = case.image_path if case.is_image_url(): - image_path = download_image_from_url(str(case.image_edit_path)) + image_path = download_image_from_url(str(case.image_path)) else: - image_path = Path(case.image_edit_path) + image_path = Path(case.image_path) warmup.run_edit_warmups( count=case.warmup_edit, - edit_prompt=case.image_edit_prompt, + edit_prompt=case.edit_prompt, image_path=image_path, ) except Exception as exc: @@ -132,12 +131,13 @@ class TestDiffusionPerformance: def _run_and_collect( self, ctx: ServerContext, - case: DiffusionCase, + case: DiffusionTestCase, generate_fn: Callable[[], None], ) -> tuple[dict, dict]: """Run generation and collect performance records.""" log_path = ctx.perf_log_path prev_len = len(read_perf_records(log_path)) + log_wait_timeout = 1200 generate_fn() @@ -145,22 +145,25 @@ class TestDiffusionPerformance: "total_inference_time", prev_len, log_path, + timeout=log_wait_timeout, ) - scenario = BASELINE_CONFIG.scenarios[case.scenario_name] - stage_metrics, _ = wait_for_stage_metrics( - perf_record.get("request_id", ""), - prev_len, - len(scenario.stages_ms), - log_path, - ) + stage_metrics = {} + if perf_record: + + stage_metrics, _ = wait_for_stage_metrics( + perf_record.get("request_id", ""), + prev_len, + log_path, + timeout=log_wait_timeout, + ) return perf_record, stage_metrics def _generate_for_case( self, ctx: ServerContext, - case: DiffusionCase, + case: DiffusionTestCase, ) -> Callable[[], None]: """Return appropriate generation function for the case.""" client = self._client(ctx) @@ -192,21 +195,35 @@ class TestDiffusionPerformance: job = client.videos.create(**create_kwargs) # type: ignore[attr-defined] video_id = job.id - deadline = time.time() + 600 + job_completed = False + is_baseline_generation_mode = ( + os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" + ) + timeout = 3600.0 if is_baseline_generation_mode else 600.0 + deadline = time.time() + timeout while True: page = client.videos.list() # type: ignore[attr-defined] item = next((v for v in page.data if v.id == video_id), None) if item and getattr(item, "status", None) == "completed": + job_completed = True break if time.time() > deadline: - pytest.fail( - f"{case.id}: video job {video_id} did not complete in time" - ) + break time.sleep(5) + if not job_completed: + if is_baseline_generation_mode: + logger.warning( + f"{case.id}: video job {video_id} timed out during baseline generation. " + "Attempting to collect performance data anyway." + ) + return b"" + + pytest.fail(f"{case.id}: video job {video_id} did not complete in time") + # download video resp = client.videos.download_content(video_id=video_id) # type: ignore[attr-defined] content = resp.read() @@ -235,14 +252,14 @@ class TestDiffusionPerformance: def generate_image_edit(): """TI2I: Text + Image ? Image edit.""" - if not case.image_edit_prompt or not case.image_edit_path: + if not case.edit_prompt or not case.image_path: pytest.skip(f"{case.id}: no edit config") # Handle URL or local path if case.is_image_url(): - image_path = download_image_from_url(str(case.image_edit_path)) + image_path = download_image_from_url(str(case.image_path)) else: - image_path = Path(case.image_edit_path) + image_path = Path(case.image_path) if not image_path.exists(): pytest.skip(f"{case.id}: file missing: {image_path}") @@ -250,7 +267,7 @@ class TestDiffusionPerformance: result = client.images.edit( model=case.model_path, image=fh, - prompt=case.image_edit_prompt, + prompt=case.edit_prompt, n=1, size=case.output_size, response_format="b64_json", @@ -275,21 +292,21 @@ class TestDiffusionPerformance: def generate_image_to_video(): """I2V: Image ? Video (optional prompt).""" - if not case.image_edit_path: + if not case.image_path: pytest.skip(f"{case.id}: no input image configured") # Handle URL or local path if case.is_image_url(): - image_path = download_image_from_url(str(case.image_edit_path)) + image_path = download_image_from_url(str(case.image_path)) else: - image_path = Path(case.image_edit_path) + image_path = Path(case.image_path) if not image_path.exists(): pytest.skip(f"{case.id}: file missing: {image_path}") with image_path.open("rb") as fh: _create_and_download_video( model=case.model_path, - prompt=case.image_edit_prompt, + prompt=case.edit_prompt, size=case.output_size, seconds=video_seconds, input_reference=fh, @@ -297,49 +314,55 @@ class TestDiffusionPerformance: def generate_text_image_to_video(): """TI2V: Text + Image ? Video.""" - if not case.image_edit_prompt or not case.image_edit_path: + if not case.edit_prompt or not case.image_path: pytest.skip(f"{case.id}: no edit config") # Handle URL or local path if case.is_image_url(): - image_path = download_image_from_url(str(case.image_edit_path)) + image_path = download_image_from_url(str(case.image_path)) else: - image_path = Path(case.image_edit_path) + image_path = Path(case.image_path) if not image_path.exists(): pytest.skip(f"{case.id}: file missing: {image_path}") with image_path.open("rb") as fh: _create_and_download_video( model=case.model_path, - prompt=case.image_edit_prompt, + prompt=case.edit_prompt, size=case.output_size, seconds=video_seconds, input_reference=fh, ) if case.modality == "video": - if case.image_edit_path and case.image_edit_prompt: + if case.image_path and case.edit_prompt: return generate_text_image_to_video - elif case.image_edit_path: + elif case.image_path: return generate_image_to_video else: return generate_video # Image modality - if case.image_edit_prompt and case.image_edit_path: + if case.edit_prompt and case.image_path: return generate_image_edit return generate_image def _validate_and_record( self, - case: DiffusionCase, + case: DiffusionTestCase, perf_record: dict, stage_metrics: dict, ) -> None: """Validate metrics and record results.""" - scenario = BASELINE_CONFIG.scenarios[case.scenario_name] + is_baseline_generation_mode = os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" + scenario = BASELINE_CONFIG.scenarios.get(case.id) + if scenario is None: + if is_baseline_generation_mode: + scenario = {} # Dummy scenario + else: + pytest.fail(f"Testcase '{case.id}' not in perf_baselines.json") validator_name = case.custom_validator or "default" validator_class = VALIDATOR_REGISTRY.get(validator_name, PerformanceValidator) @@ -349,10 +372,17 @@ class TestDiffusionPerformance: step_fractions=BASELINE_CONFIG.step_fractions, ) - if isinstance(validator, VideoPerformanceValidator): - summary = validator.validate(perf_record, stage_metrics, case.num_frames) - else: - summary = validator.validate(perf_record, stage_metrics) + summary = validator.collect_metrics(perf_record, stage_metrics) + + if is_baseline_generation_mode: + self._dump_baseline_scenario(case, summary) + return + + try: + validator.validate(perf_record, stage_metrics, case.num_frames) + except AssertionError: + self._dump_baseline_scenario(case, summary) + raise if case.modality == "video" and summary.frames_per_second: logger.info( @@ -428,9 +458,46 @@ class TestDiffusionPerformance: summary.avg_frame_time_ms, ) + def _dump_baseline_scenario( + self, case: DiffusionTestCase, summary: "PerformanceSummary" + ) -> None: + """Dump performance metrics as a JSON scenario for baselines.""" + import json + + denoise_steps_formatted = { + str(k): round(v, 2) for k, v in summary.sampled_steps.items() + } + stages_formatted = {k: round(v, 2) for k, v in summary.stage_metrics.items()} + + baseline = { + "stages_ms": stages_formatted, + "denoise_step_ms": denoise_steps_formatted, + "expected_e2e_ms": round(summary.e2e_ms, 2), + "expected_avg_denoise_ms": round(summary.avg_denoise_ms, 2), + "expected_median_denoise_ms": round(summary.median_denoise_ms, 2), + } + + # Video-specific metrics + if case.modality == "video": + if "per_frame_generation" not in baseline["stages_ms"]: + baseline["stages_ms"]["per_frame_generation"] = ( + round(summary.avg_frame_time_ms, 2) + if summary.avg_frame_time_ms + else None + ) + + output = f""" +To add this baseline, copy the following JSON snippet into +the "scenarios" section of perf_baselines.json: + +"{case.id}": {json.dumps(baseline, indent=4)} + +""" + print(output) + def test_diffusion_perf( self, - case: DiffusionCase, + case: DiffusionTestCase, diffusion_server: ServerContext, ): """Single parametrized test that runs for all cases. diff --git a/python/sglang/multimodal_gen/test/server/diffusion_server.py b/python/sglang/multimodal_gen/test/server/test_server_utils.py similarity index 69% rename from python/sglang/multimodal_gen/test/server/diffusion_server.py rename to python/sglang/multimodal_gen/test/server/test_server_utils.py index 6a5df3eb9..eb32bc98a 100644 --- a/python/sglang/multimodal_gen/test/server/diffusion_server.py +++ b/python/sglang/multimodal_gen/test/server/test_server_utils.py @@ -7,7 +7,9 @@ from __future__ import annotations import os import statistics import subprocess +import sys import tempfile +import threading import time from dataclasses import dataclass, field from pathlib import Path @@ -18,7 +20,7 @@ from openai import OpenAI from sglang.multimodal_gen.runtime.utils.common import kill_process_tree from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger -from sglang.multimodal_gen.test.server.diffusion_config import ( +from sglang.multimodal_gen.test.server.testcase_configs import ( PerformanceSummary, ScenarioConfig, ToleranceConfig, @@ -74,6 +76,7 @@ class ServerContext: perf_log_path: Path log_dir: Path _stdout_fh: Any = field(repr=False) + _log_thread: threading.Thread | None = field(default=None, repr=False) def cleanup(self) -> None: """Clean up server resources.""" @@ -127,19 +130,42 @@ class ServerManager: command.extend(self.extra_args.strip().split()) env = os.environ.copy() - env["SGL_DIFFUSION_STAGE_LOGGING"] = "1" + env["SGLANG_DIFFUSION_STAGE_LOGGING"] = "1" env["SGLANG_PERF_LOG_DIR"] = log_dir.as_posix() stdout_fh = stdout_path.open("w", encoding="utf-8", buffering=1) process = subprocess.Popen( command, - stdout=stdout_fh, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, env=env, ) + log_thread = None + if process.stdout: + + def _log_pipe(pipe: Any, file: Any) -> None: + """Read from pipe and write to file and stdout.""" + try: + with pipe: + for line in iter(pipe.readline, ""): + sys.stdout.write(line) + file.write(line) + file.flush() + except Exception as e: + logger.error("Log pipe thread error: %s", e) + finally: + file.close() + logger.debug("Log pipe thread finished.") + + log_thread = threading.Thread( + target=_log_pipe, args=(process.stdout, stdout_fh) + ) + log_thread.daemon = True + log_thread.start() + logger.info( "[server-test] Starting server pid=%s, model=%s, log=%s", process.pid, @@ -157,6 +183,7 @@ class ServerManager: perf_log_path=perf_log_path, log_dir=log_dir, _stdout_fh=stdout_fh, + _log_thread=log_thread, ) def _wait_for_ready(self, process: subprocess.Popen, stdout_path: Path) -> None: @@ -264,6 +291,8 @@ class WarmupRunner: class PerformanceValidator: """Validates performance metrics against expectations.""" + is_video_gen: bool = False + def __init__( self, scenario: ScenarioConfig, @@ -273,104 +302,134 @@ class PerformanceValidator: self.scenario = scenario self.tolerances = tolerances self.step_fractions = step_fractions + self.is_baseline_generation_mode = ( + os.environ.get("SGLANG_GEN_BASELINE", "0") == "1" + ) + + def _assert_le(self, name: str, actual: float, expected: float, tolerance: float): + """Assert that actual is less than or equal to expected within a tolerance.""" + upper_bound = expected * (1 + tolerance) + assert actual <= upper_bound, ( + f"Validation failed for '{name}'.\n" + f" - Actual: {actual:.4f}ms\n" + f" - Expected: {expected:.4f}ms\n" + f" - Limit: {upper_bound:.4f}ms (tolerance: {tolerance:.1%})" + ) def validate( + self, perf_record: dict, stage_metrics: dict, *args, **kwargs + ) -> PerformanceSummary: + """Validate all performance metrics and return summary.""" + summary = self.collect_metrics(perf_record, stage_metrics) + if self.is_baseline_generation_mode: + return summary + + self._validate_e2e(summary) + self._validate_denoise_agg(summary) + self._validate_denoise_steps(summary) + self._validate_stages(summary) + + return summary + + def collect_metrics( self, perf_record: dict, stage_metrics: dict, ) -> PerformanceSummary: - """Validate all performance metrics and return summary.""" - self._validate_e2e(perf_record) - avg_denoise, median_denoise = self._validate_denoise_agg(perf_record) - sampled_steps = self._validate_denoise_steps(perf_record) - self._validate_stages(stage_metrics) - - return PerformanceSummary( - e2e_ms=float(perf_record["total_duration_ms"]), - avg_denoise_ms=avg_denoise, - median_denoise_ms=median_denoise, - stage_metrics=stage_metrics, - sampled_steps=sampled_steps, - ) - - def _validate_e2e(self, perf_record: dict) -> None: - """Validate end-to-end performance.""" + """Collect all performance metrics into a summary without validation.""" e2e_ms = float(perf_record.get("total_duration_ms", 0.0)) - assert e2e_ms > 0, "E2E duration missing" - - upper = self.scenario.expected_e2e_ms * (1 + self.tolerances.e2e) - assert e2e_ms <= upper, f"E2E {e2e_ms:.2f}ms exceeds {upper:.2f}ms" - - def _validate_denoise_agg(self, perf_record: dict) -> tuple[float, float]: - """Validate aggregate denoising metrics.""" steps = [ s for s in perf_record.get("steps", []) or [] if s.get("name") == "denoising_step_guided" and "duration_ms" in s ] - assert steps, "Denoising step timings missing" - durations = [float(s["duration_ms"]) for s in steps] - avg = sum(durations) / len(durations) - median = statistics.median(durations) - - avg_upper = self.scenario.expected_avg_denoise_ms * ( - 1 + self.tolerances.denoise_agg - ) - med_upper = self.scenario.expected_median_denoise_ms * ( - 1 + self.tolerances.denoise_agg - ) - - assert avg <= avg_upper, f"Avg denoise {avg:.2f}ms exceeds {avg_upper:.2f}ms" - assert ( - median <= med_upper - ), f"Median denoise {median:.2f}ms exceeds {med_upper:.2f}ms" - - return avg, median - - def _validate_denoise_steps(self, perf_record: dict) -> dict[int, float]: - """Validate individual denoising steps.""" - steps = [ - s - for s in perf_record.get("steps", []) or [] - if s.get("name") == "denoising_step_guided" and "duration_ms" in s - ] + avg_denoise = 0.0 + median_denoise = 0.0 + if steps: + durations = [float(s["duration_ms"]) for s in steps] + avg_denoise = sum(durations) / len(durations) + median_denoise = statistics.median(durations) per_step = { int(s["index"]): float(s["duration_ms"]) for s in steps if s.get("index") is not None } - sample_indices = sample_step_indices(per_step, self.step_fractions) - sampled = {idx: per_step[idx] for idx in sample_indices} + sampled_steps = {idx: per_step[idx] for idx in sample_indices} - for idx in sample_indices: + return PerformanceSummary( + e2e_ms=e2e_ms, + avg_denoise_ms=avg_denoise, + median_denoise_ms=median_denoise, + stage_metrics=stage_metrics, + sampled_steps=sampled_steps, + ) + + def _validate_e2e(self, summary: PerformanceSummary) -> None: + """Validate end-to-end performance.""" + assert summary.e2e_ms > 0, "E2E duration missing" + self._assert_le( + "E2E Latency", + summary.e2e_ms, + self.scenario.expected_e2e_ms, + self.tolerances.e2e, + ) + + def _validate_denoise_agg(self, summary: PerformanceSummary) -> None: + """Validate aggregate denoising metrics.""" + assert summary.avg_denoise_ms > 0, "Denoising step timings missing" + + self._assert_le( + "Average Denoise Step", + summary.avg_denoise_ms, + self.scenario.expected_avg_denoise_ms, + self.tolerances.denoise_agg, + ) + self._assert_le( + "Median Denoise Step", + summary.median_denoise_ms, + self.scenario.expected_median_denoise_ms, + self.tolerances.denoise_agg, + ) + + def _validate_denoise_steps(self, summary: PerformanceSummary) -> None: + """Validate individual denoising steps.""" + for idx, actual in summary.sampled_steps.items(): expected = self.scenario.denoise_step_ms.get(idx) if expected is None: continue + self._assert_le( + f"Denoise Step {idx}", + actual, + expected, + self.tolerances.denoise_step, + ) - actual = per_step[idx] - upper = expected * (1 + self.tolerances.denoise_step) - assert actual <= upper, f"Step {idx}: {actual:.2f}ms > {upper:.2f}ms" - - return sampled - - def _validate_stages(self, stage_metrics: dict) -> None: + def _validate_stages(self, summary: PerformanceSummary) -> None: """Validate stage-level metrics.""" - assert stage_metrics, "Stage metrics missing" + assert summary.stage_metrics, "Stage metrics missing" for stage, expected in self.scenario.stages_ms.items(): - actual = stage_metrics.get(stage) + if stage == "per_frame_generation" and self.is_video_gen: + continue + actual = summary.stage_metrics.get(stage) assert actual is not None, f"Stage {stage} timing missing" - upper = expected * (1 + self.tolerances.stage) - assert actual <= upper, f"Stage {stage}: {actual:.2f}ms > {upper:.2f}ms" + self._assert_le( + f"Stage '{stage}'", + actual, + expected, + self.tolerances.stage, + ) class VideoPerformanceValidator(PerformanceValidator): """Extended validator for video diffusion with frame-level metrics.""" + is_video_gen = True + def validate( self, perf_record: dict, @@ -385,7 +444,8 @@ class VideoPerformanceValidator(PerformanceValidator): summary.avg_frame_time_ms = summary.e2e_ms / num_frames summary.frames_per_second = 1000.0 / summary.avg_frame_time_ms - self._validate_frame_rate(summary) + if not self.is_baseline_generation_mode: + self._validate_frame_rate(summary) return summary @@ -393,24 +453,12 @@ class VideoPerformanceValidator(PerformanceValidator): """Validate frame generation performance.""" expected_frame_time = self.scenario.stages_ms.get("per_frame_generation") if expected_frame_time and summary.avg_frame_time_ms: - upper = expected_frame_time * (1 + self.tolerances.stage) - assert ( - summary.avg_frame_time_ms <= upper - ), f"Avg frame time {summary.avg_frame_time_ms:.2f}ms exceeds {upper:.2f}ms" - - def _validate_stages(self, stage_metrics: dict) -> None: - """Validate video-specific stages.""" - assert stage_metrics, "Stage metrics missing" - - for stage, expected in self.scenario.stages_ms.items(): - if stage == "per_frame_generation": - continue - - actual = stage_metrics.get(stage) - assert actual is not None, f"Stage {stage} timing missing" - - upper = expected * (1 + self.tolerances.stage) - assert actual <= upper, f"Stage {stage}: {actual:.2f}ms > {upper:.2f}ms" + self._assert_le( + "Average Frame Time", + summary.avg_frame_time_ms, + expected_frame_time, + self.tolerances.stage, + ) # Registry of validators by name diff --git a/python/sglang/multimodal_gen/test/server/diffusion_config.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py similarity index 66% rename from python/sglang/multimodal_gen/test/server/diffusion_config.py rename to python/sglang/multimodal_gen/test/server/testcase_configs.py index 11131a297..94bfcb8b3 100644 --- a/python/sglang/multimodal_gen/test/server/diffusion_config.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -1,5 +1,19 @@ """ Configuration and data structures for diffusion performance tests. + +Usage: + +pytest python/sglang/multimodal_gen/test/server/test_server_performance.py +# for a single testcase, look for the name of the testcases in DIFFUSION_CASES +pytest python/sglang/multimodal_gen/test/server/test_server_performance.py -k qwen_image_t2i + + +To add a new testcase: +1. add your testcase with case-id: `my_new_test_case_id` to DIFFUSION_CASES +2. run `SGLANG_GEN_BASELINE=1 pytest -s python/sglang/multimodal_gen/test/server/test_server_performance.py -k my_new_test_case_id` +3. insert or override the corresponding scenario in `scenarios` section of perf_baselines.json with the output baseline of step-2 + + """ from __future__ import annotations @@ -78,34 +92,36 @@ class BaselineConfig: @dataclass(frozen=True) -class DiffusionCase: +class DiffusionTestCase: """Configuration for a single model/scenario test case.""" - id: str # pytest test id + id: str # pytest test id and scenario name model_path: str # HF repo or local path - scenario_name: str # key into BASELINE_CONFIG.scenarios modality: str = "image" # "image" or "video" or "3d" - prompt: str | None = None # text prompt for generation output_size: str = "1024x1024" # output image dimensions (or video resolution) + + # inputs and conditioning + prompt: str | None = None # text prompt for generation + edit_prompt: str | None = None # prompt for editing + image_path: Path | str | None = None # input image/video for editing (Path or URL) + + # duration + seconds: int = 4 # for video: duration in seconds num_frames: int | None = None # for video: number of frames fps: int | None = None # for video: frames per second + warmup_text: int = 1 # number of text-to-image/video warmups warmup_edit: int = 0 # number of image/video-edit warmups - image_edit_prompt: str | None = None # prompt for editing - image_edit_path: Path | str | None = ( - None # input image/video for editing (Path or URL) - ) startup_grace_seconds: float = 0.0 # wait time after server starts custom_validator: str | None = None # optional custom validator name - seconds: int = 4 # for video: duration in seconds def is_image_url(self) -> bool: """Check if image_edit_path is a URL.""" - if self.image_edit_path is None: + if self.image_path is None: return False - return isinstance(self.image_edit_path, str) and ( - self.image_edit_path.startswith("http://") - or self.image_edit_path.startswith("https://") + return isinstance(self.image_path, str) and ( + self.image_path.startswith("http://") + or self.image_path.startswith("https://") ) @@ -128,12 +144,11 @@ IMAGE_INPUT_FILE = Path(__file__).resolve().parents[1] / "test_files" / "girl.jp # All test cases with clean default values # To test different models, simply add more DiffusionCase entries -DIFFUSION_CASES: list[DiffusionCase] = [ +DIFFUSION_CASES: list[DiffusionTestCase] = [ # === Text to Image (T2I) === - DiffusionCase( + DiffusionTestCase( id="qwen_image_t2i", model_path="Qwen/Qwen-Image", - scenario_name="text_to_image", modality="image", prompt="A futuristic cityscape at sunset with flying cars", output_size="1024x1024", @@ -141,10 +156,9 @@ DIFFUSION_CASES: list[DiffusionCase] = [ warmup_edit=0, startup_grace_seconds=30.0, ), - DiffusionCase( + DiffusionTestCase( id="flux_image_t2i", model_path="black-forest-labs/FLUX.1-dev", - scenario_name="text_to_image", modality="image", prompt="A futuristic cityscape at sunset with flying cars", output_size="1024x1024", @@ -153,24 +167,23 @@ DIFFUSION_CASES: list[DiffusionCase] = [ startup_grace_seconds=30.0, ), # === Text and Image to Image (TI2I) === - DiffusionCase( + DiffusionTestCase( id="qwen_image_edit_ti2i", model_path="Qwen/Qwen-Image-Edit", - scenario_name="image_edit", modality="image", prompt=None, # not used for editing output_size="1024x1536", warmup_text=0, warmup_edit=1, - image_edit_prompt="Convert 2D style to 3D style", - image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + edit_prompt="Convert 2D style to 3D style", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", startup_grace_seconds=30.0, ), # === Text to Video (T2V) === - DiffusionCase( + # TODO: FastWan2.1, FastWan2.2 + DiffusionTestCase( id="fastwan2_1_t2v", model_path="Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - scenario_name="text_to_video", modality="video", prompt="A curious raccoon", output_size="848x480", @@ -181,39 +194,64 @@ DIFFUSION_CASES: list[DiffusionCase] = [ custom_validator="video", ), # === Image to Video (I2V) === - DiffusionCase( - id="wan2_2_i2v", + DiffusionTestCase( + id="wan2_2_i2v_a14b", model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers", - scenario_name="image_to_video", modality="video", prompt="generate", # passing in something since failing if no prompt is passed warmup_text=0, # warmups only for image gen models warmup_edit=0, output_size="832x1104", - image_edit_prompt="generate", - image_edit_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", + edit_prompt="generate", + image_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true", startup_grace_seconds=30.0, custom_validator="video", seconds=1, ), # === Text and Image to Video (TI2V) === - DiffusionCase( - id="wan2_2_ti2v_5b", - model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers", - scenario_name="text_image_to_video", + DiffusionTestCase( + id="wan2_1_i2v_14b_480P", + model_path="Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + output_size="832x1104", modality="video", prompt="Animate this image", + edit_prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + warmup_text=0, # warmups only for image gen models + warmup_edit=0, + startup_grace_seconds=30.0, + custom_validator="video", + seconds=1, + ), + DiffusionTestCase( + id="wan2_2_i2v_14b_720P", + model_path="Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + modality="video", + prompt="Animate this image", + edit_prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", output_size="832x1104", warmup_text=0, # warmups only for image gen models warmup_edit=0, - image_edit_prompt="Add dynamic motion to the scene", - image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", startup_grace_seconds=30.0, custom_validator="video", - seconds=4, + seconds=1, + ), + DiffusionTestCase( + id="wan2_2_ti2v_5b", + model_path="Wan-AI/Wan2.2-TI2V-5B-Diffusers", + modality="video", + output_size="832x1104", + prompt="Animate this image", + edit_prompt="Add dynamic motion to the scene", + image_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", + warmup_text=0, # warmups only for image gen models + warmup_edit=0, + startup_grace_seconds=30.0, + custom_validator="video", + seconds=1, ), ] - # Load global configuration BASELINE_CONFIG = BaselineConfig.load(Path(__file__).with_name("perf_baselines.json")) diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index a1e315f07..6d81203ad 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -172,6 +172,11 @@ def wait_for_perf_record( if rec.get("tag") == tag: return rec, len(records) time.sleep(0.5) + + if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1": + records = read_perf_records(log_path) + return {}, len(records) + raise AssertionError( f"Timeout waiting for perf log entry '{tag}' (start_len={prev_len})" ) @@ -180,15 +185,21 @@ def wait_for_perf_record( def wait_for_stage_metrics( request_id: str, prev_len: int, - expected_count: int, log_path: Path, - timeout: float = 120.0, + timeout: float = 300.0, ) -> tuple[dict[str, float], int]: deadline = time.time() + timeout metrics: dict[str, float] = {} while time.time() < deadline: records = read_perf_records(log_path) for rec in records[prev_len:]: + # Check if the request is completed + if ( + rec.get("tag") == "total_inference_time" + and rec.get("request_id") == request_id + ): + return metrics, len(records) + if ( rec.get("tag") == "pipeline_stage_metric" and rec.get("request_id") == request_id @@ -197,13 +208,12 @@ def wait_for_stage_metrics( duration = rec.get("duration_ms") if stage is not None and duration is not None: metrics[str(stage)] = float(duration) - if len(metrics) >= expected_count: - return metrics, len(records) time.sleep(0.5) - raise AssertionError( - f"Timeout waiting for stage metrics for request {request_id} " - f"(collected={len(metrics)} expected={expected_count})" - ) + + if os.environ.get("SGLANG_GEN_BASELINE", "0") == "1": + records = read_perf_records(log_path) + return {}, len(records) + raise AssertionError(f"Timeout waiting for stage metrics for request {request_id} ") def sample_step_indices( diff --git a/python/sglang/multimodal_gen/utils.py b/python/sglang/multimodal_gen/utils.py index c435928dd..ae0386e5b 100644 --- a/python/sglang/multimodal_gen/utils.py +++ b/python/sglang/multimodal_gen/utils.py @@ -48,8 +48,8 @@ PRECISION_TO_TYPE = { "bf16": torch.bfloat16, } -STR_BACKEND_ENV_VAR: str = "SGL_DIFFUSION_ATTENTION_BACKEND" -STR_ATTN_CONFIG_ENV_VAR: str = "SGL_DIFFUSION_ATTENTION_CONFIG" +STR_BACKEND_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_BACKEND" +STR_ATTN_CONFIG_ENV_VAR: str = "SGLANG_DIFFUSION_ATTENTION_CONFIG" def find_nccl_library() -> str: @@ -59,12 +59,12 @@ def find_nccl_library() -> str: After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be found by `ctypes` automatically. """ - so_file = envs.SGL_DIFFUSION_NCCL_SO_PATH + so_file = envs.SGLANG_DIFFUSION_NCCL_SO_PATH # manually load the nccl library if so_file: logger.info( - "Found nccl from environment variable SGL_DIFFUSION_NCCL_SO_PATH=%s", + "Found nccl from environment variable SGLANG_DIFFUSION_NCCL_SO_PATH=%s", so_file, ) else: diff --git a/scripts/ci/validate_and_download_models.py b/scripts/ci/validate_and_download_models.py index 07ad507e6..f310d4cb6 100755 --- a/scripts/ci/validate_and_download_models.py +++ b/scripts/ci/validate_and_download_models.py @@ -66,11 +66,16 @@ RUNNER_LABEL_MODEL_MAP: Dict[str, List[str]] = { "Qwen/Qwen3-Embedding-8B", "Qwen/QwQ-32B-AWQ", "Qwen/Qwen3-30B-A3B", + "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", + # diffusion "Qwen/Qwen-Image", "Qwen/Qwen-Image-Edit", - "Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", - "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "black-forest-labs/FLUX.1-dev", + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + "Wan-AI/Wan2.2-I2V-A14B-Diffusers", ], "2-gpu-runner": [ "mistralai/Mixtral-8x7B-Instruct-v0.1",