kvcache-ai/sglang
1
0
Fork 0
mirror of https://github.com/kvcache-ai/sglang.git synced 2026-07-11 01:36:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add feature flag for mm inputs processing optimization (#13278)

Browse Source 
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-11-15 14:43:44 +08:00
committed by GitHub
parent 7ee3e36412
commit f0021c0dc8
2 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/managers/scheduler.py
View File

@@ -1239,6 +1239,12 @@ class Scheduler(
return image_inputs
def _get_multimodal_inputs(self, mm_inputs_dict: dict):
if self.server_args.enable_broadcast_mm_inputs_process:
return self._process_and_broadcast_mm_inputs(mm_inputs_dict)
else:
return MultimodalInputs.from_dict(mm_inputs_dict)
def handle_generate_request(
self,
recv_req: TokenizedGenerateReqInput,
@@ -1320,7 +1326,7 @@ class Scheduler(
# Handle multimodal inputs
if recv_req.mm_inputs is not None:
image_inputs = self._process_and_broadcast_mm_inputs(recv_req.mm_inputs)
image_inputs = self._get_multimodal_inputs(recv_req.mm_inputs)
# The following steps are already fast, execute locally on each rank.
# Expand a single image token into multiple dummy tokens for receiving image embeddings
@@ -1555,7 +1561,7 @@ class Scheduler(
# Handle multimodal inputs
if recv_req.image_inputs is not None:
image_inputs = self._process_and_broadcast_mm_inputs(recv_req.image_inputs)
image_inputs = self._get_multimodal_inputs(recv_req.image_inputs)
# Expand a single image token into multiple dummy tokens for receiving image embeddings
req.origin_input_ids = self.pad_input_ids_func(
req.origin_input_ids, image_inputs

7
python/sglang/srt/server_args.py
View File

@@ -565,6 +565,7 @@ class ServerArgs:
# For Multi-Modal
mm_max_concurrent_calls: int = 32
mm_per_request_timeout: float = 10.0
enable_broadcast_mm_inputs_process: bool = False
# For checkpoint decryption
decrypted_config_file: Optional[str] = None
@@ -3656,6 +3657,12 @@ class ServerArgs:
default=ServerArgs.mm_per_request_timeout,
help="The timeout for each multi-modal request in seconds.",
)
parser.add_argument(
"--enable-broadcast-mm-inputs-process",
action="store_true",
default=ServerArgs.enable_broadcast_mm_inputs_process,
help="Enable broadcast mm-inputs process in scheduler.",
)
# For checkpoint decryption
parser.add_argument(