diff --git a/vllm/envs.py b/vllm/envs.py index 4191cd6a97..24ec92c3d7 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -245,6 +245,7 @@ if TYPE_CHECKING: VLLM_DEBUG_WORKSPACE: bool = False VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256 + VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 4096 VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary" VLLM_USE_V2_MODEL_RUNNER: bool = False VLLM_LOG_MODEL_INSPECTION: bool = False @@ -1678,6 +1679,17 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD": lambda: int( int(os.getenv("VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD", 256)) ), + # Token-count cutoff for multi-stream overlap of the attention input + # GEMM with auxiliary GEMMs (e.g. fused_wqa_wkv overlapped with indexer + # weights / kv-score projections in DeepSeek-V4). At or below this many + # tokens the FP8 main GEMM has idle SMs to share with the bf16 aux GEMMs + # and overlap is a 5-45% win; above it the FP8 GEMM saturates the device + # and the cross-stream sync becomes pure overhead. Set to 0 to disable + # the multi-stream path entirely. Empirical crossover on B300 (148 SMs) + # is ~4096; B200 (132 SMs) is expected ~3072. + "VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int( + os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "4096") + ), # Format for saving torch.compile cache artifacts # - "binary": saves as binary file # Safe for multiple vllm serve processes accessing the same torch compile cache. diff --git a/vllm/model_executor/layers/deepseek_v4_attention.py b/vllm/model_executor/layers/deepseek_v4_attention.py index 74b494dc48..847c3eee55 100644 --- a/vllm/model_executor/layers/deepseek_v4_attention.py +++ b/vllm/model_executor/layers/deepseek_v4_attention.py @@ -13,6 +13,7 @@ import torch.nn as nn import torch.nn.functional as F from transformers import DeepseekV2Config, DeepseekV3Config +import vllm.envs as envs from vllm.model_executor.layers.linear import ( ReplicatedLinear, ) @@ -385,6 +386,8 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer): self.ln_events[0], self.ln_events[1:4], self.aux_stream_list[:3], + enable=hidden_states.shape[0] + <= envs.VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD, ) return qr_kv, kv_score, indexer_kv_score, indexer_weights diff --git a/vllm/utils/multi_stream_utils.py b/vllm/utils/multi_stream_utils.py index c00f08f933..2203221c5a 100644 --- a/vllm/utils/multi_stream_utils.py +++ b/vllm/utils/multi_stream_utils.py @@ -64,6 +64,7 @@ def execute_in_parallel( start_event: torch.cuda.Event, done_events: list[torch.cuda.Event], aux_streams: list[torch.cuda.Stream] | None = None, + enable: bool = False, ) -> tuple[Any, list[Any]]: """Run default_fn on the current stream and aux_fns concurrently on aux_streams. @@ -74,8 +75,9 @@ def execute_in_parallel( start_event fans out from the current stream to every launched aux stream; done_events[i] is recorded after aux_fns[i] so the current stream joins - before returning. When aux_streams is None, all aux_fns run sequentially - on the current stream. + before returning. Falls back to sequential execution on the current stream + when aux_streams is None or enable is False; in that case default_fn runs + first, then aux_fns in order. Args: default_fn: Callable for the default (current) stream. @@ -86,13 +88,17 @@ def execute_in_parallel( corresponding aux_fn. Length must match aux_fns. aux_streams: Per-aux CUDA streams. Length must match aux_fns. Multi-stream is disabled when None. + enable: Opt-in switch for the multi-stream path. Defaults to False, + so callers that pass aux_streams must also pass enable=True + (typically gated by an env var) to actually overlap. When False, + execution falls back to sequential on the current stream. Returns: Tuple of (default_result, aux_results) where aux_results[i] is the result of aux_fns[i] (or None when skipped). """ aux_results: list[Any] - if aux_streams is None: + if aux_streams is None or not enable: default_result = default_fn() aux_results = [fn() if fn is not None else None for fn in aux_fns] return default_result, aux_results