diff --git a/vllm/v1/attention/backends/mla/triton_mla.py b/vllm/v1/attention/backends/mla/triton_mla.py index db11cd2845..3b91c22516 100644 --- a/vllm/v1/attention/backends/mla/triton_mla.py +++ b/vllm/v1/attention/backends/mla/triton_mla.py @@ -25,13 +25,58 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd +from vllm.v1.worker.workspace import ( + current_workspace_manager, + is_workspace_manager_initialized, +) logger = init_logger(__name__) +# num_kv_splits selection (shared by forward_mqa and the workspace reservation +# so the two cannot drift). Both are hardware dependent. +_MIN_WORK_PER_SPLIT = 512 +_SPLIT_OCCUPANCY_MULTIPLIER = 2 + + +def _compute_num_kv_splits(max_seq_len: int, sm_count: int) -> int: + # Power of 2 to avoid excessive kernel instantiations, capped by an SM-based + # maximum (occupancy multiplier allows multiple blocks per SM + # for latency hiding). + ideal_splits = triton.next_power_of_2(max(1, max_seq_len // _MIN_WORK_PER_SPLIT)) + max_splits = sm_count * _SPLIT_OCCUPANCY_MULTIPLIER + return min(ideal_splits, max_splits) + class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + def __init__(self, kv_cache_spec, layer_names, vllm_config, device): + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + self._reserve_attn_logits_workspace() + + def _reserve_attn_logits_workspace(self) -> None: + """Pre-size the shared workspace for the decode split-KV attn logits. + + Reserving at the worst case (max_model_len -> max num_kv_splits, + max_num_seqs decode tokens) before warmup/cudagraph capture means the + per-call ``get_simultaneous`` in ``forward_mqa`` never has to grow the + buffer at runtime (which would raise once the workspace is locked). + """ + if not is_workspace_manager_initialized(): + return + # Decode reorder threshold is 1, so decode tokens <= max_num_seqs. + B = self.vllm_config.scheduler_config.max_num_seqs + # DCP all-gathers the query heads before forward_mqa. + q_num_heads = self.num_heads * self.dcp_world_size + max_splits = _compute_num_kv_splits( + self.model_config.max_model_len, + current_platform.num_compute_units(), + ) + lse_dim = self.mla_dims.kv_lora_rank + 1 + current_workspace_manager().get_simultaneous( + ((B, q_num_heads, max_splits, lse_dim), torch.float32), + ) + class TritonMLABackend(MLACommonBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] @@ -166,35 +211,25 @@ class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]): if envs.VLLM_BATCH_INVARIANT: num_kv_splits = 1 else: - # Minimum work per split - # hardware dependent - min_work_per_split = 512 + num_kv_splits = _compute_num_kv_splits( + attn_metadata.max_seq_len, self._sm_count + ) - ideal_splits = max(1, attn_metadata.max_seq_len // min_work_per_split) - - # use power of 2 to avoid excessive kernel instantiations - ideal_splits = triton.next_power_of_2(ideal_splits) - - # Calculate SM-based maximum splits with occupancy multiplier - # 2-4x allows multiple blocks per SM for latency hiding - # hardware dependent - occupancy_multiplier = 2 - max_splits = self._sm_count * occupancy_multiplier - num_kv_splits = min(ideal_splits, max_splits) - - # TODO(lucas) Allocate ahead of time - attn_logits = torch.empty( - ( - B, - q_num_heads, - num_kv_splits, - # NOTE: the +1 stores the LogSumExp (LSE) that the stage2 - # kernel uses to merge partial attention outputs across splits. - self.kv_lora_rank + 1, - ), - dtype=torch.float32, - device=q.device, - ) + # NOTE: the +1 stores the LogSumExp (LSE) that the stage2 kernel uses to + # merge partial attention outputs across splits. The scratch is served + # from the shared workspace (reserved at max in the metadata builder), so + # there is no per-call allocation on the decode hot path. Fall back to a + # direct allocation when the workspace manager is not initialized (e.g. + # unit tests without a GPUModelRunner). + logits_shape = (B, q_num_heads, num_kv_splits, self.kv_lora_rank + 1) + if is_workspace_manager_initialized(): + (attn_logits,) = current_workspace_manager().get_simultaneous( + (logits_shape, torch.float32), + ) + else: + attn_logits = torch.empty( + logits_shape, dtype=torch.float32, device=q.device + ) # Add a head dim of 1 kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.unsqueeze(2)