[Kernel] Triton MLA logits workspace (#46819)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-06-29 07:54:29 -07:00
committed by GitHub
parent 49e28e8e91
commit 6149187a4c
+63 -28
View File
@@ -25,13 +25,58 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
)
logger = init_logger(__name__)
# num_kv_splits selection (shared by forward_mqa and the workspace reservation
# so the two cannot drift). Both are hardware dependent.
_MIN_WORK_PER_SPLIT = 512
_SPLIT_OCCUPANCY_MULTIPLIER = 2
def _compute_num_kv_splits(max_seq_len: int, sm_count: int) -> int:
# Power of 2 to avoid excessive kernel instantiations, capped by an SM-based
# maximum (occupancy multiplier allows multiple blocks per SM
# for latency hiding).
ideal_splits = triton.next_power_of_2(max(1, max_seq_len // _MIN_WORK_PER_SPLIT))
max_splits = sm_count * _SPLIT_OCCUPANCY_MULTIPLIER
return min(ideal_splits, max_splits)
class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self._reserve_attn_logits_workspace()
def _reserve_attn_logits_workspace(self) -> None:
"""Pre-size the shared workspace for the decode split-KV attn logits.
Reserving at the worst case (max_model_len -> max num_kv_splits,
max_num_seqs decode tokens) before warmup/cudagraph capture means the
per-call ``get_simultaneous`` in ``forward_mqa`` never has to grow the
buffer at runtime (which would raise once the workspace is locked).
"""
if not is_workspace_manager_initialized():
return
# Decode reorder threshold is 1, so decode tokens <= max_num_seqs.
B = self.vllm_config.scheduler_config.max_num_seqs
# DCP all-gathers the query heads before forward_mqa.
q_num_heads = self.num_heads * self.dcp_world_size
max_splits = _compute_num_kv_splits(
self.model_config.max_model_len,
current_platform.num_compute_units(),
)
lse_dim = self.mla_dims.kv_lora_rank + 1
current_workspace_manager().get_simultaneous(
((B, q_num_heads, max_splits, lse_dim), torch.float32),
)
class TritonMLABackend(MLACommonBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
@@ -166,35 +211,25 @@ class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]):
if envs.VLLM_BATCH_INVARIANT:
num_kv_splits = 1
else:
# Minimum work per split
# hardware dependent
min_work_per_split = 512
num_kv_splits = _compute_num_kv_splits(
attn_metadata.max_seq_len, self._sm_count
)
ideal_splits = max(1, attn_metadata.max_seq_len // min_work_per_split)
# use power of 2 to avoid excessive kernel instantiations
ideal_splits = triton.next_power_of_2(ideal_splits)
# Calculate SM-based maximum splits with occupancy multiplier
# 2-4x allows multiple blocks per SM for latency hiding
# hardware dependent
occupancy_multiplier = 2
max_splits = self._sm_count * occupancy_multiplier
num_kv_splits = min(ideal_splits, max_splits)
# TODO(lucas) Allocate ahead of time
attn_logits = torch.empty(
(
B,
q_num_heads,
num_kv_splits,
# NOTE: the +1 stores the LogSumExp (LSE) that the stage2
# kernel uses to merge partial attention outputs across splits.
self.kv_lora_rank + 1,
),
dtype=torch.float32,
device=q.device,
)
# NOTE: the +1 stores the LogSumExp (LSE) that the stage2 kernel uses to
# merge partial attention outputs across splits. The scratch is served
# from the shared workspace (reserved at max in the metadata builder), so
# there is no per-call allocation on the decode hot path. Fall back to a
# direct allocation when the workspace manager is not initialized (e.g.
# unit tests without a GPUModelRunner).
logits_shape = (B, q_num_heads, num_kv_splits, self.kv_lora_rank + 1)
if is_workspace_manager_initialized():
(attn_logits,) = current_workspace_manager().get_simultaneous(
(logits_shape, torch.float32),
)
else:
attn_logits = torch.empty(
logits_shape, dtype=torch.float32, device=q.device
)
# Add a head dim of 1
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.unsqueeze(2)