[Attention][SM120] Occupancy-adaptive Triton split-K segment count (#58482)

Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: vadiklyutiy <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
Venky
2026-10-01 02:20:33 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 vadiklyutiy
parent 76d3929916
commit edaf3e7b61
3 changed files with 114 additions and 5 deletions
+40 -4
View File
@@ -18,7 +18,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache
from vllm.v1.attention.backend import (
AttentionBackend,
@@ -152,10 +152,29 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
)
self.num_par_softmax_segments = NUM_PAR_SOFTMAX_SEGMENTS
# On SM120, batches whose 16-segment grid under-fills the SMs use 64.
self.max_seqs_64_segments = 0
if current_platform.is_cuda() and current_platform.is_device_capability(
(12, 0)
):
self.max_seqs_64_segments = (current_platform.num_compute_units() - 1) // (
NUM_PAR_SOFTMAX_SEGMENTS * self.num_heads_kv
)
max_num_tokens_3d = self.seq_threshold_3D
if self.max_seqs_64_segments > 0:
self.num_par_softmax_segments = 64
# build() reuses this scratch at 16 segments with proportionally more rows.
max_num_tokens_3d = max(
min(self.max_seqs_64_segments, max_num_tokens_3d),
cdiv(
max_num_tokens_3d,
self.num_par_softmax_segments // NUM_PAR_SOFTMAX_SEGMENTS,
),
)
headdim_padded = next_power_of_2(self.headdim)
self.softmax_segm_output = torch.empty(
(
self.seq_threshold_3D,
max_num_tokens_3d,
self.num_heads_q,
self.num_par_softmax_segments,
headdim_padded,
@@ -164,12 +183,12 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
device=device,
)
self.softmax_segm_max = torch.empty(
(self.seq_threshold_3D, self.num_heads_q, self.num_par_softmax_segments),
(max_num_tokens_3d, self.num_heads_q, self.num_par_softmax_segments),
dtype=torch.float32,
device=device,
)
self.softmax_segm_expsum = torch.empty(
(self.seq_threshold_3D, self.num_heads_q, self.num_par_softmax_segments),
(max_num_tokens_3d, self.num_heads_q, self.num_par_softmax_segments),
dtype=torch.float32,
device=device,
)
@@ -246,6 +265,23 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
softmax_segm_max=self.softmax_segm_max,
softmax_segm_expsum=self.softmax_segm_expsum,
)
if self.max_seqs_64_segments > 0 and (
self.max_seqs_64_segments < seq_lens.shape[0]
or self.softmax_segm_max.shape[0] < num_actual_tokens
):
# The 16-segment grid fills the SMs, or the 64-segment rows cannot
# hold every query token: same scratch, 16-segment layout.
segments = NUM_PAR_SOFTMAX_SEGMENTS
attn_metadata.num_par_softmax_segments = segments
attn_metadata.softmax_segm_output = self.softmax_segm_output.view(
-1, self.num_heads_q, segments, self.softmax_segm_output.shape[-1]
)
attn_metadata.softmax_segm_max = self.softmax_segm_max.view(
-1, self.num_heads_q, segments
)
attn_metadata.softmax_segm_expsum = self.softmax_segm_expsum.view(
-1, self.num_heads_q, segments
)
mm_ranges = common_attn_metadata.mm_req_doc_ranges
if mm_ranges is not None:
@@ -56,7 +56,7 @@ class TritonAttentionDiffKVMetadataBuilder(TritonAttentionMetadataBuilder):
head_size_v_padded = next_power_of_2(head_size_v)
self.softmax_segm_output = torch.empty(
(
self.seq_threshold_3D,
self.softmax_segm_max.shape[0],
self.num_heads_q,
self.num_par_softmax_segments,
head_size_v_padded,