[Feature] Enable TRITON_ATTN for Batch Invariance (#33688)

Signed-off-by: frankwang28 <frank.wbb@hotmail.com>
This commit is contained in:
Frank Wang
2026-02-04 13:27:34 +08:00
committed by GitHub
parent 5e1e0a0fbd
commit 45f8fd6f97
4 changed files with 13 additions and 4 deletions
@@ -10,10 +10,12 @@
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.batch_invariant import vllm_is_batch_invariant
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
logger = init_logger(__name__)
is_batch_invariant = vllm_is_batch_invariant()
float8_info = torch.finfo(current_platform.fp8_dtype())
@@ -972,7 +974,8 @@ def unified_attention(
# Launch the 2D kernel if
# 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or
# 2. The batch includes at least one prefill request, or
# 3. The number of sequences exceeds the configured threshold
# 3. The number of sequences exceeds the configured threshold, or
# 4. Batch invariance is enabled
if (
seq_threshold_3D is None
or num_par_softmax_segments is None
@@ -981,6 +984,7 @@ def unified_attention(
or softmax_segm_expsum is None
or max_seqlen_q > 1
or num_seqs > seq_threshold_3D
or is_batch_invariant
):
kernel_unified_attention_2d[
(