[Feature] Enable TRITON_ATTN for Batch Invariance (#33688)
Signed-off-by: frankwang28 <frank.wbb@hotmail.com>
This commit is contained in:
@@ -10,10 +10,12 @@
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.batch_invariant import vllm_is_batch_invariant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
is_batch_invariant = vllm_is_batch_invariant()
|
||||
float8_info = torch.finfo(current_platform.fp8_dtype())
|
||||
|
||||
|
||||
@@ -972,7 +974,8 @@ def unified_attention(
|
||||
# Launch the 2D kernel if
|
||||
# 1. No intermediate tiled softmax buffers for the 3D kernel have been allocated, or
|
||||
# 2. The batch includes at least one prefill request, or
|
||||
# 3. The number of sequences exceeds the configured threshold
|
||||
# 3. The number of sequences exceeds the configured threshold, or
|
||||
# 4. Batch invariance is enabled
|
||||
if (
|
||||
seq_threshold_3D is None
|
||||
or num_par_softmax_segments is None
|
||||
@@ -981,6 +984,7 @@ def unified_attention(
|
||||
or softmax_segm_expsum is None
|
||||
or max_seqlen_q > 1
|
||||
or num_seqs > seq_threshold_3D
|
||||
or is_batch_invariant
|
||||
):
|
||||
kernel_unified_attention_2d[
|
||||
(
|
||||
|
||||
Reference in New Issue
Block a user