Force TRTLLM attention for gpt-oss on SM100 (#22678)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2025-08-12 21:22:16 -07:00
committed by GitHub
parent b1361c7273
commit c6b928798e
4 changed files with 20 additions and 9 deletions
+8
View File
@@ -154,6 +154,7 @@ def use_trtllm_attention(
num_qo_heads: Optional[int],
num_kv_heads: Optional[int],
attn_head_size: Optional[int],
has_sinks: bool = False,
) -> bool:
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
if not (current_platform.is_device_capability(100)
@@ -165,6 +166,13 @@ def use_trtllm_attention(
or num_qo_heads % num_kv_heads != 0):
return False
# If sinks are being used, we must use TRTLLM attention as it's
# the only backend that supports them
if has_sinks:
logger.info_once(
"Using TRTLLM attention (required for attention sinks).")
return True
env_value = envs.VLLM_USE_TRTLLM_ATTENTION
if env_value is not None:
logger.info_once("VLLM_USE_TRTLLM_ATTENTION is set to %s", env_value)