Xqa decode kernels (#43232)

Signed-off-by: Dan Blanaru <48605845+DanBlanaru@users.noreply.github.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Dan Blanaru
2026-07-02 12:32:05 -07:00
committed by GitHub
co-authored by Matthew Bonanni
parent d715b3aa1e
commit d29125c085
7 changed files with 538 additions and 143 deletions
+37 -18
View File
@@ -371,19 +371,27 @@ def has_nvidia_artifactory() -> bool:
@functools.cache
def supports_trtllm_attention() -> bool:
"""
TRTLLM attention is supported if the platform is SM100,
NVIDIA artifactory is accessible, and batch-invariant mode is not enabled.
def supports_trtllm_attention(is_prefill: bool = False) -> bool:
"""Return whether TRTLLM attention is available on the current platform
for the given attention phase.
SM90 (Hopper) supports the XQA decode kernel but not TRTLLM prefill.
SM100+ supports TRTLLM for both phases. All others are unsupported.
"""
# Batch-invariant mode disables TRTLLM attention
if envs.VLLM_BATCH_INVARIANT:
return False
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
return (
current_platform.is_device_capability_family(100) and has_nvidia_artifactory()
)
# Requires NVIDIA artifactory to be accessible to download cubins
if not has_nvidia_artifactory():
return False
# SM90 has XQA decode; prefill is not supported.
if current_platform.is_device_capability(90):
return not is_prefill
# SM100/SM103 has both prefill and decode TRTLLM kernels.
return current_platform.is_device_capability_family(100)
def force_use_trtllm_attention() -> bool | None:
@@ -400,12 +408,15 @@ def force_use_trtllm_attention() -> bool | None:
return vllm_config.attention_config.use_trtllm_attention
def can_use_trtllm_attention(num_qo_heads: int, num_kv_heads: int) -> bool:
def can_use_trtllm_attention(
num_qo_heads: int, num_kv_heads: int, is_prefill: bool = False
) -> bool:
"""Check if the current configuration supports TRTLLM attention."""
if force_use_trtllm_attention() is False:
return False
has_trtllm = supports_trtllm_attention()
return has_trtllm and (num_qo_heads % num_kv_heads == 0)
return supports_trtllm_attention(is_prefill=is_prefill) and (
num_qo_heads % num_kv_heads == 0
)
def use_trtllm_attention(
@@ -437,11 +448,12 @@ def use_trtllm_attention(
return False
# The platform is not supported
if not supports_trtllm_attention():
if not supports_trtllm_attention(is_prefill=is_prefill):
if force_use_trtllm:
logger.warning_once(
"TRTLLM attention is not supported on this platform, "
"but --attention-config.use_trtllm_attention is set to 1"
"TRTLLM attention is not supported on this platform for %s, "
"but --attention-config.use_trtllm_attention is set to 1",
"prefill" if is_prefill else "decode",
)
return False
@@ -476,13 +488,20 @@ def use_trtllm_attention(
if is_prefill:
# Prefill auto-detection
use_trtllm = kv_cache_dtype == "auto"
if use_trtllm:
logger.warning_once("Using TRTLLM prefill attention (auto-detected).")
elif current_platform.is_device_capability(90) and kv_cache_dtype.startswith(
"fp8"
):
# SM90 + FP8 KV cache: prefer the XQA decode kernel. XQA does not
# support NVFP4 KV (that is an SM100 trtllm-gen path only).
use_trtllm = True
else:
# Decode auto-detection
use_trtllm = num_tokens <= 256 and kv_cache_dtype == "auto"
if use_trtllm:
logger.warning_once("Using TRTLLM decode attention (auto-detected).")
if use_trtllm:
logger.warning_once(
"Using TRTLLM %s attention (auto-detected).",
"prefill" if is_prefill else "decode",
)
return use_trtllm
# CLI argument is set to 1 - respect it