Xqa decode kernels (#43232)
Signed-off-by: Dan Blanaru <48605845+DanBlanaru@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
d715b3aa1e
commit
d29125c085
+37
-18
@@ -371,19 +371,27 @@ def has_nvidia_artifactory() -> bool:
|
||||
|
||||
|
||||
@functools.cache
|
||||
def supports_trtllm_attention() -> bool:
|
||||
"""
|
||||
TRTLLM attention is supported if the platform is SM100,
|
||||
NVIDIA artifactory is accessible, and batch-invariant mode is not enabled.
|
||||
def supports_trtllm_attention(is_prefill: bool = False) -> bool:
|
||||
"""Return whether TRTLLM attention is available on the current platform
|
||||
for the given attention phase.
|
||||
|
||||
SM90 (Hopper) supports the XQA decode kernel but not TRTLLM prefill.
|
||||
SM100+ supports TRTLLM for both phases. All others are unsupported.
|
||||
"""
|
||||
# Batch-invariant mode disables TRTLLM attention
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
return False
|
||||
|
||||
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
|
||||
return (
|
||||
current_platform.is_device_capability_family(100) and has_nvidia_artifactory()
|
||||
)
|
||||
# Requires NVIDIA artifactory to be accessible to download cubins
|
||||
if not has_nvidia_artifactory():
|
||||
return False
|
||||
|
||||
# SM90 has XQA decode; prefill is not supported.
|
||||
if current_platform.is_device_capability(90):
|
||||
return not is_prefill
|
||||
|
||||
# SM100/SM103 has both prefill and decode TRTLLM kernels.
|
||||
return current_platform.is_device_capability_family(100)
|
||||
|
||||
|
||||
def force_use_trtllm_attention() -> bool | None:
|
||||
@@ -400,12 +408,15 @@ def force_use_trtllm_attention() -> bool | None:
|
||||
return vllm_config.attention_config.use_trtllm_attention
|
||||
|
||||
|
||||
def can_use_trtllm_attention(num_qo_heads: int, num_kv_heads: int) -> bool:
|
||||
def can_use_trtllm_attention(
|
||||
num_qo_heads: int, num_kv_heads: int, is_prefill: bool = False
|
||||
) -> bool:
|
||||
"""Check if the current configuration supports TRTLLM attention."""
|
||||
if force_use_trtllm_attention() is False:
|
||||
return False
|
||||
has_trtllm = supports_trtllm_attention()
|
||||
return has_trtllm and (num_qo_heads % num_kv_heads == 0)
|
||||
return supports_trtllm_attention(is_prefill=is_prefill) and (
|
||||
num_qo_heads % num_kv_heads == 0
|
||||
)
|
||||
|
||||
|
||||
def use_trtllm_attention(
|
||||
@@ -437,11 +448,12 @@ def use_trtllm_attention(
|
||||
return False
|
||||
|
||||
# The platform is not supported
|
||||
if not supports_trtllm_attention():
|
||||
if not supports_trtllm_attention(is_prefill=is_prefill):
|
||||
if force_use_trtllm:
|
||||
logger.warning_once(
|
||||
"TRTLLM attention is not supported on this platform, "
|
||||
"but --attention-config.use_trtllm_attention is set to 1"
|
||||
"TRTLLM attention is not supported on this platform for %s, "
|
||||
"but --attention-config.use_trtllm_attention is set to 1",
|
||||
"prefill" if is_prefill else "decode",
|
||||
)
|
||||
return False
|
||||
|
||||
@@ -476,13 +488,20 @@ def use_trtllm_attention(
|
||||
if is_prefill:
|
||||
# Prefill auto-detection
|
||||
use_trtllm = kv_cache_dtype == "auto"
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM prefill attention (auto-detected).")
|
||||
elif current_platform.is_device_capability(90) and kv_cache_dtype.startswith(
|
||||
"fp8"
|
||||
):
|
||||
# SM90 + FP8 KV cache: prefer the XQA decode kernel. XQA does not
|
||||
# support NVFP4 KV (that is an SM100 trtllm-gen path only).
|
||||
use_trtllm = True
|
||||
else:
|
||||
# Decode auto-detection
|
||||
use_trtllm = num_tokens <= 256 and kv_cache_dtype == "auto"
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM decode attention (auto-detected).")
|
||||
if use_trtllm:
|
||||
logger.warning_once(
|
||||
"Using TRTLLM %s attention (auto-detected).",
|
||||
"prefill" if is_prefill else "decode",
|
||||
)
|
||||
return use_trtllm
|
||||
|
||||
# CLI argument is set to 1 - respect it
|
||||
|
||||
Reference in New Issue
Block a user