[Attention] Add FlashInfer XQA decode support on SM12x (#49718)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andrii Skliar
OpenAI Codex
parent
a367cdbca0
commit
1d2d83a07f
@@ -101,6 +101,20 @@ def test_supports_sm90_decode_only(_art, _family, _cap):
|
||||
assert supports_trtllm_attention(is_prefill=True) is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability", return_value=False
|
||||
)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
side_effect=lambda capability: capability == 120,
|
||||
)
|
||||
@patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=True)
|
||||
def test_supports_sm12x_decode_only(_art, _family, _cap):
|
||||
assert supports_trtllm_attention(is_prefill=False) is True
|
||||
assert supports_trtllm_attention(is_prefill=True) is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
|
||||
Reference in New Issue
Block a user