[Attention] Allow an NVFP4 KV cache on SM90 with fa2 (#60620)

Signed-off-by: lesj0610 <lesj0610@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
SeongJun Lee
2026-10-08 12:16:22 -07:00
committed by GitHub
co-authored by Misha Goin
parent fc494822ea
commit fff10d2072
2 changed files with 5 additions and 7 deletions
+1 -1
View File
@@ -706,7 +706,7 @@ def test_flashinfer_decode_with_paged_fp8_kv(
@pytest.mark.skipif(
not any(current_platform.is_device_capability_family(f) for f in (80, 120)),
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
reason="NVFP4 fa2 path",
)
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
+4 -6
View File
@@ -108,9 +108,7 @@ trtllm_workspace_buffer = None
def _nvfp4_kv_on_fa2() -> bool:
return current_platform.is_device_capability_family(
80
) or current_platform.is_device_capability_family(120)
return any(current_platform.is_device_capability_family(f) for f in (80, 90, 120))
_KVPair = tuple[torch.Tensor, torch.Tensor]
@@ -557,7 +555,7 @@ class FlashInferBackend(AttentionBackend):
if (
kv_cache_dtype is not None
and kv_cache_dtype.startswith("nvfp4")
and device_capability.major in (8, 12)
and device_capability.major in (8, 9, 12)
):
if head_size == 64:
return "fa2 prefill misreads an NVFP4 KV cache at head_size 64"
@@ -1311,7 +1309,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
)
else:
# NVFP4 KV cache requires the trtllm-gen backend inside
# the wrapper on SM100; SM8x and SM12x read it with fa2.
# the wrapper on SM100; SM8x, SM90 and SM12x read it with fa2.
backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
self._get_workspace_buffer(),
@@ -1337,7 +1335,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
paged_kv_indices = None
paged_kv_last_page_len = None
# NVFP4 KV cache requires the trtllm-gen backend inside
# the wrapper on SM100; SM8x and SM12x read it with fa2.
# the wrapper on SM100; SM8x, SM90 and SM12x read it with fa2.
backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
self._get_workspace_buffer(),