[Attention] Allow an NVFP4 KV cache on SM90 with fa2 (#60620)
Signed-off-by: lesj0610 <lesj0610@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Misha Goin
parent
fc494822ea
commit
fff10d2072
@@ -706,7 +706,7 @@ def test_flashinfer_decode_with_paged_fp8_kv(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not any(current_platform.is_device_capability_family(f) for f in (80, 120)),
|
||||
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
|
||||
reason="NVFP4 fa2 path",
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
|
||||
|
||||
@@ -108,9 +108,7 @@ trtllm_workspace_buffer = None
|
||||
|
||||
|
||||
def _nvfp4_kv_on_fa2() -> bool:
|
||||
return current_platform.is_device_capability_family(
|
||||
80
|
||||
) or current_platform.is_device_capability_family(120)
|
||||
return any(current_platform.is_device_capability_family(f) for f in (80, 90, 120))
|
||||
|
||||
|
||||
_KVPair = tuple[torch.Tensor, torch.Tensor]
|
||||
@@ -557,7 +555,7 @@ class FlashInferBackend(AttentionBackend):
|
||||
if (
|
||||
kv_cache_dtype is not None
|
||||
and kv_cache_dtype.startswith("nvfp4")
|
||||
and device_capability.major in (8, 12)
|
||||
and device_capability.major in (8, 9, 12)
|
||||
):
|
||||
if head_size == 64:
|
||||
return "fa2 prefill misreads an NVFP4 KV cache at head_size 64"
|
||||
@@ -1311,7 +1309,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
)
|
||||
else:
|
||||
# NVFP4 KV cache requires the trtllm-gen backend inside
|
||||
# the wrapper on SM100; SM8x and SM12x read it with fa2.
|
||||
# the wrapper on SM100; SM8x, SM90 and SM12x read it with fa2.
|
||||
backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
|
||||
self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
|
||||
self._get_workspace_buffer(),
|
||||
@@ -1337,7 +1335,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
paged_kv_indices = None
|
||||
paged_kv_last_page_len = None
|
||||
# NVFP4 KV cache requires the trtllm-gen backend inside
|
||||
# the wrapper on SM100; SM8x and SM12x read it with fa2.
|
||||
# the wrapper on SM100; SM8x, SM90 and SM12x read it with fa2.
|
||||
backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
|
||||
decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
|
||||
self._get_workspace_buffer(),
|
||||
|
||||
Reference in New Issue
Block a user