[Attention] Support NVFP4 KV cache on SM8x and SM12x with FlashInfer (#46963)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
lesj0610
mgoin
Claude
parent
3d6853b32c
commit
554340f3d3
@@ -6,6 +6,7 @@ import pytest
|
|||||||
|
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.torch_utils import set_random_seed
|
from vllm.utils.torch_utils import set_random_seed
|
||||||
|
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import flashinfer
|
import flashinfer
|
||||||
@@ -702,3 +703,35 @@ def test_flashinfer_decode_with_paged_fp8_kv(
|
|||||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
|
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
|
||||||
f"{torch.max(torch.abs(output - ref_output))}",
|
f"{torch.max(torch.abs(output - ref_output))}",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
not any(current_platform.is_device_capability_family(f) for f in (80, 120)),
|
||||||
|
reason="NVFP4 fa2 path",
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
|
||||||
|
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
|
||||||
|
from vllm.v1.attention.backends import flashinfer as fi
|
||||||
|
|
||||||
|
torch.set_default_device("cuda")
|
||||||
|
order = KVCacheLayout[layout].stride_order
|
||||||
|
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
|
||||||
|
shape = (2, 6, 4, 16, 64 + 8)
|
||||||
|
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
|
||||||
|
layers = layers.permute(*(order.index(i) for i in range(5)))
|
||||||
|
one = torch.ones(())
|
||||||
|
layer = torch.nn.Module()
|
||||||
|
layer._k_scale = layer._v_scale = one
|
||||||
|
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
|
||||||
|
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
|
||||||
|
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
|
||||||
|
|
||||||
|
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
|
||||||
|
pages = torch.arange(6, dtype=torch.int32)[None]
|
||||||
|
seq_lens = torch.tensor([96], dtype=torch.int32)
|
||||||
|
deq = torch.empty_like(kv)[:, None]
|
||||||
|
flashinfer.nvfp4_kv_dequantize_paged(
|
||||||
|
data, scales, pages, seq_lens, one, one, *deq, kv_layout
|
||||||
|
)
|
||||||
|
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
|
||||||
|
assert not layers[0].any()
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from flashinfer import (
|
|||||||
BatchPrefillWithRaggedKVCacheWrapper,
|
BatchPrefillWithRaggedKVCacheWrapper,
|
||||||
MultiLevelCascadeAttentionWrapper,
|
MultiLevelCascadeAttentionWrapper,
|
||||||
get_seq_lens,
|
get_seq_lens,
|
||||||
|
nvfp4_quantize_append_paged_kv_cache_with_slot_mapping,
|
||||||
)
|
)
|
||||||
from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache
|
from flashinfer.decode import fast_decode_plan, trtllm_batch_decode_with_kv_cache
|
||||||
from flashinfer.prefill import trtllm_batch_context_with_kv_cache
|
from flashinfer.prefill import trtllm_batch_context_with_kv_cache
|
||||||
@@ -106,6 +107,38 @@ logger = init_logger(__name__)
|
|||||||
trtllm_workspace_buffer = None
|
trtllm_workspace_buffer = None
|
||||||
|
|
||||||
|
|
||||||
|
def _nvfp4_kv_on_fa2() -> bool:
|
||||||
|
return current_platform.is_device_capability_family(
|
||||||
|
80
|
||||||
|
) or current_platform.is_device_capability_family(120)
|
||||||
|
|
||||||
|
|
||||||
|
_KVPair = tuple[torch.Tensor, torch.Tensor]
|
||||||
|
|
||||||
|
|
||||||
|
def _nvfp4_kv_views(
|
||||||
|
kv_cache: torch.Tensor, num_kv_heads: int
|
||||||
|
) -> tuple[_KVPair, _KVPair, str]:
|
||||||
|
"""Views of NVFP4 pages laid out as [K data | K scale | V data | V scale]."""
|
||||||
|
num_blocks, num_slots, block_size, full_dim = kv_cache.shape
|
||||||
|
hnd = kv_cache.stride()[1:] == (block_size * full_dim, full_dim, 1)
|
||||||
|
if not hnd and kv_cache.stride()[1:] != (full_dim, num_slots * full_dim, 1):
|
||||||
|
raise ValueError(f"NVFP4 KV pages must be dense, got {kv_cache.stride()}")
|
||||||
|
dims = (num_kv_heads, block_size) if hnd else (block_size, num_kv_heads)
|
||||||
|
side_size = num_kv_heads * block_size * full_dim
|
||||||
|
(k_data, k_scale), (v_data, v_scale) = (
|
||||||
|
nvfp4_split_data_scale(
|
||||||
|
kv_cache.as_strided(
|
||||||
|
(num_blocks, *dims, full_dim),
|
||||||
|
(kv_cache.stride(0), dims[1] * full_dim, full_dim, 1),
|
||||||
|
kv_cache.storage_offset() + side * side_size,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for side in range(2)
|
||||||
|
)
|
||||||
|
return (k_data, v_data), (k_scale, v_scale), "HND" if hnd else "NHD"
|
||||||
|
|
||||||
|
|
||||||
def _get_trtllm_workspace_buffer():
|
def _get_trtllm_workspace_buffer():
|
||||||
global trtllm_workspace_buffer
|
global trtllm_workspace_buffer
|
||||||
if trtllm_workspace_buffer is None:
|
if trtllm_workspace_buffer is None:
|
||||||
@@ -498,6 +531,9 @@ class FlashInferBackend(AttentionBackend):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||||
if kv_cache_dtype is not None and kv_cache_dtype.startswith("nvfp4"):
|
if kv_cache_dtype is not None and kv_cache_dtype.startswith("nvfp4"):
|
||||||
|
if _nvfp4_kv_on_fa2():
|
||||||
|
# The slot-mapping writer has no scale search.
|
||||||
|
return kv_cache_dtype == "nvfp4"
|
||||||
return (
|
return (
|
||||||
current_platform.is_device_capability_family(100)
|
current_platform.is_device_capability_family(100)
|
||||||
and supports_trtllm_attention(is_prefill=True)
|
and supports_trtllm_attention(is_prefill=True)
|
||||||
@@ -505,6 +541,30 @@ class FlashInferBackend(AttentionBackend):
|
|||||||
)
|
)
|
||||||
return super().supports_kv_cache_dtype(kv_cache_dtype)
|
return super().supports_kv_cache_dtype(kv_cache_dtype)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def supports_combination(
|
||||||
|
cls,
|
||||||
|
head_size: int,
|
||||||
|
dtype: torch.dtype,
|
||||||
|
kv_cache_dtype: CacheDType | None,
|
||||||
|
block_size: int | None,
|
||||||
|
use_mla: bool,
|
||||||
|
has_sink: bool,
|
||||||
|
use_sparse: bool,
|
||||||
|
use_mm_prefix: bool,
|
||||||
|
device_capability: DeviceCapability,
|
||||||
|
) -> str | None:
|
||||||
|
if (
|
||||||
|
kv_cache_dtype is not None
|
||||||
|
and kv_cache_dtype.startswith("nvfp4")
|
||||||
|
and device_capability.major in (8, 12)
|
||||||
|
):
|
||||||
|
if head_size == 64:
|
||||||
|
return "fa2 prefill misreads an NVFP4 KV cache at head_size 64"
|
||||||
|
if has_sink:
|
||||||
|
return "the sink prefill wrapper cannot read an NVFP4 KV cache"
|
||||||
|
return None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_supported_head_sizes(cls) -> list[int]:
|
def get_supported_head_sizes(cls) -> list[int]:
|
||||||
# https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
|
# https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
|
||||||
@@ -777,7 +837,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
# Cannot use self.kv_cache_spec.dtype here because kv_cache_spec
|
# Cannot use self.kv_cache_spec.dtype here because kv_cache_spec
|
||||||
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
|
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
|
||||||
self.is_kvcache_nvfp4 = self.cache_dtype.startswith("nvfp4")
|
self.is_kvcache_nvfp4 = self.cache_dtype.startswith("nvfp4")
|
||||||
if self.is_kvcache_nvfp4:
|
# fa2 plans NVFP4 by its uint8 storage dtype.
|
||||||
|
self.nvfp4_fa2 = self.is_kvcache_nvfp4 and _nvfp4_kv_on_fa2()
|
||||||
|
self.nvfp4_trtllm = self.is_kvcache_nvfp4 and not self.nvfp4_fa2
|
||||||
|
if self.nvfp4_fa2 and self.use_dcp:
|
||||||
|
raise NotImplementedError(
|
||||||
|
"The DCP prefill wrapper cannot read NVFP4 block scales."
|
||||||
|
)
|
||||||
|
if self.nvfp4_fa2 and self.head_dim == 64:
|
||||||
|
raise NotImplementedError("fa2 prefill misreads NVFP4 at head_size 64.")
|
||||||
|
if self.nvfp4_trtllm:
|
||||||
if (
|
if (
|
||||||
force_use_trtllm_attention() is False
|
force_use_trtllm_attention() is False
|
||||||
or not supports_trtllm_attention(is_prefill=True)
|
or not supports_trtllm_attention(is_prefill=True)
|
||||||
@@ -798,6 +867,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
else:
|
else:
|
||||||
self.cache_dtype = "auto"
|
self.cache_dtype = "auto"
|
||||||
self.is_kvcache_nvfp4 = False
|
self.is_kvcache_nvfp4 = False
|
||||||
|
self.nvfp4_fa2 = self.nvfp4_trtllm = False
|
||||||
assert self.kv_cache_spec.dtype == self.model_config.dtype
|
assert self.kv_cache_spec.dtype == self.model_config.dtype
|
||||||
self.kv_cache_dtype = self.kv_cache_spec.dtype
|
self.kv_cache_dtype = self.kv_cache_spec.dtype
|
||||||
|
|
||||||
@@ -810,8 +880,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
|
|
||||||
# Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel
|
# Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel
|
||||||
# must be selected statically for FULL cudagraph capture.
|
# must be selected statically for FULL cudagraph capture.
|
||||||
can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention(
|
can_use_xqa_or_trtllm_gen_decode = (
|
||||||
self.num_qo_heads, self.num_kv_heads, is_prefill=False
|
can_use_trtllm_attention(
|
||||||
|
self.num_qo_heads, self.num_kv_heads, is_prefill=False
|
||||||
|
)
|
||||||
|
and not self.nvfp4_fa2
|
||||||
)
|
)
|
||||||
# Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by
|
# Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by
|
||||||
# get_supported_kernel_block_sizes).
|
# get_supported_kernel_block_sizes).
|
||||||
@@ -894,7 +967,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
self.window_left = self.global_hyperparameters.window_left
|
self.window_left = self.global_hyperparameters.window_left
|
||||||
self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap
|
self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap
|
||||||
self.has_sinks = self.global_hyperparameters.has_sinks
|
self.has_sinks = self.global_hyperparameters.has_sinks
|
||||||
if self.has_sinks and not FlashInferBackend.supports_sink():
|
if self.has_sinks and (self.nvfp4_fa2 or not FlashInferBackend.supports_sink()):
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"FlashInfer backend currently does not support attention "
|
"FlashInfer backend currently does not support attention "
|
||||||
"sinks, please use trtllm on blackwell or flash attention on "
|
"sinks, please use trtllm on blackwell or flash attention on "
|
||||||
@@ -970,6 +1043,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype)
|
return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype)
|
||||||
return self.model_config.dtype
|
return self.model_config.dtype
|
||||||
if cache_dtype.startswith("nvfp4"):
|
if cache_dtype.startswith("nvfp4"):
|
||||||
|
if self.nvfp4_fa2:
|
||||||
|
return self.model_config.dtype
|
||||||
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
|
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
|
||||||
return self.kv_cache_spec.dtype
|
return self.kv_cache_spec.dtype
|
||||||
|
|
||||||
@@ -1002,7 +1077,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
num_qo_heads=num_qo_heads,
|
num_qo_heads=num_qo_heads,
|
||||||
num_kv_heads=spec.num_kv_heads,
|
num_kv_heads=spec.num_kv_heads,
|
||||||
is_prefill=False,
|
is_prefill=False,
|
||||||
) or (is_xqa_arch and not _is_xqa_head_dim_supported(spec.head_size)):
|
) or (
|
||||||
|
is_xqa_arch
|
||||||
|
and (
|
||||||
|
spec.kv_quant_mode.is_nvfp4
|
||||||
|
or not _is_xqa_head_dim_supported(spec.head_size)
|
||||||
|
)
|
||||||
|
):
|
||||||
has_uniform_batch_support = False
|
has_uniform_batch_support = False
|
||||||
break
|
break
|
||||||
|
|
||||||
@@ -1230,8 +1311,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# NVFP4 KV cache requires the trtllm-gen backend inside
|
# NVFP4 KV cache requires the trtllm-gen backend inside
|
||||||
# the wrapper; fa2/fa3 do not support nvfp4.
|
# the wrapper on SM100; SM8x and SM12x read it with fa2.
|
||||||
backend = "trtllm-gen" if self.is_kvcache_nvfp4 else "auto"
|
backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
|
||||||
self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
|
self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
|
||||||
self._get_workspace_buffer(),
|
self._get_workspace_buffer(),
|
||||||
get_flashinfer_layout_string(self.kv_cache_layout),
|
get_flashinfer_layout_string(self.kv_cache_layout),
|
||||||
@@ -1256,8 +1337,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
paged_kv_indices = None
|
paged_kv_indices = None
|
||||||
paged_kv_last_page_len = None
|
paged_kv_last_page_len = None
|
||||||
# NVFP4 KV cache requires the trtllm-gen backend inside
|
# NVFP4 KV cache requires the trtllm-gen backend inside
|
||||||
# the wrapper; fa2/fa3 do not support nvfp4.
|
# the wrapper on SM100; SM8x and SM12x read it with fa2.
|
||||||
backend = "trtllm-gen" if self.is_kvcache_nvfp4 else "auto"
|
backend = "trtllm-gen" if self.nvfp4_trtllm else "auto"
|
||||||
decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
|
decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
|
||||||
self._get_workspace_buffer(),
|
self._get_workspace_buffer(),
|
||||||
get_flashinfer_layout_string(self.kv_cache_layout),
|
get_flashinfer_layout_string(self.kv_cache_layout),
|
||||||
@@ -1710,7 +1791,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
# use FP8 o_data_type so the wrapper matches the
|
# use FP8 o_data_type so the wrapper matches the
|
||||||
# FP8 output buffer allocated in forward().
|
# FP8 output buffer allocated in forward().
|
||||||
o_dtype = (
|
o_dtype = (
|
||||||
FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype
|
FP8_DTYPE if self.nvfp4_trtllm else self.model_config.dtype
|
||||||
)
|
)
|
||||||
prefill_wrapper.plan(
|
prefill_wrapper.plan(
|
||||||
qo_indptr=qo_indptr_prefill_cpu,
|
qo_indptr=qo_indptr_prefill_cpu,
|
||||||
@@ -1797,9 +1878,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
|||||||
# NVFP4 trtllm kernel only supports FP8 output;
|
# NVFP4 trtllm kernel only supports FP8 output;
|
||||||
# use FP8 o_data_type so the wrapper matches the
|
# use FP8 o_data_type so the wrapper matches the
|
||||||
# FP8 output buffer allocated in forward().
|
# FP8 output buffer allocated in forward().
|
||||||
o_dtype = (
|
o_dtype = FP8_DTYPE if self.nvfp4_trtllm else self.model_config.dtype
|
||||||
FP8_DTYPE if self.is_kvcache_nvfp4 else self.model_config.dtype
|
|
||||||
)
|
|
||||||
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1]
|
paged_kv_indptr_cpu = self.paged_kv_indptr.cpu[: num_input_tokens + 1]
|
||||||
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[
|
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[
|
||||||
:num_input_tokens
|
:num_input_tokens
|
||||||
@@ -1891,6 +1970,8 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
)
|
)
|
||||||
self.cache_dtype = kv_cache_dtype
|
self.cache_dtype = kv_cache_dtype
|
||||||
self.is_kvcache_nvfp4 = kv_cache_dtype.startswith("nvfp4")
|
self.is_kvcache_nvfp4 = kv_cache_dtype.startswith("nvfp4")
|
||||||
|
self.nvfp4_fa2 = self.is_kvcache_nvfp4 and _nvfp4_kv_on_fa2()
|
||||||
|
self.nvfp4_trtllm = self.is_kvcache_nvfp4 and not self.nvfp4_fa2
|
||||||
self.kv_cache_dtype = "nvfp4" if self.is_kvcache_nvfp4 else kv_cache_dtype
|
self.kv_cache_dtype = "nvfp4" if self.is_kvcache_nvfp4 else kv_cache_dtype
|
||||||
self.fp4_data_dim = head_size // 2 if self.is_kvcache_nvfp4 else 0
|
self.fp4_data_dim = head_size // 2 if self.is_kvcache_nvfp4 else 0
|
||||||
self.logits_soft_cap = logits_soft_cap
|
self.logits_soft_cap = logits_soft_cap
|
||||||
@@ -1937,7 +2018,7 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
self.o_sf_scale: float | None = None
|
self.o_sf_scale: float | None = None
|
||||||
|
|
||||||
# Pre-allocated FP8 output buffer for NVFP4 without fused output quant.
|
# Pre-allocated FP8 output buffer for NVFP4 without fused output quant.
|
||||||
if self.is_kvcache_nvfp4 and vllm_config is not None:
|
if self.nvfp4_trtllm and vllm_config is not None:
|
||||||
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||||
self._nvfp4_fp8_out = torch.empty(
|
self._nvfp4_fp8_out = torch.empty(
|
||||||
(max_num_tokens, num_heads, head_size),
|
(max_num_tokens, num_heads, head_size),
|
||||||
@@ -2195,10 +2276,9 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)),
|
canonicalize_singleton_dim_strides(k_cache.permute(*stride_order)),
|
||||||
canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)),
|
canonicalize_singleton_dim_strides(v_cache.permute(*stride_order)),
|
||||||
)
|
)
|
||||||
k_data, k_sf = nvfp4_split_data_scale(kv_cache_tuple[0])
|
nvfp4_kv_data, nvfp4_kv_block_scales, _ = _nvfp4_kv_views(
|
||||||
v_data, v_sf = nvfp4_split_data_scale(kv_cache_tuple[1])
|
kv_cache, self.num_kv_heads
|
||||||
nvfp4_kv_data = (k_data, v_data)
|
)
|
||||||
nvfp4_kv_block_scales = (k_sf, v_sf)
|
|
||||||
else:
|
else:
|
||||||
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
|
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
|
||||||
|
|
||||||
@@ -2276,7 +2356,7 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
# Use a pre-allocated FP8 buffer and dequantize
|
# Use a pre-allocated FP8 buffer and dequantize
|
||||||
# afterwards.
|
# afterwards.
|
||||||
needs_fp8_out_prefill = (
|
needs_fp8_out_prefill = (
|
||||||
self.is_kvcache_nvfp4 and output.dtype != FP8_DTYPE
|
self.nvfp4_trtllm and output.dtype != FP8_DTYPE
|
||||||
)
|
)
|
||||||
if needs_fp8_out_prefill:
|
if needs_fp8_out_prefill:
|
||||||
out_prefill = self._nvfp4_fp8_out[:num_prefill_tokens]
|
out_prefill = self._nvfp4_fp8_out[:num_prefill_tokens]
|
||||||
@@ -2299,7 +2379,8 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
prefill_wrapper.run(
|
prefill_wrapper.run(
|
||||||
prefill_query,
|
prefill_query,
|
||||||
kv_cache_for_fi,
|
kv_cache_for_fi,
|
||||||
q_scale=layer._q_scale_float,
|
# The fa2 NVFP4 query stays in the model dtype.
|
||||||
|
q_scale=1.0 if self.nvfp4_fa2 else layer._q_scale_float,
|
||||||
k_scale=layer._k_scale_float,
|
k_scale=layer._k_scale_float,
|
||||||
v_scale=layer._v_scale_float,
|
v_scale=layer._v_scale_float,
|
||||||
out=out_prefill,
|
out=out_prefill,
|
||||||
@@ -2451,7 +2532,7 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
|
|
||||||
# NVFP4 kernel only supports FP8 output.
|
# NVFP4 kernel only supports FP8 output.
|
||||||
# Use a pre-allocated FP8 buffer and dequantize afterwards.
|
# Use a pre-allocated FP8 buffer and dequantize afterwards.
|
||||||
needs_fp8_out = self.is_kvcache_nvfp4 and output.dtype != FP8_DTYPE
|
needs_fp8_out = self.nvfp4_trtllm and output.dtype != FP8_DTYPE
|
||||||
if needs_fp8_out:
|
if needs_fp8_out:
|
||||||
out_decode = self._nvfp4_fp8_out[:num_decode_tokens]
|
out_decode = self._nvfp4_fp8_out[:num_decode_tokens]
|
||||||
else:
|
else:
|
||||||
@@ -2488,7 +2569,7 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
decode_wrapper.run(
|
decode_wrapper.run(
|
||||||
decode_query,
|
decode_query,
|
||||||
kv_cache_for_fi,
|
kv_cache_for_fi,
|
||||||
q_scale=layer._q_scale_float,
|
q_scale=1.0 if self.nvfp4_fa2 else layer._q_scale_float,
|
||||||
k_scale=layer._k_scale_float,
|
k_scale=layer._k_scale_float,
|
||||||
v_scale=layer._v_scale_float,
|
v_scale=layer._v_scale_float,
|
||||||
out=out_decode,
|
out=out_decode,
|
||||||
@@ -2668,6 +2749,19 @@ class FlashInferImpl(AttentionImpl):
|
|||||||
# and value[:num_actual_tokens] because the reshape_and_cache_flash
|
# and value[:num_actual_tokens] because the reshape_and_cache_flash
|
||||||
# op uses the slot_mapping's shape to determine the number of
|
# op uses the slot_mapping's shape to determine the number of
|
||||||
# actual tokens.
|
# actual tokens.
|
||||||
|
if self.nvfp4_fa2:
|
||||||
|
data, scales, kv_layout = _nvfp4_kv_views(kv_cache, self.num_kv_heads)
|
||||||
|
nvfp4_quantize_append_paged_kv_cache_with_slot_mapping(
|
||||||
|
key,
|
||||||
|
value,
|
||||||
|
slot_mapping,
|
||||||
|
data,
|
||||||
|
scales,
|
||||||
|
layer._k_scale,
|
||||||
|
layer._v_scale,
|
||||||
|
kv_layout=kv_layout,
|
||||||
|
)
|
||||||
|
return
|
||||||
if self.is_kvcache_nvfp4:
|
if self.is_kvcache_nvfp4:
|
||||||
# (B, 2*H, N, full_dim) -> ((B, N, H, full_dim),
|
# (B, 2*H, N, full_dim) -> ((B, N, H, full_dim),
|
||||||
# (B, N, H, full_dim));
|
# (B, N, H, full_dim));
|
||||||
|
|||||||
Reference in New Issue
Block a user