Add nvfp4 support to reshape_and_cache_flash (#37332)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-04-17 07:28:00 -07:00
committed by GitHub
parent 70770268c3
commit 6b2b7bd0eb
10 changed files with 679 additions and 51 deletions
+27 -3
View File
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import get_dtype_size
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
logger = init_logger(__name__)
@@ -38,11 +38,20 @@ class KVQuantMode(IntEnum):
FP8_PER_TENSOR = 1 # per-tensor scales (current fp8 path)
INT8_PER_TOKEN_HEAD = 2 # per-token-head dynamic scales for int8
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
NVFP4 = 4 # packed fp4 data + fp8 block scales
@property
def is_per_token_head(self) -> bool:
"""True for any per-token-head quantization mode."""
return self >= 2
return self in (
KVQuantMode.INT8_PER_TOKEN_HEAD,
KVQuantMode.FP8_PER_TOKEN_HEAD,
)
@property
def is_nvfp4(self) -> bool:
"""True for NVFP4 packed quantization mode."""
return self == KVQuantMode.NVFP4
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
@@ -51,7 +60,9 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
return KVQuantMode.INT8_PER_TOKEN_HEAD
if kv_cache_dtype == "fp8_per_token_head":
return KVQuantMode.FP8_PER_TOKEN_HEAD
if kv_cache_dtype.startswith("fp8"):
if kv_cache_dtype == "nvfp4":
return KVQuantMode.NVFP4
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
return KVQuantMode.FP8_PER_TENSOR
return KVQuantMode.NONE
@@ -237,6 +248,19 @@ class FullAttentionSpec(AttentionSpec):
@property
def real_page_size_bytes(self) -> int:
if self.kv_quant_mode.is_nvfp4:
# Packed layout per head: fp4 data + fp8 block scales.
# fp4 data: head_size//2 bytes (2 fp4 values per byte)
# fp8 block scale: head_size//16 bytes (1 scale per 16 elements)
last_dim = nvfp4_kv_cache_full_dim(
self.head_size
) + nvfp4_kv_cache_full_dim(self.head_size_v)
return (
self.block_size
* self.num_kv_heads
* last_dim
* get_dtype_size(self.dtype)
)
return (
self.block_size
* self.num_kv_heads