Add nvfp4 support to reshape_and_cache_flash (#37332)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -17,7 +17,7 @@ from vllm.logger import init_logger
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -38,11 +38,20 @@ class KVQuantMode(IntEnum):
|
||||
FP8_PER_TENSOR = 1 # per-tensor scales (current fp8 path)
|
||||
INT8_PER_TOKEN_HEAD = 2 # per-token-head dynamic scales for int8
|
||||
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
|
||||
NVFP4 = 4 # packed fp4 data + fp8 block scales
|
||||
|
||||
@property
|
||||
def is_per_token_head(self) -> bool:
|
||||
"""True for any per-token-head quantization mode."""
|
||||
return self >= 2
|
||||
return self in (
|
||||
KVQuantMode.INT8_PER_TOKEN_HEAD,
|
||||
KVQuantMode.FP8_PER_TOKEN_HEAD,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_nvfp4(self) -> bool:
|
||||
"""True for NVFP4 packed quantization mode."""
|
||||
return self == KVQuantMode.NVFP4
|
||||
|
||||
|
||||
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
@@ -51,7 +60,9 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
return KVQuantMode.INT8_PER_TOKEN_HEAD
|
||||
if kv_cache_dtype == "fp8_per_token_head":
|
||||
return KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
return KVQuantMode.NVFP4
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
|
||||
return KVQuantMode.FP8_PER_TENSOR
|
||||
return KVQuantMode.NONE
|
||||
|
||||
@@ -237,6 +248,19 @@ class FullAttentionSpec(AttentionSpec):
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.kv_quant_mode.is_nvfp4:
|
||||
# Packed layout per head: fp4 data + fp8 block scales.
|
||||
# fp4 data: head_size//2 bytes (2 fp4 values per byte)
|
||||
# fp8 block scale: head_size//16 bytes (1 scale per 16 elements)
|
||||
last_dim = nvfp4_kv_cache_full_dim(
|
||||
self.head_size
|
||||
) + nvfp4_kv_cache_full_dim(self.head_size_v)
|
||||
return (
|
||||
self.block_size
|
||||
* self.num_kv_heads
|
||||
* last_dim
|
||||
* get_dtype_size(self.dtype)
|
||||
)
|
||||
return (
|
||||
self.block_size
|
||||
* self.num_kv_heads
|
||||
|
||||
Reference in New Issue
Block a user