Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
sychen52
2026-07-04 02:29:13 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3b6d97aa
commit 67ff0ae30f
6 changed files with 228 additions and 5 deletions
+9 -1
View File
@@ -150,6 +150,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVQuantMode,
MambaSpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
@@ -7147,12 +7148,19 @@ class GPUModelRunner(
else:
shape_block_size = kernel_block_size
# Skipped layers (--kv-cache-dtype-skip-layers) need
# the unquantized shape.
layer_cache_dtype_str = (
"auto"
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
else self.cache_config.cache_dtype
)
kv_cache_shape = attn_backend.get_kv_cache_shape(
kernel_num_blocks,
shape_block_size,
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype_str=self.cache_config.cache_dtype,
cache_dtype_str=layer_cache_dtype_str,
)
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()