Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3b6d97aa
commit
67ff0ae30f
@@ -23,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
@@ -300,12 +301,20 @@ def _reshape_kv_cache(
|
||||
kv_cache_spec.storage_block_size // kernel_block_size
|
||||
)
|
||||
kernel_num_blocks = num_blocks * num_blocks_per_kv_block
|
||||
# Skipped layers (--kv-cache-dtype-skip-layers) keep the
|
||||
# unquantized shape; only the quantized primary uses the
|
||||
# quantized cache dtype's (possibly packed) layout.
|
||||
layer_cache_dtype = (
|
||||
"auto"
|
||||
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
|
||||
else cache_dtype
|
||||
)
|
||||
kv_cache_shape = group.backend.get_kv_cache_shape(
|
||||
kernel_num_blocks,
|
||||
kernel_block_size,
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=cache_dtype,
|
||||
cache_dtype_str=layer_cache_dtype,
|
||||
)
|
||||
|
||||
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
|
||||
@@ -377,11 +386,18 @@ def _update_hybrid_attention_layout(
|
||||
kv_cache_spec = group.kv_cache_spec
|
||||
if not isinstance(kv_cache_spec, AttentionSpec):
|
||||
continue
|
||||
# Mirror the per-layer dtype selection used when building the shape
|
||||
# above. The block-dim index is dtype-independent for current backends
|
||||
# (quantization only changes the last dim), so this is a no-op today,
|
||||
# but it keeps both call sites consistent for skip layers.
|
||||
layer_cache_dtype = (
|
||||
"auto" if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE else cache_dtype
|
||||
)
|
||||
block_dim = group.backend.get_kv_cache_block_dim(
|
||||
kernel_block_sizes[group.kv_cache_group_id],
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=cache_dtype,
|
||||
cache_dtype_str=layer_cache_dtype,
|
||||
)
|
||||
# if the first dim of the kvcache's layout is already num_blocks, continue
|
||||
if block_dim == 0:
|
||||
|
||||
Reference in New Issue
Block a user