Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3b6d97aa
commit
67ff0ae30f
@@ -150,6 +150,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
@@ -7147,12 +7148,19 @@ class GPUModelRunner(
|
||||
else:
|
||||
shape_block_size = kernel_block_size
|
||||
|
||||
# Skipped layers (--kv-cache-dtype-skip-layers) need
|
||||
# the unquantized shape.
|
||||
layer_cache_dtype_str = (
|
||||
"auto"
|
||||
if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE
|
||||
else self.cache_config.cache_dtype
|
||||
)
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
kernel_num_blocks,
|
||||
shape_block_size,
|
||||
kv_cache_spec.num_kv_heads,
|
||||
kv_cache_spec.head_size,
|
||||
cache_dtype_str=self.cache_config.cache_dtype,
|
||||
cache_dtype_str=layer_cache_dtype_str,
|
||||
)
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
|
||||
Reference in New Issue
Block a user