Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
sychen52
2026-07-04 02:29:13 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab3b6d97aa
commit 67ff0ae30f
6 changed files with 228 additions and 5 deletions
+6
View File
@@ -119,6 +119,11 @@ class CacheConfig:
mamba_page_size_padded: int | None = None
""" Optional override for mamba page size; used by hybrid mamba/attention
models to ensure exact alignment with attention page size."""
skip_page_size_padded: int | None = None
"""Optional override for the page size of layers skipped from KV cache
quantization (``--kv-cache-dtype-skip-layers``); set during block-size
alignment so unquantized skip layers pad up to the quantized primary's
page."""
mamba_block_size: int | None = Field(default=None, gt=0)
"""Size of a contiguous cache block in number of tokens for mamba cache.
Can be set only when prefix caching is enabled.
@@ -207,6 +212,7 @@ class CacheConfig:
# Prefix-caching implementation detail (doesn't affect compiled graph).
"hash_block_size",
"mamba_page_size_padded",
"skip_page_size_padded",
"user_specified_block_size",
"user_specified_mamba_block_size",
"_block_size_resolved",