Support nvfp4 kv with kv-cache-dtype-skip-layers sliding_window (#42890)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3b6d97aa
commit
67ff0ae30f
@@ -119,6 +119,11 @@ class CacheConfig:
|
||||
mamba_page_size_padded: int | None = None
|
||||
""" Optional override for mamba page size; used by hybrid mamba/attention
|
||||
models to ensure exact alignment with attention page size."""
|
||||
skip_page_size_padded: int | None = None
|
||||
"""Optional override for the page size of layers skipped from KV cache
|
||||
quantization (``--kv-cache-dtype-skip-layers``); set during block-size
|
||||
alignment so unquantized skip layers pad up to the quantized primary's
|
||||
page."""
|
||||
mamba_block_size: int | None = Field(default=None, gt=0)
|
||||
"""Size of a contiguous cache block in number of tokens for mamba cache.
|
||||
Can be set only when prefix caching is enabled.
|
||||
@@ -207,6 +212,7 @@ class CacheConfig:
|
||||
# Prefix-caching implementation detail (doesn't affect compiled graph).
|
||||
"hash_block_size",
|
||||
"mamba_page_size_padded",
|
||||
"skip_page_size_padded",
|
||||
"user_specified_block_size",
|
||||
"user_specified_mamba_block_size",
|
||||
"_block_size_resolved",
|
||||
|
||||
Reference in New Issue
Block a user