Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+10
-24
@@ -59,7 +59,7 @@ CacheDType = Literal[
|
||||
|
||||
|
||||
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
|
||||
MambaCacheMode = Literal["all", "align", "none"]
|
||||
MambaCacheMode = Literal["align", "none"]
|
||||
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
|
||||
KVOffloadingBackend = Literal["native", "lmcache"]
|
||||
|
||||
@@ -75,8 +75,6 @@ class CacheConfig:
|
||||
Accepts None (meaning "use default"). After construction, always int."""
|
||||
user_specified_block_size: bool = field(default=False, init=False)
|
||||
"""Whether block_size was explicitly provided. Derived automatically."""
|
||||
user_specified_mamba_block_size: bool = field(default=False, init=False)
|
||||
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
|
||||
kv_cache_layout: str | None = field(default=None, init=False)
|
||||
"""Resolved physical KV cache layout name (a ``KVCacheLayout`` member).
|
||||
|
||||
@@ -191,7 +189,6 @@ class CacheConfig:
|
||||
"""The cache strategy for Mamba layers:
|
||||
|
||||
- "none": set when prefix caching is disabled.
|
||||
- "all": cache the mamba state of all tokens at position i * block_size.
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size. This is the default when prefix
|
||||
caching is enabled.
|
||||
@@ -204,15 +201,16 @@ class CacheConfig:
|
||||
Mamba block size."""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM logical history length B for Mamba2. Triton uses B physical
|
||||
rows and FlashInfer uses B+1. Kimi-K3 speculative decode does not use B.
|
||||
Default 16."""
|
||||
rows and FlashInfer uses B+T, where T is the target verification length.
|
||||
Kimi-K3 speculative decode does not use B. Default 16."""
|
||||
use_replayssm: bool = False
|
||||
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
|
||||
the per-step full-state store, writing the checkpoint back only on flush.
|
||||
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
|
||||
or FlashInfer mamba backend; standard (non-speculative) decode only. In align
|
||||
mode flushes are most efficient when mamba_block_size is a multiple of
|
||||
replayssm_buffer_len, but this is not required."""
|
||||
or FlashInfer mamba backend. Mamba2 speculative decode requires FlashInfer
|
||||
and mamba_cache_mode 'none'. In align mode flushes are most efficient when
|
||||
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
|
||||
required."""
|
||||
use_recoverssm: bool = field(default=False, init=False)
|
||||
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
|
||||
|
||||
@@ -242,7 +240,9 @@ class CacheConfig:
|
||||
swa_bounded_replay: bool = True
|
||||
"""Keep the sliding-window KV of models that support it (DeepSeek-V4.1)
|
||||
out of prefix caching and rebuild it after a prefix hit by recomputing the
|
||||
hit's last window. Requires model runner V2."""
|
||||
hit's last window. The layers past the last KV-source layer then also run
|
||||
eager prefill steps on each request's trailing window only. Requires model
|
||||
runner V2."""
|
||||
|
||||
kv_cache_memory_bytes: int | None = None
|
||||
"""Size of KV Cache per GPU in bytes. By default, this is set to None
|
||||
@@ -290,7 +290,6 @@ class CacheConfig:
|
||||
"mamba_page_size_padded",
|
||||
"skip_page_size_padded",
|
||||
"user_specified_block_size",
|
||||
"user_specified_mamba_block_size",
|
||||
"_block_size_resolved",
|
||||
# Post-init/derived counters
|
||||
"num_gpu_blocks",
|
||||
@@ -334,21 +333,8 @@ class CacheConfig:
|
||||
self.block_size = self.DEFAULT_BLOCK_SIZE
|
||||
else:
|
||||
self.user_specified_block_size = True
|
||||
if self.mamba_block_size is not None:
|
||||
self.user_specified_mamba_block_size = True
|
||||
return self
|
||||
|
||||
@field_validator("mamba_cache_mode", mode="after")
|
||||
@classmethod
|
||||
def _validate_mamba_cache_mode(cls, mode: MambaCacheMode) -> MambaCacheMode:
|
||||
if mode == "all":
|
||||
logger.warning_once(
|
||||
"Mamba cache mode 'all' is deprecated and will be removed in an "
|
||||
"upcoming release. If this is a problem, please open an issue "
|
||||
"at https://github.com/vllm-project/vllm/issues."
|
||||
)
|
||||
return mode
|
||||
|
||||
@field_validator("cache_dtype", mode="after")
|
||||
@classmethod
|
||||
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
|
||||
|
||||
Reference in New Issue
Block a user