Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
+10 -24
View File
@@ -59,7 +59,7 @@ CacheDType = Literal[
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
MambaCacheMode = Literal["all", "align", "none"]
MambaCacheMode = Literal["align", "none"]
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
KVOffloadingBackend = Literal["native", "lmcache"]
@@ -75,8 +75,6 @@ class CacheConfig:
Accepts None (meaning "use default"). After construction, always int."""
user_specified_block_size: bool = field(default=False, init=False)
"""Whether block_size was explicitly provided. Derived automatically."""
user_specified_mamba_block_size: bool = field(default=False, init=False)
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
kv_cache_layout: str | None = field(default=None, init=False)
"""Resolved physical KV cache layout name (a ``KVCacheLayout`` member).
@@ -191,7 +189,6 @@ class CacheConfig:
"""The cache strategy for Mamba layers:
- "none": set when prefix caching is disabled.
- "all": cache the mamba state of all tokens at position i * block_size.
- "align": only cache the mamba state of the last token of each scheduler step and
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
@@ -204,15 +201,16 @@ class CacheConfig:
Mamba block size."""
replayssm_buffer_len: int = Field(default=16, gt=0)
"""ReplaySSM logical history length B for Mamba2. Triton uses B physical
rows and FlashInfer uses B+1. Kimi-K3 speculative decode does not use B.
Default 16."""
rows and FlashInfer uses B+T, where T is the target verification length.
Kimi-K3 speculative decode does not use B. Default 16."""
use_replayssm: bool = False
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
the per-step full-state store, writing the checkpoint back only on flush.
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
or FlashInfer mamba backend; standard (non-speculative) decode only. In align
mode flushes are most efficient when mamba_block_size is a multiple of
replayssm_buffer_len, but this is not required."""
or FlashInfer mamba backend. Mamba2 speculative decode requires FlashInfer
and mamba_cache_mode 'none'. In align mode flushes are most efficient when
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
required."""
use_recoverssm: bool = field(default=False, init=False)
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
@@ -242,7 +240,9 @@ class CacheConfig:
swa_bounded_replay: bool = True
"""Keep the sliding-window KV of models that support it (DeepSeek-V4.1)
out of prefix caching and rebuild it after a prefix hit by recomputing the
hit's last window. Requires model runner V2."""
hit's last window. The layers past the last KV-source layer then also run
eager prefill steps on each request's trailing window only. Requires model
runner V2."""
kv_cache_memory_bytes: int | None = None
"""Size of KV Cache per GPU in bytes. By default, this is set to None
@@ -290,7 +290,6 @@ class CacheConfig:
"mamba_page_size_padded",
"skip_page_size_padded",
"user_specified_block_size",
"user_specified_mamba_block_size",
"_block_size_resolved",
# Post-init/derived counters
"num_gpu_blocks",
@@ -334,21 +333,8 @@ class CacheConfig:
self.block_size = self.DEFAULT_BLOCK_SIZE
else:
self.user_specified_block_size = True
if self.mamba_block_size is not None:
self.user_specified_mamba_block_size = True
return self
@field_validator("mamba_cache_mode", mode="after")
@classmethod
def _validate_mamba_cache_mode(cls, mode: MambaCacheMode) -> MambaCacheMode:
if mode == "all":
logger.warning_once(
"Mamba cache mode 'all' is deprecated and will be removed in an "
"upcoming release. If this is a problem, please open an issue "
"at https://github.com/vllm-project/vllm/issues."
)
return mode
@field_validator("cache_dtype", mode="after")
@classmethod
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType: