Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
+14 -15
View File
@@ -121,6 +121,7 @@ class CacheConfig:
cache_dtype: CacheDType = "auto"
"""Data type for kv cache storage. If "auto", will use model data type.
An explicit float16 or bfloat16 must match the model data type.
CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc).
Some models (namely DeepSeekV3.2) default to fp8, set to bfloat16 to use
@@ -176,8 +177,7 @@ class CacheConfig:
page."""
mamba_block_size: int | None = Field(default=None, gt=0)
"""Size of a contiguous cache block in number of tokens for mamba cache.
Can be set only when prefix caching is enabled.
Value must be a multiple of 8 to align with causal_conv1d kernel."""
Can be set only when prefix caching is enabled."""
mamba_cache_dtype: MambaDType = "auto"
"""The data type to use for the Mamba cache (both the conv as well as the
ssm state). If set to 'auto', the data type will be inferred from the model
@@ -187,19 +187,19 @@ class CacheConfig:
still be controlled by mamba_cache_dtype). If set to 'auto', the data type
for the ssm state will be determined by mamba_cache_dtype."""
mamba_cache_mode: MambaCacheMode = "none"
"""The cache strategy for Mamba layers:
"""The cache strategy for Mamba layers, derived from `enable_prefix_caching`:
- "none": set when prefix caching is disabled.
- "align": only cache the mamba state of the last token of each scheduler step and
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
- "none": prefix caching is disabled.
- "align": prefix caching is enabled. Only cache the mamba state of the last
token of each scheduler step and when the token is at position
i * block_size.
"""
enable_mamba_shared_prefix_checkpoint: bool = False
"""Also register a Mamba "align" checkpoint at the shared-prefix junction --
where an EAGLE/MTP sibling was observed to resume -- instead of only at the
prompt tail. Off by default; only takes effect with `mamba_cache_mode`
"align", EAGLE on the Mamba group, and a prefix match unit smaller than the
Mamba block size."""
prompt tail. Off by default; only takes effect with prefix caching enabled,
EAGLE on the Mamba group, and a prefix match unit smaller than the Mamba
block size."""
replayssm_buffer_len: int = Field(default=16, gt=0)
"""ReplaySSM logical history length B for Mamba2. Triton uses B physical
rows and FlashInfer uses B+T, where T is the target verification length.
@@ -207,11 +207,10 @@ class CacheConfig:
use_replayssm: bool = False
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
the per-step full-state store, writing the checkpoint back only on flush.
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
or FlashInfer mamba backend. Mamba2 speculative decode requires FlashInfer
and mamba_cache_mode 'none'. In align mode flushes are most efficient when
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
required."""
Requires the Triton or FlashInfer mamba backend; FlashInfer does not support
prefix caching yet. Mamba2 speculative decode requires FlashInfer. With
prefix caching, flushes are most efficient when mamba_block_size is a
multiple of replayssm_buffer_len, but this is not required."""
use_recoverssm: bool = field(default=False, init=False)
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""