Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+14
-15
@@ -121,6 +121,7 @@ class CacheConfig:
|
||||
|
||||
cache_dtype: CacheDType = "auto"
|
||||
"""Data type for kv cache storage. If "auto", will use model data type.
|
||||
An explicit float16 or bfloat16 must match the model data type.
|
||||
CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
|
||||
fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc).
|
||||
Some models (namely DeepSeekV3.2) default to fp8, set to bfloat16 to use
|
||||
@@ -176,8 +177,7 @@ class CacheConfig:
|
||||
page."""
|
||||
mamba_block_size: int | None = Field(default=None, gt=0)
|
||||
"""Size of a contiguous cache block in number of tokens for mamba cache.
|
||||
Can be set only when prefix caching is enabled.
|
||||
Value must be a multiple of 8 to align with causal_conv1d kernel."""
|
||||
Can be set only when prefix caching is enabled."""
|
||||
mamba_cache_dtype: MambaDType = "auto"
|
||||
"""The data type to use for the Mamba cache (both the conv as well as the
|
||||
ssm state). If set to 'auto', the data type will be inferred from the model
|
||||
@@ -187,19 +187,19 @@ class CacheConfig:
|
||||
still be controlled by mamba_cache_dtype). If set to 'auto', the data type
|
||||
for the ssm state will be determined by mamba_cache_dtype."""
|
||||
mamba_cache_mode: MambaCacheMode = "none"
|
||||
"""The cache strategy for Mamba layers:
|
||||
"""The cache strategy for Mamba layers, derived from `enable_prefix_caching`:
|
||||
|
||||
- "none": set when prefix caching is disabled.
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size. This is the default when prefix
|
||||
caching is enabled.
|
||||
- "none": prefix caching is disabled.
|
||||
- "align": prefix caching is enabled. Only cache the mamba state of the last
|
||||
token of each scheduler step and when the token is at position
|
||||
i * block_size.
|
||||
"""
|
||||
enable_mamba_shared_prefix_checkpoint: bool = False
|
||||
"""Also register a Mamba "align" checkpoint at the shared-prefix junction --
|
||||
where an EAGLE/MTP sibling was observed to resume -- instead of only at the
|
||||
prompt tail. Off by default; only takes effect with `mamba_cache_mode`
|
||||
"align", EAGLE on the Mamba group, and a prefix match unit smaller than the
|
||||
Mamba block size."""
|
||||
prompt tail. Off by default; only takes effect with prefix caching enabled,
|
||||
EAGLE on the Mamba group, and a prefix match unit smaller than the Mamba
|
||||
block size."""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM logical history length B for Mamba2. Triton uses B physical
|
||||
rows and FlashInfer uses B+T, where T is the target verification length.
|
||||
@@ -207,11 +207,10 @@ class CacheConfig:
|
||||
use_replayssm: bool = False
|
||||
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
|
||||
the per-step full-state store, writing the checkpoint back only on flush.
|
||||
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
|
||||
or FlashInfer mamba backend. Mamba2 speculative decode requires FlashInfer
|
||||
and mamba_cache_mode 'none'. In align mode flushes are most efficient when
|
||||
mamba_block_size is a multiple of replayssm_buffer_len, but this is not
|
||||
required."""
|
||||
Requires the Triton or FlashInfer mamba backend; FlashInfer does not support
|
||||
prefix caching yet. Mamba2 speculative decode requires FlashInfer. With
|
||||
prefix caching, flushes are most efficient when mamba_block_size is a
|
||||
multiple of replayssm_buffer_len, but this is not required."""
|
||||
use_recoverssm: bool = field(default=False, init=False)
|
||||
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user