[K3] support recoverssm for K3 (#51855)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f27ae25473
commit
70afdedc10
@@ -144,9 +144,8 @@ class CacheConfig:
|
||||
caching is enabled.
|
||||
"""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM history buffer length B: with use_replayssm, standard decode
|
||||
caches recent SSM inputs in a size-B ring buffer and flushes the checkpoint
|
||||
state to HBM every B steps. Default 16."""
|
||||
"""ReplaySSM history buffer length B for standard Mamba2 decode. Kimi-K3
|
||||
speculative decoding does not use B. Default 16."""
|
||||
use_replayssm: bool = False
|
||||
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
|
||||
the per-step full-state store, writing the checkpoint back only on flush.
|
||||
@@ -154,6 +153,8 @@ class CacheConfig:
|
||||
mamba backend; standard (non-speculative) decode only. In align mode flushes
|
||||
are most efficient when mamba_block_size is a multiple of replayssm_buffer_len,
|
||||
but this is not required."""
|
||||
use_kda_recoverssm: bool = field(default=False, init=False)
|
||||
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
|
||||
|
||||
# Will be set after profiling.
|
||||
num_gpu_blocks: int | None = field(default=None, init=False)
|
||||
|
||||
Reference in New Issue
Block a user