[Feat][Mamba2] Enable internal prefill checkpoints (#57329)

Signed-off-by: Roi Koren <roik@nvidia.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
roikoren755
2026-10-06 07:53:51 +00:00
committed by GitHub
co-authored by zjy0516
parent edfccc9469
commit e11962fc1c
8 changed files with 586 additions and 23 deletions
@@ -24,6 +24,10 @@ def server():
"4096",
"--max-num-seqs",
"2",
# Streaming and non-streaming runs must match bitwise; a prefix-cache
# hit resumes the second from a mid-prefill Mamba checkpoint, whose
# numerics differ slightly from the cold single-pass prefill.
"--no-enable-prefix-caching",
]
with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
yield server