[Feat][Mamba2] Enable internal prefill checkpoints (#57329)
Signed-off-by: Roi Koren <roik@nvidia.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -24,6 +24,10 @@ def server():
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
"2",
|
||||
# Streaming and non-streaming runs must match bitwise; a prefix-cache
|
||||
# hit resumes the second from a mid-prefill Mamba checkpoint, whose
|
||||
# numerics differ slightly from the cold single-pass prefill.
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
|
||||
yield server
|
||||
|
||||
Reference in New Issue
Block a user