[Bugfix] Fix Mamba all-mode CPU offload boundary alignment (#51100)
Signed-off-by: qianxu <admin@picasso-lab.com> Co-authored-by: qianxu <admin@picasso-lab.com>
This commit is contained in:
@@ -542,10 +542,13 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
[
|
||||
# ("Qwen/Qwen3.6-35B-A3B", 1056, 2),
|
||||
# ("tiiuae/falcon-mamba-7b", 16, 1),
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1)
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1),
|
||||
],
|
||||
)
|
||||
def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
|
||||
def test_mamba_cpu_offload_boundary(
|
||||
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
|
||||
):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
@@ -562,7 +565,11 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
language_model_only=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
# Use lossless state storage so this exact-equality regression isolates
|
||||
# offload boundary selection from low-precision Mamba checkpointing.
|
||||
mamba_cache_dtype="float32",
|
||||
mamba_ssm_cache_dtype="float32",
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
|
||||
@@ -599,12 +606,10 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
)
|
||||
|
||||
try:
|
||||
# Mamba has only a single state. The CPU cache stores are triggered
|
||||
# at offload block boundaries. When the prompt is exactly at the boundary,
|
||||
# The CPU offload should not load the cached block.
|
||||
# This is because we'd use that state to recompute the last token. This
|
||||
# does not work for mamba as there is only one KV value and that is for
|
||||
# for the token at the boundary.
|
||||
# A cached Mamba entry is a point state. CPU cache stores are triggered
|
||||
# at offload block boundaries. When the prompt is exactly at a boundary,
|
||||
# CPU offload must not load that boundary's cached state because it already
|
||||
# includes the last token that must be recomputed.
|
||||
# This is fine for other attention types as we have all the necessary
|
||||
# token KV values in the hit blocks.
|
||||
prompt = TokensPrompt(prompt_token_ids=initial_ids)
|
||||
|
||||
Reference in New Issue
Block a user