[Bugfix] Fix Mamba all-mode CPU offload boundary alignment (#51100)

Signed-off-by: qianxu <admin@picasso-lab.com>
Co-authored-by: qianxu <admin@picasso-lab.com>
This commit is contained in:
Qianxu Wang
2026-08-07 07:37:48 +03:00
committed by GitHub
co-authored by qianxu
parent dd856e48bb
commit c810e5ee99
2 changed files with 21 additions and 13 deletions
@@ -542,10 +542,13 @@ def test_fs_tiering_offloading(tmp_path) -> None:
[
# ("Qwen/Qwen3.6-35B-A3B", 1056, 2),
# ("tiiuae/falcon-mamba-7b", 16, 1),
("state-spaces/mamba-1.4b-hf", 16, 1)
("state-spaces/mamba-1.4b-hf", 16, 1),
],
)
def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
def test_mamba_cpu_offload_boundary(
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
):
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
@@ -562,7 +565,11 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
kv_transfer_config=kv_transfer_config,
language_model_only=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
mamba_cache_mode=mamba_cache_mode,
# Use lossless state storage so this exact-equality regression isolates
# offload boundary selection from low-precision Mamba checkpointing.
mamba_cache_dtype="float32",
mamba_ssm_cache_dtype="float32",
disable_hybrid_kv_cache_manager=False,
)
@@ -599,12 +606,10 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
)
try:
# Mamba has only a single state. The CPU cache stores are triggered
# at offload block boundaries. When the prompt is exactly at the boundary,
# The CPU offload should not load the cached block.
# This is because we'd use that state to recompute the last token. This
# does not work for mamba as there is only one KV value and that is for
# for the token at the boundary.
# A cached Mamba entry is a point state. CPU cache stores are triggered
# at offload block boundaries. When the prompt is exactly at a boundary,
# CPU offload must not load that boundary's cached state because it already
# includes the last token that must be recomputed.
# This is fine for other attention types as we have all the necessary
# token KV values in the hit blocks.
prompt = TokensPrompt(prompt_token_ids=initial_ids)
@@ -146,14 +146,17 @@ def resolve_mamba_align_size(
"""Scan all KV cache groups in *spec* and return the single mamba alignment
size, or None if no group requires mamba alignment.
For MambaSpec groups in "align" cache mode the hit window must be rounded
down to a multiple of the offloaded chunk size. Asserts that all such
groups agree on the same value.
For MambaSpec groups in "align" or "all" cache mode the hit window must be
rounded down to a multiple of the offloaded chunk size. Asserts that all
such groups agree on the same value.
"""
mamba_align_size: int | None = None
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in (
"align",
"all",
):
tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
mamba_align_size = tokens_per_chunk