[Bugfix] Fix Mamba all-mode CPU offload boundary alignment (#51100)
Signed-off-by: qianxu <admin@picasso-lab.com> Co-authored-by: qianxu <admin@picasso-lab.com>
This commit is contained in:
@@ -542,10 +542,13 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
[
|
||||
# ("Qwen/Qwen3.6-35B-A3B", 1056, 2),
|
||||
# ("tiiuae/falcon-mamba-7b", 16, 1),
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1)
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1),
|
||||
],
|
||||
)
|
||||
def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
|
||||
def test_mamba_cpu_offload_boundary(
|
||||
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
|
||||
):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
@@ -562,7 +565,11 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
language_model_only=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
# Use lossless state storage so this exact-equality regression isolates
|
||||
# offload boundary selection from low-precision Mamba checkpointing.
|
||||
mamba_cache_dtype="float32",
|
||||
mamba_ssm_cache_dtype="float32",
|
||||
disable_hybrid_kv_cache_manager=False,
|
||||
)
|
||||
|
||||
@@ -599,12 +606,10 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int):
|
||||
)
|
||||
|
||||
try:
|
||||
# Mamba has only a single state. The CPU cache stores are triggered
|
||||
# at offload block boundaries. When the prompt is exactly at the boundary,
|
||||
# The CPU offload should not load the cached block.
|
||||
# This is because we'd use that state to recompute the last token. This
|
||||
# does not work for mamba as there is only one KV value and that is for
|
||||
# for the token at the boundary.
|
||||
# A cached Mamba entry is a point state. CPU cache stores are triggered
|
||||
# at offload block boundaries. When the prompt is exactly at a boundary,
|
||||
# CPU offload must not load that boundary's cached state because it already
|
||||
# includes the last token that must be recomputed.
|
||||
# This is fine for other attention types as we have all the necessary
|
||||
# token KV values in the hit blocks.
|
||||
prompt = TokensPrompt(prompt_token_ids=initial_ids)
|
||||
|
||||
@@ -146,14 +146,17 @@ def resolve_mamba_align_size(
|
||||
"""Scan all KV cache groups in *spec* and return the single mamba alignment
|
||||
size, or None if no group requires mamba alignment.
|
||||
|
||||
For MambaSpec groups in "align" cache mode the hit window must be rounded
|
||||
down to a multiple of the offloaded chunk size. Asserts that all such
|
||||
groups agree on the same value.
|
||||
For MambaSpec groups in "align" or "all" cache mode the hit window must be
|
||||
rounded down to a multiple of the offloaded chunk size. Asserts that all
|
||||
such groups agree on the same value.
|
||||
"""
|
||||
mamba_align_size: int | None = None
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in (
|
||||
"align",
|
||||
"all",
|
||||
):
|
||||
tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
|
||||
assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
|
||||
mamba_align_size = tokens_per_chunk
|
||||
|
||||
Reference in New Issue
Block a user