diff --git a/tests/v1/kv_connector/unit/test_offloading_connector.py b/tests/v1/kv_connector/unit/test_offloading_connector.py index a323ae7ce5..5a6ea10a68 100644 --- a/tests/v1/kv_connector/unit/test_offloading_connector.py +++ b/tests/v1/kv_connector/unit/test_offloading_connector.py @@ -542,10 +542,13 @@ def test_fs_tiering_offloading(tmp_path) -> None: [ # ("Qwen/Qwen3.6-35B-A3B", 1056, 2), # ("tiiuae/falcon-mamba-7b", 16, 1), - ("state-spaces/mamba-1.4b-hf", 16, 1) + ("state-spaces/mamba-1.4b-hf", 16, 1), ], ) -def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int): +@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"]) +def test_mamba_cpu_offload_boundary( + model: str, block_size: int, tp_size: int, mamba_cache_mode: str +): kv_transfer_config = KVTransferConfig( kv_connector="OffloadingConnector", kv_role="kv_both", @@ -562,7 +565,11 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int): kv_transfer_config=kv_transfer_config, language_model_only=True, enable_prefix_caching=True, - mamba_cache_mode="align", + mamba_cache_mode=mamba_cache_mode, + # Use lossless state storage so this exact-equality regression isolates + # offload boundary selection from low-precision Mamba checkpointing. + mamba_cache_dtype="float32", + mamba_ssm_cache_dtype="float32", disable_hybrid_kv_cache_manager=False, ) @@ -599,12 +606,10 @@ def test_mamba_align_cpu_offload(model: str, block_size: int, tp_size: int): ) try: - # Mamba has only a single state. The CPU cache stores are triggered - # at offload block boundaries. When the prompt is exactly at the boundary, - # The CPU offload should not load the cached block. - # This is because we'd use that state to recompute the last token. This - # does not work for mamba as there is only one KV value and that is for - # for the token at the boundary. + # A cached Mamba entry is a point state. CPU cache stores are triggered + # at offload block boundaries. When the prompt is exactly at a boundary, + # CPU offload must not load that boundary's cached state because it already + # includes the last token that must be recomputed. # This is fine for other attention types as we have all the necessary # token KV values in the hit blocks. prompt = TokensPrompt(prompt_token_ids=initial_ids) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 8abd0aff40..8bc4451967 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -146,14 +146,17 @@ def resolve_mamba_align_size( """Scan all KV cache groups in *spec* and return the single mamba alignment size, or None if no group requires mamba alignment. - For MambaSpec groups in "align" cache mode the hit window must be rounded - down to a multiple of the offloaded chunk size. Asserts that all such - groups agree on the same value. + For MambaSpec groups in "align" or "all" cache mode the hit window must be + rounded down to a multiple of the offloaded chunk size. Asserts that all + such groups agree on the same value. """ mamba_align_size: int | None = None for idx, tokens_per_block in enumerate(spec.tokens_per_block): kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec - if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align": + if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in ( + "align", + "all", + ): tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk assert mamba_align_size is None or mamba_align_size == tokens_per_chunk mamba_align_size = tokens_per_chunk