[Core] Remove deprecated mamba_cache_mode "all" (#58997)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
Jiangyun Zhu
2026-10-01 02:59:37 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Thomas Parnell
parent 08d77cad7b
commit cb6458c5ce
45 changed files with 201 additions and 1476 deletions
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
num_accepted_tokens=None,
query_start_loc_d=None,
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([5]),
)
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
state_indices_tensor_d=state_indices_tensor_d,
num_accepted_tokens=None,
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([1, 1]),
)
-5
View File
@@ -568,15 +568,10 @@ def test_replayssm_mixer_selects_mtp_layout(
num_reqs=2,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
+6 -3
View File
@@ -270,9 +270,6 @@ def test_kda_recoverssm_derivation_is_revalidated():
with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"):
VllmConfig.validate_mamba_cached_kernel(config)
config.use_v2_model_runner = True
config.cache_config.mamba_cache_mode = "all"
with pytest.raises(ValueError, match="only none and align"):
VllmConfig.validate_mamba_cached_kernel(config)
config.cache_config.mamba_cache_mode = "none"
config.model_config.architecture = "NemotronHForCausalLM"
@@ -286,6 +283,12 @@ def test_kda_recoverssm_derivation_is_revalidated():
VllmConfig.validate_mamba_cached_kernel(config)
def test_mamba_cache_mode_all_is_rejected():
"""The removed 'all' mode must fail validation instead of being ignored."""
with pytest.raises(ValidationError, match="mamba_cache_mode"):
CacheConfig(mamba_cache_mode="all")
def test_per_request_spec_decode_metrics_requires_spec_decode():
# The flag only makes sense with speculative decoding configured; enabling
# it without --speculative-config should fail fast rather than silently
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for BaseMambaAttentionMetadataBuilder batch classification."""
from types import SimpleNamespace
from tests.v1.attention.utils import MockMambaBuilder
from vllm.config.compilation import CUDAGraphMode
def _make_vllm_config(
max_model_len: int,
max_num_seqs: int,
num_speculative_tokens: int = 0,
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection."""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=False,
)
if num_speculative_tokens > 0
else None
)
return SimpleNamespace(
cache_config=SimpleNamespace(
block_size=block_size,
mamba_cache_mode="none",
use_replayssm=False,
replayssm_buffer_len=16,
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.FULL,
max_cudagraph_capture_size=None,
),
speculative_config=speculative_config,
num_speculative_tokens=num_speculative_tokens,
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
model_config=SimpleNamespace(max_model_len=max_model_len),
)
def test_mamba_single_token_prompt_runs_as_prefill():
seq_lens = [8, 9, 1]
config = _make_vllm_config(256, len(seq_lens), block_size=16)
metadata = MockMambaBuilder.build_mamba_metadata(
config,
seq_lens=seq_lens,
query_lens=[1] * len(seq_lens),
is_prefilling=[False, False, True],
)
assert metadata.num_decodes == 2
assert metadata.num_prefills == 1
assert metadata.has_initial_states_p.tolist() == [False]
@@ -1,466 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
When multiple KV cache groups share the same MambaSpec (as in Nemotron
hybrid models), the metadata caching optimization reuses metadata from
an earlier group via update_block_table(). In 'all' mode with CUDA graphs,
update_block_table() must copy block_idx_last_scheduled_token and
block_idx_last_computed_token to the *current* builder's persistent
buffers, otherwise CUDA graph replay reads stale values from uninitialized
buffers.
"""
from types import SimpleNamespace
import torch
from tests.v1.attention.utils import MockMambaBuilder
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadata
from vllm.v1.kv_cache_interface import MambaSpec
def _make_vllm_config(
max_model_len: int,
max_num_seqs: int,
num_speculative_tokens: int = 0,
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection."""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=False,
)
if num_speculative_tokens > 0
else None
)
return SimpleNamespace(
cache_config=SimpleNamespace(
block_size=block_size,
mamba_cache_mode="all",
use_replayssm=False,
replayssm_buffer_len=16,
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.FULL,
max_cudagraph_capture_size=None,
),
speculative_config=speculative_config,
num_speculative_tokens=num_speculative_tokens,
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
model_config=SimpleNamespace(max_model_len=max_model_len),
)
def test_mamba_single_token_prompt_runs_as_prefill():
seq_lens = [8, 9, 1]
config = _make_vllm_config(256, len(seq_lens), block_size=16)
metadata = MockMambaBuilder.build_mamba_metadata(
config,
seq_lens=seq_lens,
query_lens=[1] * len(seq_lens),
is_prefilling=[False, False, True],
)
assert metadata.num_decodes == 2
assert metadata.num_prefills == 1
assert metadata.has_initial_states_p.tolist() == [False]
def test_update_block_table_copies_block_idx_to_persistent_buffers():
"""update_block_table() must write block_idx tensors to the current
builder's persistent buffers, not leave them pointing to a different
builder's buffers."""
block_size = 16
max_model_len = 256
num_reqs = 4
device = torch.device("cpu")
vllm_config = _make_vllm_config(max_model_len, num_reqs)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
)
# Two builders simulating two KV cache groups with the same MambaSpec.
builder_a = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
builder_b = MockMambaBuilder(spec, ["layer1"], vllm_config, device)
# Sanity: each builder has its own persistent buffer.
assert (
builder_a.block_idx_last_scheduled_token.data_ptr()
!= builder_b.block_idx_last_scheduled_token.data_ptr()
)
# Construct decode-only metadata as if builder_a.build() produced it.
max_blocks = max_model_len // block_size
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
block_idx_vals = (seq_lens - 1) // block_size # [3, 3, 3, 3]
builder_a.block_idx_last_scheduled_token[:num_reqs].copy_(block_idx_vals)
builder_a.block_idx_last_computed_token[:num_reqs].copy_(block_idx_vals)
metadata_a = BaseMambaAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_reqs,
num_decode_tokens=num_reqs,
num_reqs=num_reqs,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
query_start_loc_d=None,
num_accepted_tokens=None,
state_indices_tensor_d=builder_a.state_indices_tensor_d[:num_reqs],
block_idx_last_scheduled_token=(
builder_a.block_idx_last_scheduled_token[:num_reqs]
),
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=(
builder_a.block_idx_last_computed_token[:num_reqs]
),
block_idx_last_scheduled_token_prev_step=None,
seq_lens=seq_lens,
)
# Call update_block_table on builder_b (simulates the metadata caching
# optimization reusing metadata from builder_a's group).
blk_table = torch.randint(
0, 100, (num_reqs, max_blocks), dtype=torch.int32, device=device
)
slot_mapping = torch.zeros(num_reqs, dtype=torch.int64, device=device)
metadata_b = builder_b.update_block_table(metadata_a, blk_table, slot_mapping)
# block_idx tensors must live in builder_b's persistent buffers.
def shares_storage(tensor, buffer):
return (
tensor.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr()
)
assert shares_storage(
metadata_b.block_idx_last_scheduled_token,
builder_b.block_idx_last_scheduled_token,
), "block_idx_last_scheduled_token not in builder_b's persistent buffer"
assert shares_storage(
metadata_b.block_idx_last_computed_token,
builder_b.block_idx_last_computed_token,
), "block_idx_last_computed_token not in builder_b's persistent buffer"
# Must NOT point to builder_a's buffers.
assert not shares_storage(
metadata_b.block_idx_last_scheduled_token,
builder_a.block_idx_last_scheduled_token,
), "block_idx_last_scheduled_token still points to builder_a's buffer"
# Values must be correct (copied from metadata_a).
torch.testing.assert_close(
metadata_b.block_idx_last_scheduled_token,
block_idx_vals,
)
torch.testing.assert_close(
metadata_b.block_idx_last_computed_token,
block_idx_vals,
)
def test_state_indices_tensor_d_includes_num_speculative_blocks():
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
the cudagraph buffer for state_indices_tensor_d must allocate the same
per-request column count as the runtime block table, which includes
num_speculative_blocks trailing scratch columns."""
block_size = 16
max_model_len = 256
max_num_seqs = 4
num_speculative_tokens = 1
num_speculative_blocks = 2
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=num_speculative_blocks,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
expected_cols = (max_model_len // block_size) + num_speculative_blocks
assert builder.state_indices_tensor_d.shape == (max_num_seqs, expected_cols)
def test_block_idx_cudagraph_capture_padded_by_num_reqs():
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
Past num_decodes, the slice must be zero-filled."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
num_speculative_tokens = 1
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=2,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
builder.block_idx_last_scheduled_token.fill_(-1)
builder.block_idx_last_computed_token.fill_(-1)
num_decodes = 2
num_reqs = 3
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
state_indices_d = torch.zeros(
(num_decodes, builder.state_indices_tensor_d.shape[1]),
dtype=torch.int32,
device=device,
)
query_start_loc_d = torch.arange(
num_decodes + 1, dtype=torch.int32, device=device
) * (1 + num_speculative_tokens)
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
metadata = BaseMambaAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_reqs=num_reqs,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=state_indices_d,
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=num_accepted_tokens,
block_idx_last_scheduled_token=block_idx_vals,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=block_idx_vals,
block_idx_last_scheduled_token_prev_step=None,
seq_lens=seq_lens,
)
out = builder._update_metadata_for_cudagraph_capture(metadata)
assert out.block_idx_last_scheduled_token.shape == (num_reqs,)
assert out.block_idx_last_computed_token.shape == (num_reqs,)
torch.testing.assert_close(
out.block_idx_last_scheduled_token[:num_decodes], block_idx_vals
)
torch.testing.assert_close(
out.block_idx_last_computed_token[:num_decodes], block_idx_vals
)
assert torch.all(out.block_idx_last_scheduled_token[num_decodes:] == 0)
assert torch.all(out.block_idx_last_computed_token[num_decodes:] == 0)
def test_block_idx_prev_step_persistent_buffer_allocated():
"""With mamba_cache_mode='all' + spec decode, the builder must allocate
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
same shape as the existing block_idx_last_{scheduled,computed}_token
buffers, so cudagraph capture records a stable pointer for the prev-step
input anchor consumed by mamba_mixer2's input gather."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
num_speculative_tokens = 1
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=2,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
assert hasattr(builder, "block_idx_last_scheduled_token_prev_step")
assert builder.block_idx_last_scheduled_token_prev_step.shape == (max_num_seqs,)
assert builder.block_idx_last_scheduled_token_prev_step.dtype == torch.int32
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
"""Without spec decode, the prev-step buffer is unused and must not be
allocated — the input anchor reduces to last_computed_token."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len, max_num_seqs, num_speculative_tokens=0
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
assert not hasattr(builder, "block_idx_last_scheduled_token_prev_step")
def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
into the builder's persistent buffer (so cudagraph replay reads from the
same underlying memory), pad past num_decodes with zero, and return a
slice of the persistent buffer in the metadata."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
num_speculative_tokens = 1
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=2,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
builder.block_idx_last_scheduled_token.fill_(-1)
builder.block_idx_last_computed_token.fill_(-1)
builder.block_idx_last_scheduled_token_prev_step.fill_(-1)
num_decodes = 2
num_reqs = 3
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
prev_step_vals = torch.tensor([2, 4], dtype=torch.int32, device=device)
state_indices_d = torch.zeros(
(num_decodes, builder.state_indices_tensor_d.shape[1]),
dtype=torch.int32,
device=device,
)
query_start_loc_d = torch.arange(
num_decodes + 1, dtype=torch.int32, device=device
) * (1 + num_speculative_tokens)
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
metadata = BaseMambaAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_reqs=num_reqs,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=state_indices_d,
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=num_accepted_tokens,
block_idx_last_scheduled_token=block_idx_vals,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=block_idx_vals,
block_idx_last_scheduled_token_prev_step=prev_step_vals,
seq_lens=seq_lens,
)
out = builder._update_metadata_for_cudagraph_capture(metadata)
# Output field exists and is identity-shared with the persistent buffer.
assert out.block_idx_last_scheduled_token_prev_step is not None
assert (
out.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
== builder.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
), (
"prev-step buffer must live in the builder's persistent buffer, not "
"in the caller-provided tensor"
)
# Padded by num_reqs (not num_decode_tokens) — same fix as bug 2 for the
# other block_idx_* fields.
assert out.block_idx_last_scheduled_token_prev_step.shape == (num_reqs,)
# First num_decodes values: input values copied through.
torch.testing.assert_close(
out.block_idx_last_scheduled_token_prev_step[:num_decodes],
prev_step_vals,
)
# Tail values past num_decodes: zero-filled padding for cudagraph capture.
assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0)
def test_num_state_writes_p_matches_block_idx_tensors():
"""The prefill state save sizes its batched copy from
num_state_writes_p instead of reading the block index tensors back from
the GPU, so the host count must equal what those tensors say."""
# (block_size, seq_lens, query_lens, is_prefilling)
cases = [
# block-aligned and unaligned computed tokens, and a chunked prefill
(16, [64, 33, 17], [48, 17, 1], [True, True, True]),
(32, [200, 100, 64], [200, 36, 64], [True, True, True]),
# decode rows ahead of the prefills
(16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]),
# prefills that stay inside their first block: nothing to write
(16, [65, 66, 67], [1, 2, 3], [True, True, True]),
]
for block_size, seq_lens, query_lens, is_prefilling in cases:
metadata = MockMambaBuilder.build_mamba_metadata(
_make_vllm_config(4096, len(seq_lens), block_size=block_size),
seq_lens=seq_lens,
query_lens=query_lens,
is_prefilling=is_prefilling,
)
num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills
block_idx_last_p = metadata.block_idx_last_scheduled_token[
num_reqs - num_prefills : num_reqs
]
expected = int(
(block_idx_last_p - metadata.block_idx_first_scheduled_token_p)
.clamp(min=0)
.sum()
)
assert metadata.num_state_writes_p == expected, (
f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}"
)
+38 -38
View File
@@ -39,6 +39,7 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.diffusion_scheduler import (
DiffusionAsyncScheduler,
@@ -6751,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler(
num_blocks: int = 100,
supports_divergent_hits: bool = True,
) -> Scheduler:
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
"""FA + Mamba ("align" cache mode) scheduler with a MockKVConnector."""
model_config = ModelConfig(
model="facebook/opt-125m",
trust_remote_code=True,
@@ -6772,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler(
cache_config=CacheConfig(
block_size=block_size,
enable_prefix_caching=True,
mamba_cache_mode="all",
mamba_cache_mode="align",
),
kv_transfer_config=KVTransferConfig(
kv_connector="MockKVConnector",
@@ -6804,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
mamba_cache_mode="align",
),
),
],
@@ -6820,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler(
)
def _seed_hybrid_prefix(
manager: KVCacheManager, num_blocks: int, block_size: int
) -> tuple[list[int], list[int]]:
"""Prefill a prefix one block per step so that align-mode Mamba caches
the state at every block boundary. Returns the FA and Mamba block ids."""
[fill] = create_requests(
num_requests=1,
num_tokens=num_blocks * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
assert num_computed == 0
mamba_ids = []
for i in range(num_blocks):
manager.allocate_slots(
fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None
)
fill.num_computed_tokens += block_size
fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id)
mamba_ids.append(group_mamba_ids[i])
manager.free(fill)
return fa_ids, mamba_ids
@pytest.mark.parametrize(
"matched_tokens,expected_num_computed",
[
@@ -6845,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix so both groups cache all four boundaries
# (mamba cache mode "all" caches every block's state densely).
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
fa_ids = [b.block_id for b in blocks.blocks[0]]
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
# Seed a 4-block prefix so both groups cache all four boundaries.
fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
# Evict the FA tail and the middle mamba states; block 0 (both groups)
# and the deep mamba state at block 3 survive.
@@ -6885,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
[new_req] = output.scheduled_new_reqs
assert new_req.num_computed_tokens == expected_num_computed
@pytest.mark.parametrize(
@@ -6921,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix in both groups.
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
_, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
@@ -6955,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
[new_req] = output.scheduled_new_reqs
assert new_req.num_computed_tokens == expected_num_computed
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
@@ -60,7 +60,7 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -78,7 +78,7 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -67,7 +67,6 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
self._mamba_group_ids: list[int] = []
self._attn_group_ids: list[int] = [0]
self._num_ssm_scratch_blocks = 0
self._ssm_state_slots_are_positional = False
self._is_hma_required = False
self.kv_cache_config = SimpleNamespace(
kv_cache_groups=[None, None],
@@ -130,7 +129,7 @@ def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0):
block_size=16,
shapes=shapes,
dtypes=(torch.float32,) * num_states,
mamba_cache_mode="all",
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
)
@@ -146,7 +145,7 @@ def _gate_vllm_config(
block_size: int = 16,
num_lookahead_tokens: int = 0,
num_speculative_tokens: int = 0,
mamba_cache_mode: str = "all",
mamba_cache_mode: str = "align",
dcp_size: int = 1,
):
return SimpleNamespace(
@@ -193,7 +192,7 @@ def test_split_block_groups_ignores_transfer_disabled_group():
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
mamba_cache_mode="align",
)
config = SimpleNamespace(
kv_cache_groups=[
@@ -361,7 +360,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
block_size=4,
num_lookahead_tokens=7,
num_speculative_tokens=7,
mamba_cache_mode="align",
dcp_size=dcp_size,
)
scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config)
@@ -398,21 +396,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
)
def test_scheduler_rejects_dspark_with_positional_mamba_cache():
"""Positional mode counts lookahead in the Mamba allocation, so the
one-block recurrent-state tail bound no longer holds."""
with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(
speculative_config=_spec_config("dspark"),
block_size=4,
mamba_cache_mode="all",
),
"engine",
_dspark_wrapped_attention_config(),
)
def test_scheduler_rejects_hybrid_write():
config = SimpleNamespace(
kv_cache_groups=[
@@ -441,20 +424,7 @@ def test_split_block_groups_accepts_empty_abort_payload():
assert sched.split_block_groups(()) == ([], [])
def test_split_block_groups_keeps_positional_slots_in_all_mode():
# mamba_cache_mode="all" keeps a state per block position.
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_ssm_state_slots_are_positional=True,
)
attn, mamba = sched.split_block_groups(([1], [40, 41, 42]))
assert attn == [1]
assert mamba == [[40, 41, 42]]
def test_split_block_groups_keeps_only_running_state_outside_all_mode():
def test_split_block_groups_keeps_only_running_state():
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
@@ -464,24 +434,21 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode():
@pytest.mark.parametrize(
"positional,blocks,expected",
"blocks,expected",
[
# Trailing scratch slots go in either cache mode.
(True, [40, 41, 42], [40]),
# Outside "all" the running state is picked only after they are gone.
(False, [40, 41, 42, 43], [41]),
# The running state is picked only after the scratch slots are gone.
([40, 41, 42, 43], [41]),
# Never empty: a list shorter than the scratch count keeps one state.
(True, [40], [40]),
(False, [], []),
([40], [40]),
([], []),
],
)
def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected):
def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected):
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_num_ssm_scratch_blocks=2,
_ssm_state_slots_are_positional=positional,
)
assert sched.split_block_groups(([1], blocks)) == ([1], [expected])
@@ -555,7 +522,6 @@ def _make_read_scheduler():
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_ssm_state_slots_are_positional=True,
request_id_to_transfer_id={},
transfer_id_to_request_id={},
_reqs_need_recv={},
@@ -578,17 +544,17 @@ def _make_read_request(remote_block_ids):
def test_update_state_drops_decode_recompute_tail_block():
sched = _make_read_scheduler()
request = _make_read_request([[10, 11], [90, 91]])
request = _make_read_request([[10, 11], [91]])
blocks = _FakeBlocks(
all_groups=([100, 101, 102], [200, 201, 202]),
)
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]]
assert sched._reqs_need_recv["req"][1] == [[100, 101], [202]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [
[10, 11],
[90, 91],
[91],
]
@@ -609,7 +575,6 @@ def test_update_state_pairs_trimmed_mamba_state_with_remote_state():
"""The DSpark shape end to end: scratch slots are stripped, the running
state is then selected, and the attention list loses its lookahead tail."""
sched = _make_read_scheduler()
sched._ssm_state_slots_are_positional = False
sched._num_ssm_scratch_blocks = 2
sched._max_decode_tail_blocks = 2
request = _make_read_request([[10, 11], [90]])
@@ -856,28 +856,6 @@ def test_apply_prefix_caching_mamba_hybrid(
[[6, 7, 8, 9], [99]],
id="fa_prefix_hit_and_ssm_trim",
),
# Multi-slot SSM ("all" mode): a local prefix hit leaves fewer local
# slots; the earlier remote slots are covered locally → remote tail.
pytest.param(
10,
10,
[list(range(10)), [5, 6]],
[list(range(10)), [1, 2, 3]],
[list(range(10)), [5, 6]],
[list(range(10)), [2, 3]],
id="ssm_multi_block_local_hit_tail",
),
# Multi-slot SSM ("all" mode): the one trailing local position holds
# the token D recomputes itself → local head-clip.
pytest.param(
10,
10,
[list(range(10)), [4, 5, 6]],
[list(range(10)), [8, 9]],
[list(range(10)), [4, 5]],
[list(range(10)), [8, 9]],
id="ssm_multi_block_local_extra_head_clip",
),
],
)
def test_apply_prefix_caching_ssm_prefix_cache_hit(
@@ -2034,13 +2012,12 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_ssm_single_state():
"""In single-state cache modes, SSM lists are reduced to the running
state slot: speculative scratch slots, null placeholders and the previous
step's state carry nothing. Attention groups pass through untouched."""
"""SSM lists are reduced to the running state slot: speculative scratch
slots, null placeholders and the previous step's state carry nothing.
Attention groups pass through untouched."""
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
sched.blocks_per_sw = [0, 0]
sched._ssm_spec_blocks = [None, 2]
sched._ssm_state_slots_are_positional = False
# Align-mode list: null placeholders, state block, 2 speculative slots.
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 0, 7, 8, 9]))
@@ -2069,19 +2046,6 @@ def test_exchange_clipped_blocks_ssm_single_state():
assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],)
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_ssm_positional_states():
"""In "all" mode every position holds a state, so only the speculative
slots go; placeholders stay to keep the list position-indexed."""
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
sched.blocks_per_sw = [0, 0]
sched._ssm_spec_blocks = [None, 2]
sched._ssm_state_slots_are_positional = True
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 5, 6, 7, 8, 9]))
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
@@ -566,10 +566,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
("state-spaces/mamba-1.4b-hf", 16, 1),
],
)
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
def test_mamba_cpu_offload_boundary(
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
):
def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int):
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
@@ -586,7 +583,7 @@ def test_mamba_cpu_offload_boundary(
kv_transfer_config=kv_transfer_config,
language_model_only=True,
enable_prefix_caching=True,
mamba_cache_mode=mamba_cache_mode,
mamba_cache_mode="align",
# Use lossless state storage so this exact-equality regression isolates
# offload boundary selection from low-precision Mamba checkpointing.
mamba_cache_dtype="float32",
+1 -1
View File
@@ -474,7 +474,7 @@ def make_kv_cache_config(
mamba_enabled: bool = False,
sw_size: int = 128,
num_blocks: int = 100,
mamba_cache_mode: Literal["all", "align", "none"] = "none",
mamba_cache_mode: Literal["align", "none"] = "none",
) -> KVCacheConfig:
kv_cache_groups = [
KVCacheGroupSpec(
+1 -50
View File
@@ -2335,7 +2335,7 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
# of the hash block, so it cannot exceed the smallest of them.
hash_block_size=BLOCK_SIZE,
dcp_world_size=dcp_world_size,
mamba_cache_mode="all",
mamba_cache_mode="none",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
@@ -2747,55 +2747,6 @@ def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None:
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
def test_eager_store_scans_mamba_all_blocks_positionally() -> None:
"""Mamba all-mode retains positional identity and needs no handoff."""
block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=16,
num_gpu_blocks=24,
block_size=block_size,
mamba_cache_mode="all",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
assert attention_manager.has_positionally_stable_blocks
assert mamba_manager.has_positionally_stable_blocks
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
attention_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
)
mamba_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=1
)
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
# build_connector_meta runs before the scheduler commits this step's
# tokens, so the positional scan picks the blocks up on the next step.
first = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 2 * block_size},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
)
assert first.store_gpu_blocks == []
req.num_computed_tokens = 2 * block_size
meta = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 1},
cached_req_new_blocks={req.request_id: None},
)
)
assert set(meta.store_gpu_blocks) == {
*(block.block_id for block in attention_blocks),
*(block.block_id for block in mamba_blocks),
}
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2]
@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"])
def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None:
"""Handoffs for a request leaving this step must not be read.
+2 -10
View File
@@ -189,7 +189,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
@pytest.mark.parametrize("mamba_cache_mode", ["none", "all", "align"])
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
"""Mamba groups hold the running-state block plus the speculative tail.
@@ -227,14 +227,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
"""Attention, circular, and Mamba groups exercise every reservation branch.
"none" and "all" reach the same branch of both `_reserved_block_count` and
`MambaManager`, which tests only for "align". They are still both listed:
the mode is a spec-level input, and having the real manager confirm the
prediction for each is what keeps a future divergence between them from
landing unnoticed.
"""
"""Attention, circular, and Mamba groups exercise every reservation branch."""
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
@@ -242,7 +235,6 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
_attention_group(),
_circular_group(),
_mamba_group("none"),
_mamba_group("all"),
_mamba_group("align"),
],
)
+1 -1
View File
@@ -398,7 +398,7 @@ def _make_kv_cache_config(cfg: _TestConfig, layer_names: list[str]) -> KVCacheCo
(cfg.temporal_state_dim,),
),
dtypes=(cfg.dtype, cfg.dtype),
mamba_cache_mode="all",
mamba_cache_mode="align",
)
group = KVCacheGroupSpec(
layer_names=layer_names,