diff --git a/tests/kernels/mamba/test_cpu_short_conv.py b/tests/kernels/mamba/test_cpu_short_conv.py index c8e85a4551..85665c205b 100644 --- a/tests/kernels/mamba/test_cpu_short_conv.py +++ b/tests/kernels/mamba/test_cpu_short_conv.py @@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config): state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32), num_accepted_tokens=None, query_start_loc_d=None, - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, - num_computed_tokens_p=None, seq_lens=torch.tensor([5]), ) @@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config): state_indices_tensor_d=state_indices_tensor_d, num_accepted_tokens=None, query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32), - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, - num_computed_tokens_p=None, seq_lens=torch.tensor([1, 1]), ) diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index 082b61c43d..3b3dc2dc6b 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -568,15 +568,10 @@ def test_replayssm_mixer_selects_mtp_layout( num_reqs=2, has_initial_states_p=None, query_start_loc_p=None, - num_computed_tokens_p=None, state_indices_tensor_p=None, state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32), query_start_loc_d=query_start_loc_d, num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32), - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, seq_lens=torch.tensor([104, 102], dtype=torch.int32), replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)), replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32), diff --git a/tests/test_config.py b/tests/test_config.py index e7389d18b4..48d286b521 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -270,9 +270,6 @@ def test_kda_recoverssm_derivation_is_revalidated(): with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"): VllmConfig.validate_mamba_cached_kernel(config) config.use_v2_model_runner = True - config.cache_config.mamba_cache_mode = "all" - with pytest.raises(ValueError, match="only none and align"): - VllmConfig.validate_mamba_cached_kernel(config) config.cache_config.mamba_cache_mode = "none" config.model_config.architecture = "NemotronHForCausalLM" @@ -286,6 +283,12 @@ def test_kda_recoverssm_derivation_is_revalidated(): VllmConfig.validate_mamba_cached_kernel(config) +def test_mamba_cache_mode_all_is_rejected(): + """The removed 'all' mode must fail validation instead of being ignored.""" + with pytest.raises(ValidationError, match="mamba_cache_mode"): + CacheConfig(mamba_cache_mode="all") + + def test_per_request_spec_decode_metrics_requires_spec_decode(): # The flag only makes sense with speculative decoding configured; enabling # it without --speculative-config should fail fast rather than silently diff --git a/tests/v1/attention/test_mamba_metadata_builder.py b/tests/v1/attention/test_mamba_metadata_builder.py new file mode 100644 index 0000000000..85d9e0b764 --- /dev/null +++ b/tests/v1/attention/test_mamba_metadata_builder.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for BaseMambaAttentionMetadataBuilder batch classification.""" + +from types import SimpleNamespace + +from tests.v1.attention.utils import MockMambaBuilder +from vllm.config.compilation import CUDAGraphMode + + +def _make_vllm_config( + max_model_len: int, + max_num_seqs: int, + num_speculative_tokens: int = 0, + block_size: int | None = None, +): + """Create a minimal mock VllmConfig with only the fields the builder + accesses, avoiding any model download / HF config inspection.""" + speculative_config = ( + SimpleNamespace( + num_speculative_tokens=num_speculative_tokens, + parallel_drafting=False, + ) + if num_speculative_tokens > 0 + else None + ) + return SimpleNamespace( + cache_config=SimpleNamespace( + block_size=block_size, + mamba_cache_mode="none", + use_replayssm=False, + replayssm_buffer_len=16, + ), + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.FULL, + max_cudagraph_capture_size=None, + ), + speculative_config=speculative_config, + num_speculative_tokens=num_speculative_tokens, + parallel_config=SimpleNamespace(decode_context_parallel_size=1), + scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs), + model_config=SimpleNamespace(max_model_len=max_model_len), + ) + + +def test_mamba_single_token_prompt_runs_as_prefill(): + seq_lens = [8, 9, 1] + config = _make_vllm_config(256, len(seq_lens), block_size=16) + metadata = MockMambaBuilder.build_mamba_metadata( + config, + seq_lens=seq_lens, + query_lens=[1] * len(seq_lens), + is_prefilling=[False, False, True], + ) + + assert metadata.num_decodes == 2 + assert metadata.num_prefills == 1 + assert metadata.has_initial_states_p.tolist() == [False] diff --git a/tests/v1/attention/test_mamba_update_block_table.py b/tests/v1/attention/test_mamba_update_block_table.py deleted file mode 100644 index 49a298fe19..0000000000 --- a/tests/v1/attention/test_mamba_update_block_table.py +++ /dev/null @@ -1,466 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression test for https://github.com/vllm-project/vllm/issues/34865. - -When multiple KV cache groups share the same MambaSpec (as in Nemotron -hybrid models), the metadata caching optimization reuses metadata from -an earlier group via update_block_table(). In 'all' mode with CUDA graphs, -update_block_table() must copy block_idx_last_scheduled_token and -block_idx_last_computed_token to the *current* builder's persistent -buffers, otherwise CUDA graph replay reads stale values from uninitialized -buffers. -""" - -from types import SimpleNamespace - -import torch - -from tests.v1.attention.utils import MockMambaBuilder -from vllm.config.compilation import CUDAGraphMode -from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadata -from vllm.v1.kv_cache_interface import MambaSpec - - -def _make_vllm_config( - max_model_len: int, - max_num_seqs: int, - num_speculative_tokens: int = 0, - block_size: int | None = None, -): - """Create a minimal mock VllmConfig with only the fields the builder - accesses, avoiding any model download / HF config inspection.""" - speculative_config = ( - SimpleNamespace( - num_speculative_tokens=num_speculative_tokens, - parallel_drafting=False, - ) - if num_speculative_tokens > 0 - else None - ) - return SimpleNamespace( - cache_config=SimpleNamespace( - block_size=block_size, - mamba_cache_mode="all", - use_replayssm=False, - replayssm_buffer_len=16, - ), - compilation_config=SimpleNamespace( - cudagraph_mode=CUDAGraphMode.FULL, - max_cudagraph_capture_size=None, - ), - speculative_config=speculative_config, - num_speculative_tokens=num_speculative_tokens, - parallel_config=SimpleNamespace(decode_context_parallel_size=1), - scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs), - model_config=SimpleNamespace(max_model_len=max_model_len), - ) - - -def test_mamba_single_token_prompt_runs_as_prefill(): - seq_lens = [8, 9, 1] - config = _make_vllm_config(256, len(seq_lens), block_size=16) - metadata = MockMambaBuilder.build_mamba_metadata( - config, - seq_lens=seq_lens, - query_lens=[1] * len(seq_lens), - is_prefilling=[False, False, True], - ) - - assert metadata.num_decodes == 2 - assert metadata.num_prefills == 1 - assert metadata.has_initial_states_p.tolist() == [False] - - -def test_update_block_table_copies_block_idx_to_persistent_buffers(): - """update_block_table() must write block_idx tensors to the current - builder's persistent buffers, not leave them pointing to a different - builder's buffers.""" - block_size = 16 - max_model_len = 256 - num_reqs = 4 - device = torch.device("cpu") - - vllm_config = _make_vllm_config(max_model_len, num_reqs) - - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - ) - - # Two builders simulating two KV cache groups with the same MambaSpec. - builder_a = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - builder_b = MockMambaBuilder(spec, ["layer1"], vllm_config, device) - - # Sanity: each builder has its own persistent buffer. - assert ( - builder_a.block_idx_last_scheduled_token.data_ptr() - != builder_b.block_idx_last_scheduled_token.data_ptr() - ) - - # Construct decode-only metadata as if builder_a.build() produced it. - max_blocks = max_model_len // block_size - seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device) - block_idx_vals = (seq_lens - 1) // block_size # [3, 3, 3, 3] - - builder_a.block_idx_last_scheduled_token[:num_reqs].copy_(block_idx_vals) - builder_a.block_idx_last_computed_token[:num_reqs].copy_(block_idx_vals) - - metadata_a = BaseMambaAttentionMetadata( - num_prefills=0, - num_prefill_tokens=0, - num_decodes=num_reqs, - num_decode_tokens=num_reqs, - num_reqs=num_reqs, - has_initial_states_p=None, - query_start_loc_p=None, - num_computed_tokens_p=None, - state_indices_tensor_p=None, - query_start_loc_d=None, - num_accepted_tokens=None, - state_indices_tensor_d=builder_a.state_indices_tensor_d[:num_reqs], - block_idx_last_scheduled_token=( - builder_a.block_idx_last_scheduled_token[:num_reqs] - ), - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=( - builder_a.block_idx_last_computed_token[:num_reqs] - ), - block_idx_last_scheduled_token_prev_step=None, - seq_lens=seq_lens, - ) - - # Call update_block_table on builder_b (simulates the metadata caching - # optimization reusing metadata from builder_a's group). - blk_table = torch.randint( - 0, 100, (num_reqs, max_blocks), dtype=torch.int32, device=device - ) - slot_mapping = torch.zeros(num_reqs, dtype=torch.int64, device=device) - - metadata_b = builder_b.update_block_table(metadata_a, blk_table, slot_mapping) - - # block_idx tensors must live in builder_b's persistent buffers. - def shares_storage(tensor, buffer): - return ( - tensor.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr() - ) - - assert shares_storage( - metadata_b.block_idx_last_scheduled_token, - builder_b.block_idx_last_scheduled_token, - ), "block_idx_last_scheduled_token not in builder_b's persistent buffer" - - assert shares_storage( - metadata_b.block_idx_last_computed_token, - builder_b.block_idx_last_computed_token, - ), "block_idx_last_computed_token not in builder_b's persistent buffer" - - # Must NOT point to builder_a's buffers. - assert not shares_storage( - metadata_b.block_idx_last_scheduled_token, - builder_a.block_idx_last_scheduled_token, - ), "block_idx_last_scheduled_token still points to builder_a's buffer" - - # Values must be correct (copied from metadata_a). - torch.testing.assert_close( - metadata_b.block_idx_last_scheduled_token, - block_idx_vals, - ) - torch.testing.assert_close( - metadata_b.block_idx_last_computed_token, - block_idx_vals, - ) - - -def test_state_indices_tensor_d_includes_num_speculative_blocks(): - """Regression test for https://github.com/vllm-project/vllm/issues/39809 - bug 1: with mamba_cache_mode='all' and speculative decoding enabled, - the cudagraph buffer for state_indices_tensor_d must allocate the same - per-request column count as the runtime block table, which includes - num_speculative_blocks trailing scratch columns.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 4 - num_speculative_tokens = 1 - num_speculative_blocks = 2 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=num_speculative_blocks, - ) - - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - expected_cols = (max_model_len // block_size) + num_speculative_blocks - assert builder.state_indices_tensor_d.shape == (max_num_seqs, expected_cols) - - -def test_block_idx_cudagraph_capture_padded_by_num_reqs(): - """Regression test for https://github.com/vllm-project/vllm/issues/39809 - bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for - _cudagraph_capture must slice block_idx_last_{scheduled,computed}_token - by the request count (padded_bs == num_reqs), not by num_decode_tokens. - Past num_decodes, the slice must be zero-filled.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - num_speculative_tokens = 1 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=2, - ) - - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - builder.block_idx_last_scheduled_token.fill_(-1) - builder.block_idx_last_computed_token.fill_(-1) - - num_decodes = 2 - num_reqs = 3 - num_decode_tokens = num_decodes * (1 + num_speculative_tokens) - seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device) - block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device) - state_indices_d = torch.zeros( - (num_decodes, builder.state_indices_tensor_d.shape[1]), - dtype=torch.int32, - device=device, - ) - query_start_loc_d = torch.arange( - num_decodes + 1, dtype=torch.int32, device=device - ) * (1 + num_speculative_tokens) - num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device) - - metadata = BaseMambaAttentionMetadata( - num_prefills=0, - num_prefill_tokens=0, - num_decodes=num_decodes, - num_decode_tokens=num_decode_tokens, - num_reqs=num_reqs, - has_initial_states_p=None, - query_start_loc_p=None, - num_computed_tokens_p=None, - state_indices_tensor_p=None, - state_indices_tensor_d=state_indices_d, - query_start_loc_d=query_start_loc_d, - num_accepted_tokens=num_accepted_tokens, - block_idx_last_scheduled_token=block_idx_vals, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=block_idx_vals, - block_idx_last_scheduled_token_prev_step=None, - seq_lens=seq_lens, - ) - - out = builder._update_metadata_for_cudagraph_capture(metadata) - - assert out.block_idx_last_scheduled_token.shape == (num_reqs,) - assert out.block_idx_last_computed_token.shape == (num_reqs,) - torch.testing.assert_close( - out.block_idx_last_scheduled_token[:num_decodes], block_idx_vals - ) - torch.testing.assert_close( - out.block_idx_last_computed_token[:num_decodes], block_idx_vals - ) - assert torch.all(out.block_idx_last_scheduled_token[num_decodes:] == 0) - assert torch.all(out.block_idx_last_computed_token[num_decodes:] == 0) - - -def test_block_idx_prev_step_persistent_buffer_allocated(): - """With mamba_cache_mode='all' + spec decode, the builder must allocate - block_idx_last_scheduled_token_prev_step as a persistent buffer with the - same shape as the existing block_idx_last_{scheduled,computed}_token - buffers, so cudagraph capture records a stable pointer for the prev-step - input anchor consumed by mamba_mixer2's input gather.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - num_speculative_tokens = 1 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=2, - ) - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - assert hasattr(builder, "block_idx_last_scheduled_token_prev_step") - assert builder.block_idx_last_scheduled_token_prev_step.shape == (max_num_seqs,) - assert builder.block_idx_last_scheduled_token_prev_step.dtype == torch.int32 - - -def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode(): - """Without spec decode, the prev-step buffer is unused and must not be - allocated — the input anchor reduces to last_computed_token.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, max_num_seqs, num_speculative_tokens=0 - ) - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - ) - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - assert not hasattr(builder, "block_idx_last_scheduled_token_prev_step") - - -def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer(): - """_update_metadata_for_cudagraph_capture must copy the prev-step anchor - into the builder's persistent buffer (so cudagraph replay reads from the - same underlying memory), pad past num_decodes with zero, and return a - slice of the persistent buffer in the metadata.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - num_speculative_tokens = 1 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=2, - ) - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - builder.block_idx_last_scheduled_token.fill_(-1) - builder.block_idx_last_computed_token.fill_(-1) - builder.block_idx_last_scheduled_token_prev_step.fill_(-1) - - num_decodes = 2 - num_reqs = 3 - num_decode_tokens = num_decodes * (1 + num_speculative_tokens) - seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device) - block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device) - prev_step_vals = torch.tensor([2, 4], dtype=torch.int32, device=device) - state_indices_d = torch.zeros( - (num_decodes, builder.state_indices_tensor_d.shape[1]), - dtype=torch.int32, - device=device, - ) - query_start_loc_d = torch.arange( - num_decodes + 1, dtype=torch.int32, device=device - ) * (1 + num_speculative_tokens) - num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device) - - metadata = BaseMambaAttentionMetadata( - num_prefills=0, - num_prefill_tokens=0, - num_decodes=num_decodes, - num_decode_tokens=num_decode_tokens, - num_reqs=num_reqs, - has_initial_states_p=None, - query_start_loc_p=None, - num_computed_tokens_p=None, - state_indices_tensor_p=None, - state_indices_tensor_d=state_indices_d, - query_start_loc_d=query_start_loc_d, - num_accepted_tokens=num_accepted_tokens, - block_idx_last_scheduled_token=block_idx_vals, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=block_idx_vals, - block_idx_last_scheduled_token_prev_step=prev_step_vals, - seq_lens=seq_lens, - ) - - out = builder._update_metadata_for_cudagraph_capture(metadata) - - # Output field exists and is identity-shared with the persistent buffer. - assert out.block_idx_last_scheduled_token_prev_step is not None - assert ( - out.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr() - == builder.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr() - ), ( - "prev-step buffer must live in the builder's persistent buffer, not " - "in the caller-provided tensor" - ) - - # Padded by num_reqs (not num_decode_tokens) — same fix as bug 2 for the - # other block_idx_* fields. - assert out.block_idx_last_scheduled_token_prev_step.shape == (num_reqs,) - - # First num_decodes values: input values copied through. - torch.testing.assert_close( - out.block_idx_last_scheduled_token_prev_step[:num_decodes], - prev_step_vals, - ) - - # Tail values past num_decodes: zero-filled padding for cudagraph capture. - assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0) - - -def test_num_state_writes_p_matches_block_idx_tensors(): - """The prefill state save sizes its batched copy from - num_state_writes_p instead of reading the block index tensors back from - the GPU, so the host count must equal what those tensors say.""" - # (block_size, seq_lens, query_lens, is_prefilling) - cases = [ - # block-aligned and unaligned computed tokens, and a chunked prefill - (16, [64, 33, 17], [48, 17, 1], [True, True, True]), - (32, [200, 100, 64], [200, 36, 64], [True, True, True]), - # decode rows ahead of the prefills - (16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]), - # prefills that stay inside their first block: nothing to write - (16, [65, 66, 67], [1, 2, 3], [True, True, True]), - ] - - for block_size, seq_lens, query_lens, is_prefilling in cases: - metadata = MockMambaBuilder.build_mamba_metadata( - _make_vllm_config(4096, len(seq_lens), block_size=block_size), - seq_lens=seq_lens, - query_lens=query_lens, - is_prefilling=is_prefilling, - ) - num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills - block_idx_last_p = metadata.block_idx_last_scheduled_token[ - num_reqs - num_prefills : num_reqs - ] - expected = int( - (block_idx_last_p - metadata.block_idx_first_scheduled_token_p) - .clamp(min=0) - .sum() - ) - - assert metadata.num_state_writes_p == expected, ( - f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}" - ) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 722cbfb79e..2451ea5ec0 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -39,6 +39,7 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator +from vllm.v1.core.kv_cache_manager import KVCacheManager from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash from vllm.v1.core.sched.diffusion_scheduler import ( DiffusionAsyncScheduler, @@ -6751,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler( num_blocks: int = 100, supports_divergent_hits: bool = True, ) -> Scheduler: - """FA + Mamba ("all" cache mode) scheduler with a MockKVConnector.""" + """FA + Mamba ("align" cache mode) scheduler with a MockKVConnector.""" model_config = ModelConfig( model="facebook/opt-125m", trust_remote_code=True, @@ -6772,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler( cache_config=CacheConfig( block_size=block_size, enable_prefix_caching=True, - mamba_cache_mode="all", + mamba_cache_mode="align", ), kv_transfer_config=KVTransferConfig( kv_connector="MockKVConnector", @@ -6804,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler( block_size=block_size, shapes=((1, 1),), dtypes=(torch.float32,), - mamba_cache_mode="all", + mamba_cache_mode="align", ), ), ], @@ -6820,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler( ) +def _seed_hybrid_prefix( + manager: KVCacheManager, num_blocks: int, block_size: int +) -> tuple[list[int], list[int]]: + """Prefill a prefix one block per step so that align-mode Mamba caches + the state at every block boundary. Returns the FA and Mamba block ids.""" + [fill] = create_requests( + num_requests=1, + num_tokens=num_blocks * block_size, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["fill"], + ) + computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) + assert num_computed == 0 + mamba_ids = [] + for i in range(num_blocks): + manager.allocate_slots( + fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None + ) + fill.num_computed_tokens += block_size + fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id) + mamba_ids.append(group_mamba_ids[i]) + manager.free(fill) + return fa_ids, mamba_ids + + @pytest.mark.parametrize( "matched_tokens,expected_num_computed", [ @@ -6845,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector( manager = scheduler.kv_cache_manager assert isinstance(manager.coordinator, HybridKVCacheCoordinator) - # Seed a 4-block prefix so both groups cache all four boundaries - # (mamba cache mode "all" caches every block's state densely). - [fill] = create_requests( - num_requests=1, - num_tokens=4 * block_size, - max_tokens=1, - same_prompt=True, - block_size=block_size, - req_ids=["fill"], - ) - computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) - blocks = manager.allocate_slots( - fill, fill.num_tokens, num_computed, computed_blocks - ) - fa_ids = [b.block_id for b in blocks.blocks[0]] - mamba_ids = [b.block_id for b in blocks.blocks[1]] - manager.free(fill) + # Seed a 4-block prefix so both groups cache all four boundaries. + fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size) # Evict the FA tail and the middle mamba states; block 0 (both groups) # and the deep mamba state at block 3 survive. @@ -6885,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector( scheduler.add_request(replay) output = scheduler.schedule() - num_scheduled = output.num_scheduled_tokens[replay.request_id] - assert replay.num_tokens - num_scheduled == expected_num_computed + [new_req] = output.scheduled_new_reqs + assert new_req.num_computed_tokens == expected_num_computed @pytest.mark.parametrize( @@ -6921,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy( assert isinstance(manager.coordinator, HybridKVCacheCoordinator) # Seed a 4-block prefix in both groups. - [fill] = create_requests( - num_requests=1, - num_tokens=4 * block_size, - max_tokens=1, - same_prompt=True, - block_size=block_size, - req_ids=["fill"], - ) - computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) - blocks = manager.allocate_slots( - fill, fill.num_tokens, num_computed, computed_blocks - ) - mamba_ids = [b.block_id for b in blocks.blocks[1]] - manager.free(fill) + _, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size) # Keep all FA blocks; evict every mamba state but block 0. FA reaches 4 # blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba). @@ -6955,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy( scheduler.add_request(replay) output = scheduler.schedule() - num_scheduled = output.num_scheduled_tokens[replay.request_id] - assert replay.num_tokens - num_scheduled == expected_num_computed + [new_req] = output.scheduled_new_reqs + assert new_req.num_computed_tokens == expected_num_computed def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16): diff --git a/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh b/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh index ddd8d06b01..c6f79f3721 100755 --- a/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh @@ -60,7 +60,7 @@ vllm serve "$MODEL" \ --block-size 128 \ --trust-remote-code \ --enable-prefix-caching \ - --mamba-cache-mode all \ + --mamba-cache-mode align \ --kv-transfer-config "$KV_CONFIG" \ "${EXTRA_ARGS[@]}" & @@ -78,7 +78,7 @@ vllm serve "$MODEL" \ --block-size 128 \ --trust-remote-code \ --enable-prefix-caching \ - --mamba-cache-mode all \ + --mamba-cache-mode align \ --kv-transfer-config "$KV_CONFIG" \ "${EXTRA_ARGS[@]}" & diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py index 87ea53225a..28bc8bebe2 100644 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -67,7 +67,6 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore self._mamba_group_ids: list[int] = [] self._attn_group_ids: list[int] = [0] self._num_ssm_scratch_blocks = 0 - self._ssm_state_slots_are_positional = False self._is_hma_required = False self.kv_cache_config = SimpleNamespace( kv_cache_groups=[None, None], @@ -130,7 +129,7 @@ def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0): block_size=16, shapes=shapes, dtypes=(torch.float32,) * num_states, - mamba_cache_mode="all", + mamba_cache_mode="align", num_speculative_blocks=num_speculative_blocks, ) @@ -146,7 +145,7 @@ def _gate_vllm_config( block_size: int = 16, num_lookahead_tokens: int = 0, num_speculative_tokens: int = 0, - mamba_cache_mode: str = "all", + mamba_cache_mode: str = "align", dcp_size: int = 1, ): return SimpleNamespace( @@ -193,7 +192,7 @@ def test_split_block_groups_ignores_transfer_disabled_group(): block_size=16, shapes=((1, 1),), dtypes=(torch.float32,), - mamba_cache_mode="all", + mamba_cache_mode="align", ) config = SimpleNamespace( kv_cache_groups=[ @@ -361,7 +360,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention( block_size=4, num_lookahead_tokens=7, num_speculative_tokens=7, - mamba_cache_mode="align", dcp_size=dcp_size, ) scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config) @@ -398,21 +396,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention( ) -def test_scheduler_rejects_dspark_with_positional_mamba_cache(): - """Positional mode counts lookahead in the Mamba allocation, so the - one-block recurrent-state tail bound no longer holds.""" - with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"): - moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config( - speculative_config=_spec_config("dspark"), - block_size=4, - mamba_cache_mode="all", - ), - "engine", - _dspark_wrapped_attention_config(), - ) - - def test_scheduler_rejects_hybrid_write(): config = SimpleNamespace( kv_cache_groups=[ @@ -441,20 +424,7 @@ def test_split_block_groups_accepts_empty_abort_payload(): assert sched.split_block_groups(()) == ([], []) -def test_split_block_groups_keeps_positional_slots_in_all_mode(): - # mamba_cache_mode="all" keeps a state per block position. - sched = _FakeScheduler( - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1], - _ssm_state_slots_are_positional=True, - ) - attn, mamba = sched.split_block_groups(([1], [40, 41, 42])) - assert attn == [1] - assert mamba == [[40, 41, 42]] - - -def test_split_block_groups_keeps_only_running_state_outside_all_mode(): +def test_split_block_groups_keeps_only_running_state(): sched = _FakeScheduler( _has_mamba=True, _attn_group_ids=[0], @@ -464,24 +434,21 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode(): @pytest.mark.parametrize( - "positional,blocks,expected", + "blocks,expected", [ - # Trailing scratch slots go in either cache mode. - (True, [40, 41, 42], [40]), - # Outside "all" the running state is picked only after they are gone. - (False, [40, 41, 42, 43], [41]), + # The running state is picked only after the scratch slots are gone. + ([40, 41, 42, 43], [41]), # Never empty: a list shorter than the scratch count keeps one state. - (True, [40], [40]), - (False, [], []), + ([40], [40]), + ([], []), ], ) -def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected): +def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected): sched = _FakeScheduler( _has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1], _num_ssm_scratch_blocks=2, - _ssm_state_slots_are_positional=positional, ) assert sched.split_block_groups(([1], blocks)) == ([1], [expected]) @@ -555,7 +522,6 @@ def _make_read_scheduler(): _has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1], - _ssm_state_slots_are_positional=True, request_id_to_transfer_id={}, transfer_id_to_request_id={}, _reqs_need_recv={}, @@ -578,17 +544,17 @@ def _make_read_request(remote_block_ids): def test_update_state_drops_decode_recompute_tail_block(): sched = _make_read_scheduler() - request = _make_read_request([[10, 11], [90, 91]]) + request = _make_read_request([[10, 11], [91]]) blocks = _FakeBlocks( all_groups=([100, 101, 102], [200, 201, 202]), ) sched.update_state_after_alloc(request, blocks, num_external_tokens=256) - assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]] + assert sched._reqs_need_recv["req"][1] == [[100, 101], [202]] assert sched._req_kv_params["req"]["remote_block_ids"] == [ [10, 11], - [90, 91], + [91], ] @@ -609,7 +575,6 @@ def test_update_state_pairs_trimmed_mamba_state_with_remote_state(): """The DSpark shape end to end: scratch slots are stripped, the running state is then selected, and the attention list loses its lookahead tail.""" sched = _make_read_scheduler() - sched._ssm_state_slots_are_positional = False sched._num_ssm_scratch_blocks = 2 sched._max_decode_tail_blocks = 2 request = _make_read_request([[10, 11], [90]]) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 629e8c7ecc..199ac2fe8c 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -856,28 +856,6 @@ def test_apply_prefix_caching_mamba_hybrid( [[6, 7, 8, 9], [99]], id="fa_prefix_hit_and_ssm_trim", ), - # Multi-slot SSM ("all" mode): a local prefix hit leaves fewer local - # slots; the earlier remote slots are covered locally → remote tail. - pytest.param( - 10, - 10, - [list(range(10)), [5, 6]], - [list(range(10)), [1, 2, 3]], - [list(range(10)), [5, 6]], - [list(range(10)), [2, 3]], - id="ssm_multi_block_local_hit_tail", - ), - # Multi-slot SSM ("all" mode): the one trailing local position holds - # the token D recomputes itself → local head-clip. - pytest.param( - 10, - 10, - [list(range(10)), [4, 5, 6]], - [list(range(10)), [8, 9]], - [list(range(10)), [4, 5]], - [list(range(10)), [8, 9]], - id="ssm_multi_block_local_extra_head_clip", - ), ], ) def test_apply_prefix_caching_ssm_prefix_cache_hit( @@ -2034,13 +2012,12 @@ def test_logical_to_kernel_block_ids_with_remote_ratio( @pytest.mark.cpu_test def test_exchange_clipped_blocks_ssm_single_state(): - """In single-state cache modes, SSM lists are reduced to the running - state slot: speculative scratch slots, null placeholders and the previous - step's state carry nothing. Attention groups pass through untouched.""" + """SSM lists are reduced to the running state slot: speculative scratch + slots, null placeholders and the previous step's state carry nothing. + Attention groups pass through untouched.""" sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True) sched.blocks_per_sw = [0, 0] sched._ssm_spec_blocks = [None, 2] - sched._ssm_state_slots_are_positional = False # Align-mode list: null placeholders, state block, 2 speculative slots. clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 0, 7, 8, 9])) @@ -2069,19 +2046,6 @@ def test_exchange_clipped_blocks_ssm_single_state(): assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],) -@pytest.mark.cpu_test -def test_exchange_clipped_blocks_ssm_positional_states(): - """In "all" mode every position holds a state, so only the speculative - slots go; placeholders stay to keep the list position-indexed.""" - sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True) - sched.blocks_per_sw = [0, 0] - sched._ssm_spec_blocks = [None, 2] - sched._ssm_state_slots_are_positional = True - - clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 5, 6, 7, 8, 9])) - assert clipped == ([1, 2, 3], [0, 5, 6, 7]) - - # ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ───────────────────── diff --git a/tests/v1/kv_connector/unit/test_offloading_connector.py b/tests/v1/kv_connector/unit/test_offloading_connector.py index db7ecca621..76d2197469 100644 --- a/tests/v1/kv_connector/unit/test_offloading_connector.py +++ b/tests/v1/kv_connector/unit/test_offloading_connector.py @@ -566,10 +566,7 @@ def test_fs_tiering_offloading(tmp_path) -> None: ("state-spaces/mamba-1.4b-hf", 16, 1), ], ) -@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"]) -def test_mamba_cpu_offload_boundary( - model: str, block_size: int, tp_size: int, mamba_cache_mode: str -): +def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int): kv_transfer_config = KVTransferConfig( kv_connector="OffloadingConnector", kv_role="kv_both", @@ -586,7 +583,7 @@ def test_mamba_cpu_offload_boundary( kv_transfer_config=kv_transfer_config, language_model_only=True, enable_prefix_caching=True, - mamba_cache_mode=mamba_cache_mode, + mamba_cache_mode="align", # Use lossless state storage so this exact-equality regression isolates # offload boundary selection from low-precision Mamba checkpointing. mamba_cache_dtype="float32", diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index f154816369..2febe3109f 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -474,7 +474,7 @@ def make_kv_cache_config( mamba_enabled: bool = False, sw_size: int = 128, num_blocks: int = 100, - mamba_cache_mode: Literal["all", "align", "none"] = "none", + mamba_cache_mode: Literal["align", "none"] = "none", ) -> KVCacheConfig: kv_cache_groups = [ KVCacheGroupSpec( diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index 8a85c88d70..1c0cd74565 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -2335,7 +2335,7 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None: # of the hash block, so it cannot exceed the smallest of them. hash_block_size=BLOCK_SIZE, dcp_world_size=dcp_world_size, - mamba_cache_mode="all", + mamba_cache_mode="none", ) sched = fix.scheduler gpu_pool = fix.gpu_block_pool @@ -2747,55 +2747,6 @@ def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None: assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0] -def test_eager_store_scans_mamba_all_blocks_positionally() -> None: - """Mamba all-mode retains positional identity and needs no handoff.""" - block_size = 4 * BLOCK_SIZE - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=16, - num_gpu_blocks=24, - block_size=block_size, - mamba_cache_mode="all", - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers - assert attention_manager.has_positionally_stable_blocks - assert mamba_manager.has_positionally_stable_blocks - - req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) - attention_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 - ) - mamba_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=1 - ) - kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - # build_connector_meta runs before the scheduler commits this step's - # tokens, so the positional scan picks the blocks up on the next step. - first = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 2 * block_size}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - ) - assert first.store_gpu_blocks == [] - req.num_computed_tokens = 2 * block_size - meta = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 1}, - cached_req_new_blocks={req.request_id: None}, - ) - ) - - assert set(meta.store_gpu_blocks) == { - *(block.block_id for block in attention_blocks), - *(block.block_id for block in mamba_blocks), - } - assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2] - - @pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"]) def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None: """Handoffs for a request leaving this step must not be read. diff --git a/tests/v1/worker/test_gpu_warmup_blocks.py b/tests/v1/worker/test_gpu_warmup_blocks.py index 067f31e935..04179ae760 100644 --- a/tests/v1/worker/test_gpu_warmup_blocks.py +++ b/tests/v1/worker/test_gpu_warmup_blocks.py @@ -189,7 +189,7 @@ def test_mixed_warmup_reserves_lookahead_blocks(): _assert_covers_lookahead(recorder.steps, num_lookahead_tokens) -@pytest.mark.parametrize("mamba_cache_mode", ["none", "all", "align"]) +@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"]) def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode): """Mamba groups hold the running-state block plus the speculative tail. @@ -227,14 +227,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode): def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: - """Attention, circular, and Mamba groups exercise every reservation branch. - - "none" and "all" reach the same branch of both `_reserved_block_count` and - `MambaManager`, which tests only for "align". They are still both listed: - the mode is a spec-level input, and having the real manager confirm the - prediction for each is what keeps a future divergence between them from - landing unnoticed. - """ + """Attention, circular, and Mamba groups exercise every reservation branch.""" return KVCacheConfig( num_blocks=num_blocks, kv_cache_tensors=[], @@ -242,7 +235,6 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: _attention_group(), _circular_group(), _mamba_group("none"), - _mamba_group("all"), _mamba_group("align"), ], ) diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index 47a27795ca..50faf0fb8b 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -398,7 +398,7 @@ def _make_kv_cache_config(cfg: _TestConfig, layer_names: list[str]) -> KVCacheCo (cfg.temporal_state_dim,), ), dtypes=(cfg.dtype, cfg.dtype), - mamba_cache_mode="all", + mamba_cache_mode="align", ) group = KVCacheGroupSpec( layer_names=layer_names, diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 13665f3cff..92b984c0fa 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -59,7 +59,7 @@ CacheDType = Literal[ MambaDType = Literal["auto", "float32", "float16", "bfloat16"] -MambaCacheMode = Literal["all", "align", "none"] +MambaCacheMode = Literal["align", "none"] PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] KVOffloadingBackend = Literal["native", "lmcache"] @@ -75,8 +75,6 @@ class CacheConfig: Accepts None (meaning "use default"). After construction, always int.""" user_specified_block_size: bool = field(default=False, init=False) """Whether block_size was explicitly provided. Derived automatically.""" - user_specified_mamba_block_size: bool = field(default=False, init=False) - """Whether mamba_block_size was explicitly provided. Derived automatically.""" kv_cache_layout: str | None = field(default=None, init=False) """Resolved physical KV cache layout name (a ``KVCacheLayout`` member). @@ -191,7 +189,6 @@ class CacheConfig: """The cache strategy for Mamba layers: - "none": set when prefix caching is disabled. - - "all": cache the mamba state of all tokens at position i * block_size. - "align": only cache the mamba state of the last token of each scheduler step and when the token is at position i * block_size. This is the default when prefix caching is enabled. @@ -293,7 +290,6 @@ class CacheConfig: "mamba_page_size_padded", "skip_page_size_padded", "user_specified_block_size", - "user_specified_mamba_block_size", "_block_size_resolved", # Post-init/derived counters "num_gpu_blocks", @@ -337,21 +333,8 @@ class CacheConfig: self.block_size = self.DEFAULT_BLOCK_SIZE else: self.user_specified_block_size = True - if self.mamba_block_size is not None: - self.user_specified_mamba_block_size = True return self - @field_validator("mamba_cache_mode", mode="after") - @classmethod - def _validate_mamba_cache_mode(cls, mode: MambaCacheMode) -> MambaCacheMode: - if mode == "all": - logger.warning_once( - "Mamba cache mode 'all' is deprecated and will be removed in an " - "upcoming release. If this is a problem, please open an issue " - "at https://github.com/vllm-project/vllm/issues." - ) - return mode - @field_validator("cache_dtype", mode="after") @classmethod def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType: diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index c25ed27a28..281666a633 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -3091,9 +3091,6 @@ class VllmConfig: if self.parallel_config.use_ubatching: unsupported.extend(self._get_dbo_unsupported_features()) - if self.cache_config.mamba_cache_mode == "all": - unsupported.append("mamba cache mode 'all'") - return unsupported def _get_v1_model_runner_unsupported_features(self) -> list[str]: @@ -3499,10 +3496,6 @@ class VllmConfig: "SSM state caches, not --enable-mamba-cache-stochastic-" "rounding, which requires an explicit float16 cache" ) - if self.cache_config.mamba_cache_mode not in ("none", "align"): - raise ValueError( - "RecoverSSM supports only none and align Mamba cache modes" - ) if ( self.cache_config.mamba_cache_mode == "align" and not self.use_v2_model_runner @@ -3535,11 +3528,6 @@ class VllmConfig: "Mamba2 ReplaySSM speculative decoding requires " "--mamba-backend flashinfer" ) - elif self.cache_config.mamba_cache_mode == "all": - raise ValueError( - "--use-replayssm supports prefix caching only in align mode; " - "pass --mamba-cache-mode align" - ) elif self.mamba_config.backend == MambaBackendEnum.FLASHINFER: if self.cache_config.mamba_cache_mode == "align": raise ValueError( diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index 30021cde50..a0ead27c2f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -33,16 +33,14 @@ EngineId = str BlockIds = tuple[list[int], ...] | list[list[int]] -def clip_ssm_state_blocks( - blocks: list[int], num_spec_blocks: int, positional: bool -) -> list[int]: - """Drop speculative scratch slots, retaining position-indexed or running state.""" +def clip_ssm_state_blocks(blocks: list[int], num_spec_blocks: int) -> list[int]: + """Drop speculative scratch slots and keep only the running-state slot.""" if not blocks: return blocks # Keep at least the running-state slot, even before scratch allocation. if num_scratch := min(num_spec_blocks, len(blocks) - 1): blocks = blocks[:-num_scratch] - return blocks if positional else blocks[-1:] + return blocks[-1:] def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index b482e6a8c4..ec24f0d199 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -450,13 +450,6 @@ def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None: "MoRIIO hybrid READ supports DSpark speculative decoding only, got " f"method={speculative_config.method!r}" ) - if vllm_config.cache_config.mamba_cache_mode == "all": - # Positional allocation counts lookahead, so its local tail can exceed - # the single running-state block supported by hybrid READ. - raise MoRIIOError( - "MoRIIO hybrid READ does not support DSpark with " - "mamba_cache_mode='all'; use 'align'" - ) def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None: @@ -521,11 +514,6 @@ class MoRIIOConnectorScheduler: "MoRIIO hybrid (mamba/KDA) transfer is implemented for READ " "mode only; set kv_connector_extra_config.read_mode=true" ) - # Only "all" mode keeps a state per block position; the other modes keep - # a single running state in the last slot. - self._ssm_state_slots_are_positional = ( - vllm_config.cache_config.mamba_cache_mode == "all" - ) self._num_ssm_scratch_blocks = ( mamba_spec.num_speculative_blocks if mamba_spec is not None else 0 ) @@ -1288,8 +1276,8 @@ class MoRIIOConnectorScheduler: """Select transferable attention and Mamba block groups. The wire payload stores the attention group first, followed by every - Mamba group in transfer-group order. Outside ``mamba_cache_mode="all"``, - only each group's running-state slot is transferred. + Mamba group in transfer-group order. Only each group's running-state + slot is transferred. """ if not block_ids: return [], [] @@ -1301,7 +1289,6 @@ class MoRIIOConnectorScheduler: clip_ssm_state_blocks( list(transfer_block_ids[group_id]), self._num_ssm_scratch_blocks, - self._ssm_state_slots_are_positional, ) for group_id in self._mamba_group_ids ] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index 0705f944f6..92f0cbc58a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -154,11 +154,6 @@ class NixlBaseConnectorScheduler: else None for g in kv_cache_config.transfer_groups ] - # Only "all" mode keeps a state per block position; the other modes - # keep a single running state in the last non-speculative slot. - self._ssm_state_slots_are_positional = ( - vllm_config.cache_config.mamba_cache_mode == "all" - ) # Threshold to decide whether to compute kv cache locally # or pull from a remote node: minimum number of remote @@ -258,11 +253,9 @@ class NixlBaseConnectorScheduler: out-of-window blocks only prior to the `request_finished_all_groups` hook. - SSM groups keep only their state-bearing slots: the trailing - speculative scratch slots always go, and in single-state cache modes - so does everything before the running state (null placeholders and - the previous step's superseded state). "all" mode keeps its remaining - slots, which the worker pairs position-wise. + SSM groups keep only their state-bearing slot: the trailing + speculative scratch slots and everything before the running state + (null placeholders and the previous step's superseded state) go. Use this at every block-id exchange point. Pass ``clip_ssm=False`` for per-step partial lists (host-buffer save), where the SSM strip @@ -289,9 +282,7 @@ class NixlBaseConnectorScheduler: and blocks and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None ): - blocks = clip_ssm_state_blocks( - blocks, n_spec_blocks, self._ssm_state_slots_are_positional - ) + blocks = clip_ssm_state_blocks(blocks, n_spec_blocks) clipped.append(blocks) return tuple(clipped) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 0cbdd786d8..1e22180b3d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -3510,13 +3510,12 @@ class NixlBaseConnectorWorker: if _is_ssm_spec(self._group_spec_types[i]): if num_decode_blocks == num_prefill_blocks: continue - # Only state-bearing slots reach here, single-state modes - # just one (see get_exchange_clipped_blocks), so differing - # counts mean position-indexed "all"-mode lists. A longer - # prefill list carries earlier positions the decode side - # already has (prefix hit) -> take its tail; a longer decode - # list holds the position D recomputes itself, which gets - # no prefill state. + # Only state-bearing slots reach here, normally just one + # (see get_exchange_clipped_blocks). A longer prefill list + # carries earlier positions the decode side already has + # (prefix hit) -> take its tail; a longer decode list holds + # the position D recomputes itself, which gets no prefill + # state. assert num_decode_blocks - num_prefill_blocks <= 1, ( f"Group {i}: unpairable SSM state slots, " f"decode={num_decode_blocks} prefill={num_prefill_blocks}" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 47aa8654fe..64b3ad5bb2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -156,7 +156,7 @@ def resolve_mamba_align_size( """Scan all KV cache groups in *spec* and return the single mamba alignment size, or None if no group requires mamba alignment. - For MambaSpec groups in "align" or "all" cache mode the hit window must be + For MambaSpec groups in "align" cache mode the hit window must be rounded down to a multiple of the offloaded chunk size. Asserts that all such groups agree on the same value. """ @@ -164,10 +164,7 @@ def resolve_mamba_align_size( for group in spec.config.groups: tokens_per_block = group.tokens_per_block kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec - if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in ( - "align", - "all", - ): + if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align": tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk assert mamba_align_size is None or mamba_align_size == tokens_per_chunk mamba_align_size = tokens_per_chunk diff --git a/vllm/model_executor/layers/mamba/mamba_mixer.py b/vllm/model_executor/layers/mamba/mamba_mixer.py index 2b1b1f9c9d..8f3b8a2638 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer.py @@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer): attn_metadata_raw = forward_context.attn_metadata assert self.cache_config is not None - mamba_block_size = self.cache_config.mamba_block_size - is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: @@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer): num_prefill_tokens = attn_metadata.num_prefill_tokens # token count num_decode_tokens = attn_metadata.num_decode_tokens - num_prefills = attn_metadata.num_prefills # request count - num_decodes = attn_metadata.num_decode_tokens # token count (=request) has_prefill = num_prefill_tokens > 0 has_decode = num_decode_tokens > 0 num_actual_tokens = num_prefill_tokens + num_decode_tokens @@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer): gate_p = prefill_decode_split.gate_p gate_d = prefill_decode_split.gate_d - if is_mamba_cache_all: - block_idx_last_computed_token_d, block_idx_last_computed_token_p = ( - torch.split( - attn_metadata.block_idx_last_computed_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = ( - torch.split( - attn_metadata.block_idx_last_scheduled_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - - block_idx_first_scheduled_token_p = ( - attn_metadata.block_idx_first_scheduled_token_p - ) - num_computed_tokens_p = attn_metadata.num_computed_tokens_p - else: - block_idx_last_computed_token_d = None - block_idx_last_computed_token_p = None - block_idx_last_scheduled_token_d = None - block_idx_last_scheduled_token_p = None - block_idx_first_scheduled_token_p = None - num_computed_tokens_p = None - ssm_outputs = [] if has_prefill: @@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer): has_initial_state=has_initial_states_p, cache_indices=state_indices_tensor_p, query_start_loc=query_start_loc_p, - block_idx_first_scheduled_token=block_idx_first_scheduled_token_p, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_p, - initial_state_idx=block_idx_last_computed_token_p, - num_computed_tokens=num_computed_tokens_p, - block_size_to_align=mamba_block_size, metadata=attn_metadata, ) # 3. State Space Model sequence transformations. @@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer): cache_indices=state_indices_tensor_p, has_initial_state=has_initial_states_p, query_start_loc=query_start_loc_p, - block_size=mamba_block_size, - block_idx_first_scheduled_token=block_idx_first_scheduled_token_p, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_p, - initial_state_idx=block_idx_last_computed_token_p, cu_chunk_seqlen=cu_chunk_seqlen_p, last_chunk_indices=last_chunk_indices_p, ) @@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer): # state_indices_tensor_d is assigned when attn_metadata is not None, # and has_decode is only True when attn_metadata is not None assert state_indices_tensor_d is not None - if is_mamba_cache_all: - state_indices_tensor_d_input = state_indices_tensor_d.gather( - 1, block_idx_last_computed_token_d.unsqueeze(1) - ).squeeze(1) - state_indices_tensor_d_output = state_indices_tensor_d.gather( - 1, block_idx_last_scheduled_token_d.unsqueeze(1) - ).squeeze(1) - else: - state_indices_tensor_d_input = state_indices_tensor_d - state_indices_tensor_d_output = state_indices_tensor_d # 2. Convolution sequence transformation conv_out_d = causal_conv1d_update( hidden_states_BC_d.transpose(0, 1), @@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer): self.conv1d.bias, self.activation, conv_state_indices=state_indices_tensor_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_d, - initial_state_idx=block_idx_last_computed_token_d, ).transpose(0, 1) # 3. State Space Model sequence transformation. @@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer): time_proj_bias, z=gate_d.transpose(0, 1), dt_softplus=True, - state_batch_indices=state_indices_tensor_d_input, - dst_state_batch_indices=state_indices_tensor_d_output, + state_batch_indices=state_indices_tensor_d, + dst_state_batch_indices=state_indices_tensor_d, out=scan_outputs_d, ) scan_outputs_d = scan_outputs_d.transpose(0, 1) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index fc58aed91c..a67480ebc2 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -549,12 +549,6 @@ class MambaMixer2(MambaBase, PluggableLayer): self._updates_replayssm_trackers = True self.num_spec = vllm_config.num_speculative_tokens - if self.num_spec > 0: - self.register_buffer( - "_decode_state_offsets", - torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0), - persistent=False, - ) # Pre-compute sizes for forward pass self.tped_intermediate_size = self.intermediate_size // self.tp_size @@ -732,8 +726,6 @@ class MambaMixer2(MambaBase, PluggableLayer): attn_metadata_raw = forward_context.attn_metadata assert self.cache_config is not None - mamba_block_size = self.cache_config.mamba_block_size - is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" use_spec_decode = self.num_spec > 0 ring_start = prev_num_accepted = prev_query_len = None @@ -803,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer): dim=0, ) - if is_mamba_cache_all: - # If prefix caching is enabled, retrieve the relevant variables - # for prefill and decode - block_idx_last_computed_token_d, block_idx_last_computed_token_p = ( - torch.split( - attn_metadata.block_idx_last_computed_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = ( - torch.split( - attn_metadata.block_idx_last_scheduled_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - if attn_metadata.block_idx_last_scheduled_token_prev_step is not None: - block_idx_last_scheduled_token_prev_step_d, _ = torch.split( - attn_metadata.block_idx_last_scheduled_token_prev_step, - [num_decodes, num_prefills], - dim=0, - ) - else: - block_idx_last_scheduled_token_prev_step_d = None - # Prefill-only variables: - block_idx_first_scheduled_token_p = ( - attn_metadata.block_idx_first_scheduled_token_p - ) - num_computed_tokens_p = attn_metadata.num_computed_tokens_p - else: - block_idx_last_computed_token_p = None - block_idx_last_scheduled_token_p = None - block_idx_first_scheduled_token_p = None - block_idx_last_scheduled_token_d = None - block_idx_last_computed_token_d = None - block_idx_last_scheduled_token_prev_step_d = None - num_computed_tokens_p = None - preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split( output[:num_actual_tokens], [num_decode_tokens, num_prefill_tokens], @@ -858,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer): # to by "state_indices_tensor_p". # In particular, it will always write the state at the # sequence end. - # In addition, "block_idx_first_scheduled_token_p" and - # "block_idx_last_scheduled_token_p" - # are provided (which are pointers into - # "state_indices_tensor_p"), it will write additional cache - # states aligned at "block_size_to_align". x = hidden_states_B_C_p.transpose( 0, 1 ) # this is the form that causal-conv see @@ -874,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer): conv_states=conv_state, has_initial_state=has_initial_states_p, cache_indices=state_indices_tensor_p, - block_idx_first_scheduled_token=block_idx_first_scheduled_token_p, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_p, - initial_state_idx=block_idx_last_computed_token_p, - num_computed_tokens=num_computed_tokens_p, - block_size_to_align=mamba_block_size, metadata=attn_metadata, query_start_loc=query_start_loc_p, ).transpose(0, 1)[:num_prefill_tokens] @@ -891,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer): initial_states = None if has_initial_states_p is not None and prep_initial_states: assert state_indices_tensor_p is not None - kernel_ssm_indices = state_indices_tensor_p - if is_mamba_cache_all: - kernel_ssm_indices = state_indices_tensor_p.gather( - 1, block_idx_last_computed_token_p.unsqueeze(1) - ).squeeze(1) initial_states = torch.where( has_initial_states_p[:, None, None, None], - ssm_state[kernel_ssm_indices], + ssm_state[state_indices_tensor_p], 0, ) @@ -921,130 +859,29 @@ class MambaMixer2(MambaBase, PluggableLayer): cu_chunk_seqlens=cu_chunk_seqlen_p, last_chunk_indices=last_chunk_indices_p, initial_states=initial_states, - return_intermediate_states=is_mamba_cache_all, dt_softplus=True, dt_limit=(0.0, float("inf")), out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim), state_dtype=ssm_state.dtype, ) - if is_mamba_cache_all: - assert mamba_block_size is not None - assert state_indices_tensor_p is not None - assert block_idx_first_scheduled_token_p is not None - assert block_idx_last_scheduled_token_p is not None - assert last_chunk_indices_p is not None - assert num_computed_tokens_p is not None - - # The chunk_stride is the number of chunks per mamba block - # e.g., if mamba_block_size = 512 and chunk_size = 256, - # then chunk_stride = 2 - chunk_stride = mamba_block_size // chunk_size - - # Save state for sequences with more than just final state, - # batched over sequences so that the number of kernel launches - # does not scale with the number of prefills. - n_blocks_to_fill = ( - block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p - ).clamp(min=0) - total_fill = attn_metadata.num_state_writes_p - if total_fill > 0: - first_chunk = torch.zeros_like(last_chunk_indices_p) - first_chunk[1:] = last_chunk_indices_p[:-1] + 1 - - # The chunk that completes this sequence's first mamba block. - # A block spans chunk_stride chunks. When the sequence resumed - # mid-block, the chunks before this step are not in varlen_states, - # so subtract them: the boundary sits earlier by their count. - num_unaligned_computed_tokens = ( - num_computed_tokens_p % mamba_block_size - ) - unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size - first_aligned_chunk = ( - first_chunk + (chunk_stride - 1) - unaligned_chunk_shift - ) - - # Flatten all writes into one list: fill_counts=[2,1,2] - # -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1] - fill_counts = n_blocks_to_fill.long() - # output_size, or repeat_interleave reads the size off the device - seq_ids = torch.repeat_interleave( - torch.arange(num_prefills, device=ssm_state.device), - fill_counts, - output_size=total_fill, - ) - write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts - write_offsets = torch.arange( - total_fill, device=ssm_state.device - ) - torch.repeat_interleave( - write_starts, fill_counts, output_size=total_fill - ) - - cache_blocks_to_fill = state_indices_tensor_p[ - seq_ids, - block_idx_first_scheduled_token_p[seq_ids] + write_offsets, - ] - # Write the states. Each row goes to its own cache block, - # so the indexed copy has no duplicate destinations. - ssm_state[cache_blocks_to_fill] = varlen_states[ - first_aligned_chunk[seq_ids] + write_offsets * chunk_stride - ] - - # For all seqs, store the last state (note: might be partial): - assert state_indices_tensor_p is not None - ssm_state[ - state_indices_tensor_p.gather( - 1, block_idx_last_scheduled_token_p.unsqueeze(1) - ).squeeze(1) - ] = varlen_states[last_chunk_indices_p] - - else: - # update ssm states - # - varlen state is a (num_prefills, nheads, headdim, dstate) - # tensor - assert state_indices_tensor_p is not None - ssm_state[state_indices_tensor_p] = varlen_states - if ring_start is not None and self._updates_replayssm_trackers: - assert prev_num_accepted is not None - reset_replayssm_ring_trackers( - ring_start, - prev_num_accepted, - prev_query_len, - state_indices_tensor_p, - ) + # update ssm states + # - varlen state is a (num_prefills, nheads, headdim, dstate) + # tensor + assert state_indices_tensor_p is not None + ssm_state[state_indices_tensor_p] = varlen_states + if ring_start is not None and self._updates_replayssm_trackers: + assert prev_num_accepted is not None + reset_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_indices_tensor_p, + ) # Process decode requests if has_decode: assert state_indices_tensor_d is not None - if is_mamba_cache_all: - if use_spec_decode: - assert block_idx_last_scheduled_token_prev_step_d is not None - input_indices = ( - block_idx_last_scheduled_token_prev_step_d.unsqueeze(1) - + self._decode_state_offsets - ) - output_indices = ( - block_idx_last_scheduled_token_d.unsqueeze(1) - + self._decode_state_offsets - ) - state_indices_tensor_d_input = state_indices_tensor_d.gather( - 1, input_indices - ) - state_indices_tensor_d_output = state_indices_tensor_d.gather( - 1, output_indices - ) - else: - state_indices_tensor_d_input = state_indices_tensor_d.gather( - 1, block_idx_last_computed_token_d.unsqueeze(1) - ).squeeze(1) - state_indices_tensor_d_output = state_indices_tensor_d.gather( - 1, block_idx_last_scheduled_token_d.unsqueeze(1) - ).squeeze(1) - else: - # Without caching, read and write in-place to the same blocks: - state_indices_tensor_d_input = state_indices_tensor_d - state_indices_tensor_d_output = state_indices_tensor_d - if ( self.use_replayssm and self.mamba_config.backend == MambaBackendEnum.FLASHINFER @@ -1081,8 +918,6 @@ class MambaMixer2(MambaBase, PluggableLayer): self.conv1d.bias, self.activation, conv_state_indices=state_indices_tensor_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_d, - initial_state_idx=block_idx_last_computed_token_d, num_accepted_tokens=num_accepted_tokens, query_start_loc=query_start_loc_d, # ReplaySSM keeps one physical state block while a speculative @@ -1207,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer): write_pos=attn_metadata.write_pos_d, is_flush=attn_metadata.is_flush_d, max_cache_len=self.replayssm_buffer_len, - state_batch_indices=state_indices_tensor_d_input, + state_batch_indices=state_indices_tensor_d, out=preallocated_ssm_out_d, enable_stochastic_rounding=( self.mamba_config.enable_stochastic_rounding @@ -1227,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer): D_d, dt_bias, dt_softplus=True, - state_batch_indices=state_indices_tensor_d_input, - dst_state_batch_indices=state_indices_tensor_d_output, + state_batch_indices=state_indices_tensor_d, + dst_state_batch_indices=state_indices_tensor_d, out=preallocated_ssm_out_d, num_accepted_tokens=num_accepted_tokens, cu_seqlens=query_start_loc_d, diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 3c87b06167..8831c5f6c4 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig): cache_config.mamba_cache_mode, model_config.architecture, ) - if ( - cache_config.mamba_cache_mode == "all" - and not model_config.supports_mamba_prefix_caching - ): - cache_config.mamba_cache_mode = "align" - logger.warning( - "Hybrid or mamba-based model detected without support " - "for prefix caching with Mamba cache 'all' mode: " - "falling back to 'align' mode." - ) - if cache_config.mamba_cache_mode == "align": - assert vllm_config.scheduler_config.enable_chunked_prefill, ( - "Chunked prefill is required for mamba cache mode 'align'." - ) + assert vllm_config.scheduler_config.enable_chunked_prefill, ( + "Chunked prefill is required for mamba cache mode 'align'." + ) # By default, mamba block size will be set to max_model_len (see # below). When enabling prefix caching, we align mamba block size # to the block size as the basic granularity for prefix caching. diff --git a/vllm/model_executor/models/interns2_mobius.py b/vllm/model_executor/models/interns2_mobius.py index e186dc575b..4ff312515b 100644 --- a/vllm/model_executor/models/interns2_mobius.py +++ b/vllm/model_executor/models/interns2_mobius.py @@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase): self.quant_config = vllm_config.quant_config self.scheduler_config = vllm_config.scheduler_config - if vllm_config.cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Intern-S2-Mobius does not support 'all' prefix caching; " - "use '--mamba-cache-mode=align' instead." - ) - nn.Module.__init__(self) self.config = config self.model = InternS2MobiusModel( diff --git a/vllm/model_executor/models/lfm2.py b/vllm/model_executor/models/lfm2.py index c43a9dcbca..be6dfbd3e4 100644 --- a/vllm/model_executor/models/lfm2.py +++ b/vllm/model_executor/models/lfm2.py @@ -458,13 +458,6 @@ class Lfm2ForCausalLM( def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Lfm2 currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - super().__init__() self.config = config self.model = Lfm2Model( diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 29bc56a696..f6e819c263 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM( def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - cache_config = vllm_config.cache_config - - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Lfm2Moe currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) super().__init__() self.config = config diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 2f5e01f7ae..32660be515 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -329,14 +329,8 @@ class Qwen3_5ForCausalLMBase( config = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config self.model_config = vllm_config.model_config - cache_config = vllm_config.cache_config scheduler_config = vllm_config.scheduler_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3.5 currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 421292e880..c17d17014f 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -227,13 +227,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): config = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3_5MTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index e431c36f30..19fc8c1562 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -828,14 +828,8 @@ class Qwen3NextForCausalLM( config = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config self.model_config = vllm_config.model_config - cache_config = vllm_config.cache_config scheduler_config = vllm_config.scheduler_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3Next currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 6b527acd62..e5e306bb86 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): config = vllm_config.model_config.hf_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3NextMTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index fe50bd4f53..e5a63f13a3 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -104,7 +104,7 @@ def _mamba_get_block_table_tensor( kv_cache_spec: MambaSpec, mamba_cache_mode: str, ) -> torch.Tensor: - if mamba_cache_mode in ("all", "none"): + if mamba_cache_mode == "none": return block_table assert block_table.is_cuda and seq_lens.is_cuda diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index db94a2a7d6..2cdc3c26fa 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -638,11 +638,6 @@ class Qwen4ExpForCausalLM( self.quant_config = vllm_config.quant_config self.config = config self.scheduler_config = vllm_config.scheduler_config - if vllm_config.cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4Exp currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.model = Qwen4ExpModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py index a2b420603f..f4302aa757 100644 --- a/vllm/models/qwen4_exp/amd/mtp.py +++ b/vllm/models/qwen4_exp/amd/mtp.py @@ -395,13 +395,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4ExpMTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 6f35bea130..20521f1800 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -669,11 +669,6 @@ class Qwen4ExpForCausalLM( self.quant_config = vllm_config.quant_config self.config = config self.scheduler_config = vllm_config.scheduler_config - if vllm_config.cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4Exp currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.model = Qwen4ExpModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index af3bcd4659..d9f26fc022 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -375,13 +375,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4ExpMTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 5b053c4c78..769ffe5e8e 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -939,13 +939,6 @@ class Platform: if mamba_page_size == 0: return - # mamba_block_size here should either be user specified value or None - mamba_block_size = ( - cache_config.mamba_block_size - if cache_config.user_specified_mamba_block_size - else None - ) - # Get kernel block alignment from the backend's supported sizes with set_current_vllm_config(vllm_config): kernel_block_alignment_size = max( @@ -962,27 +955,15 @@ class Platform: # multiple of 128 so split kernel blocks keep that invariant. kernel_block_alignment_size = max(kernel_block_alignment_size, 128) - if cache_config.mamba_cache_mode == "all": - # With prefix caching, align to mamba chunk size for kernel perf - # TODO(tdoublep): this constraint can be relaxed fairly - # easily by changing the way we layout chunks in the - # mamba2 kernels. - base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size() - assert base_chunk_size is not None - attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token) - chunk_size = lcm(base_chunk_size, kernel_block_alignment_size) - attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size) - cache_config.mamba_block_size = attn_block_size - else: - # Without prefix caching, use minimum block size that satisfies - # both backend alignment and mamba page size compatibility - attn_block_size = kernel_block_alignment_size * cdiv( - mamba_page_size, - kernel_block_alignment_size * attn_page_size_1_token, - ) - indexer_align = cls._get_indexer_block_alignment(vllm_config) - if indexer_align: - attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align) + # Use minimum block size that satisfies both backend alignment and + # mamba page size compatibility + attn_block_size = kernel_block_alignment_size * cdiv( + mamba_page_size, + kernel_block_alignment_size * attn_page_size_1_token, + ) + indexer_align = cls._get_indexer_block_alignment(vllm_config) + if indexer_align: + attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align) if cache_config.block_size < attn_block_size: cache_config.block_size = attn_block_size diff --git a/vllm/v1/attention/backends/mamba1_attn.py b/vllm/v1/attention/backends/mamba1_attn.py index 925fceb024..53dc5973fb 100644 --- a/vllm/v1/attention/backends/mamba1_attn.py +++ b/vllm/v1/attention/backends/mamba1_attn.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import Any from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata @@ -42,23 +42,4 @@ class Mamba1AttentionMetadataBuilder( fast_build: bool = False, **kwargs: Any, ) -> Mamba1AttentionMetadata: - common = self._compute_common_metadata(common_attn_metadata) - - if ( - common.num_prefills > 0 - and self.vllm_config.cache_config.mamba_cache_mode == "all" - ): - cu_chunk_seqlen_p, _, last_chunk_indices_p = ( - self._build_chunk_metadata_tensors( - self.kv_cache_spec.block_size, - common, - common_attn_metadata, - ) - ) - return replace( - common, - cu_chunk_seqlen_p=cu_chunk_seqlen_p, - last_chunk_indices_p=last_chunk_indices_p, - ) - - return common + return self._compute_common_metadata(common_attn_metadata) diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index fc71e3485f..e5e3827bcd 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -139,7 +139,6 @@ class Mamba2AttentionMetadataBuilder( common = self._compute_common_metadata( common_attn_metadata, num_accepted_tokens=kwargs.get("num_accepted_tokens"), - prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"), num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"), ) diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 7c543570aa..9953461500 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -39,7 +39,6 @@ class BaseMambaAttentionMetadata: # the batch has no prefill requests. has_initial_states_p: torch.Tensor | None query_start_loc_p: torch.Tensor | None - num_computed_tokens_p: torch.Tensor | None state_indices_tensor_p: torch.Tensor | None # The following tensors are used for decode requests and @@ -52,13 +51,6 @@ class BaseMambaAttentionMetadata: # Includes the bonus token (so minimum is 1) num_accepted_tokens: torch.Tensor | None # shape: [batch,] - # The following tensors are only used for prefix caching in all mode and - # are None if disabled - block_idx_last_scheduled_token: torch.Tensor | None - block_idx_first_scheduled_token_p: torch.Tensor | None - block_idx_last_computed_token: torch.Tensor | None - block_idx_last_scheduled_token_prev_step: torch.Tensor | None - # The following tensor is only used for prefix caching in align mode seq_lens: torch.Tensor @@ -70,9 +62,6 @@ class BaseMambaAttentionMetadata: # last_chunk_indices_p is a tensor of shape (batch,) that contains the # index of the last chunk for every sequence in the (prefill) batch. last_chunk_indices_p: torch.Tensor | None = None - # Number of block-aligned SSM states the prefill path writes, i.e. - # sum(block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p). - num_state_writes_p: int = 0 # The following attributes are for triton implementation of causal_conv1d nums_dict: dict | None = None @@ -130,47 +119,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): self.compilation_config.max_cudagraph_capture_size, ) - if self.vllm_config.cache_config.mamba_cache_mode == "all": - max_num_blocks = ( - cdiv( - self.vllm_config.model_config.max_model_len, - kv_cache_spec.block_size, - ) - + kv_cache_spec.num_speculative_blocks - ) - # TODO: reduce this size as needed for decode-only cudagraph capture - self.state_indices_tensor_d: torch.Tensor = torch.empty( - ( - self.decode_cudagraph_max_bs, - max_num_blocks, - ), - dtype=torch.int32, - device=device, - ) - self.block_idx_last_scheduled_token: torch.Tensor = torch.empty( - (self.decode_cudagraph_max_bs,), - dtype=torch.int32, - device=device, - ) - self.block_idx_last_computed_token: torch.Tensor = torch.empty( - (self.decode_cudagraph_max_bs,), - dtype=torch.int32, - device=device, - ) - if self.use_spec_decode: - self.block_idx_last_scheduled_token_prev_step: torch.Tensor = ( - torch.empty( - (self.decode_cudagraph_max_bs,), - dtype=torch.int32, - device=device, - ) - ) - else: - self.state_indices_tensor_d = torch.empty( - (self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens), - dtype=torch.int32, - device=device, - ) + self.state_indices_tensor_d = torch.empty( + (self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens), + dtype=torch.int32, + device=device, + ) # For speculative decoding, we need to store the following buffers # for CUDA graph capture during decode @@ -259,23 +212,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): if self.num_spec_tokens > 0: num_accepted_tokens = torch.diff(m.query_start_loc) - prev_last_scheduled_idx = None - if ( - self.use_spec_decode - and self.vllm_config.cache_config.mamba_cache_mode == "all" - ): - prev_last_scheduled_idx = torch.zeros( - (m.num_reqs,), - dtype=torch.int32, - device=m.query_start_loc.device, - ) - - return self.build( - 0, - m, - num_accepted_tokens=num_accepted_tokens, - prev_last_scheduled_idx=prev_last_scheduled_idx, - ) + return self.build(0, m, num_accepted_tokens=num_accepted_tokens) def build( self, @@ -284,7 +221,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): fast_build: bool = False, *, num_accepted_tokens: torch.Tensor | None = None, - prev_last_scheduled_idx: torch.Tensor | None = None, num_decode_draft_tokens_cpu: torch.Tensor | None = None, **kwargs: Any, ) -> M: @@ -294,7 +230,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): return self._compute_common_metadata( common_attn_metadata, num_accepted_tokens=num_accepted_tokens, - prev_last_scheduled_idx=prev_last_scheduled_idx, num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, ) @@ -423,56 +358,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): ) return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p - def _compute_prefix_caching_block_indices( - self, - common_attn_metadata: CommonAttentionMetadata, - mamba_block_size: int, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() - # Block index of the last computed token - block_idx_last_computed_token = ( - torch.div( - num_computed_tokens + mamba_block_size - 1, - mamba_block_size, - rounding_mode="floor", - ) - - 1 - ) - # which is <= block index for the first scheduled token - block_idx_first_scheduled_token = ( - torch.div( - num_computed_tokens + mamba_block_size, - mamba_block_size, - rounding_mode="floor", - ) - - 1 - ) - # which is <= block index of the last scheduled token - block_idx_last_scheduled_token = ( - torch.div( - common_attn_metadata.seq_lens + mamba_block_size - 1, - mamba_block_size, - rounding_mode="floor", - ) - - 1 - ) - # -1 in case it's non-computed and causes later issues with indexing - block_idx_last_computed_token.clamp_(min=0) - # -1 in the case we have a padded request (0 seq-len) - block_idx_last_scheduled_token.clamp_(min=0) - - return ( - block_idx_last_computed_token, - block_idx_first_scheduled_token, - block_idx_last_scheduled_token, - ) - def _compute_common_metadata( self, common_attn_metadata: CommonAttentionMetadata, *, num_accepted_tokens: torch.Tensor | None = None, - prev_last_scheduled_idx: torch.Tensor | None = None, num_decode_draft_tokens_cpu: torch.Tensor | None = None, ) -> M: """Compute metadata common to both Mamba1 and Mamba2.""" @@ -540,16 +430,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): has_initial_states_p = None query_start_loc_p = None query_start_loc_d = None - num_computed_tokens = None - num_computed_tokens_p = None - - # for prefix caching - block_idx_first_scheduled_token = None - block_idx_first_scheduled_token_p = None - block_idx_last_computed_token = None - block_idx_last_scheduled_token = None - block_idx_last_scheduled_token_prev_step = None - num_state_writes_p = 0 # for causal_conv1d nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None @@ -557,35 +437,12 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): is_flush_d = None replayssm_scratch = None - if self.vllm_config.cache_config.mamba_cache_mode == "all": - num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() - - # Return a tensor of shape (#requests, #max blocks) - state_indices_tensor = common_attn_metadata.block_table_tensor - # Additional cache-related variables: - mamba_block_size = self.kv_cache_spec.block_size - ( - block_idx_last_computed_token, - block_idx_first_scheduled_token, - block_idx_last_scheduled_token, - ) = self._compute_prefix_caching_block_indices( - common_attn_metadata, mamba_block_size - ) - if self.use_spec_decode and prev_last_scheduled_idx is not None: - fallback = (num_computed_tokens - 1) // mamba_block_size - fallback.clamp_(min=0) - block_idx_last_scheduled_token_prev_step = torch.where( - prev_last_scheduled_idx >= 0, - prev_last_scheduled_idx, - fallback, - ) - else: - state_indices_tensor = mamba_get_block_table_tensor( - common_attn_metadata.block_table_tensor, - common_attn_metadata.seq_lens, - self.kv_cache_spec, - self.vllm_config.cache_config.mamba_cache_mode, - ) + state_indices_tensor = mamba_get_block_table_tensor( + common_attn_metadata.block_table_tensor, + common_attn_metadata.seq_lens, + self.kv_cache_spec, + self.vllm_config.cache_config.mamba_cache_mode, + ) if state_indices_tensor.dim() == 1: state_indices_tensor = state_indices_tensor.unsqueeze(-1) @@ -595,11 +452,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): [num_decodes, num_prefills], dim=0, ) - if self.vllm_config.cache_config.mamba_cache_mode != "all": - state_indices_tensor_d = state_indices_tensor_d[ - :, : 1 + self.num_spec_tokens - ] - state_indices_tensor_p = state_indices_tensor_p[:, 0] + state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens] + state_indices_tensor_p = state_indices_tensor_p[:, 0] if num_decodes > 0 and self.use_spec_decode: query_start_loc_d = common_attn_metadata.query_start_loc[: num_decodes + 1] @@ -613,8 +467,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_accepted_tokens = num_accepted_tokens[:num_decodes] if num_prefills > 0: - if num_computed_tokens is None: - num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() + num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() query_start_loc_p = ( common_attn_metadata.query_start_loc[-num_prefills - 1 :] @@ -636,33 +489,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): ) ) - if self.vllm_config.cache_config.mamba_cache_mode == "all": - assert num_computed_tokens is not None - num_computed_tokens_p = num_computed_tokens[ - num_reqs - num_prefills : num_reqs - ] - assert block_idx_first_scheduled_token is not None - block_idx_first_scheduled_token_p = block_idx_first_scheduled_token[ - num_reqs - num_prefills : num_reqs - ] - - # How many block-aligned states prefill will write: the same - # (block_idx_last - block_idx_first) as above, from CPU data so - # the mixer needs no D2H read. The -1 in both indices cancels. - block_size = self.kv_cache_spec.block_size - seq_lens_p_cpu = seq_lens_cpu[num_reqs - num_prefills : num_reqs] - num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata( - common_attn_metadata, num_reqs, num_prefills, num_decode_tokens - ) - num_state_writes_p = int( - ( - (seq_lens_p_cpu + block_size - 1) // block_size - - (num_computed_tokens_p_cpu + block_size) // block_size - ) - .clamp(min=0) - .sum() - ) - if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0: decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound @@ -764,14 +590,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): replayssm_scratch=replayssm_scratch, num_accepted_tokens=num_accepted_tokens, query_start_loc_d=query_start_loc_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token, - block_idx_first_scheduled_token_p=block_idx_first_scheduled_token_p, - num_state_writes_p=num_state_writes_p, - block_idx_last_computed_token=block_idx_last_computed_token, - block_idx_last_scheduled_token_prev_step=( - block_idx_last_scheduled_token_prev_step - ), - num_computed_tokens_p=num_computed_tokens_p, num_reqs=num_reqs, seq_lens=common_attn_metadata.seq_lens, nums_dict=nums_dict, @@ -791,11 +609,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): state_indices_tensor_d = metadata.state_indices_tensor_d query_start_loc_d = metadata.query_start_loc_d num_accepted_tokens = metadata.num_accepted_tokens - block_idx_last_scheduled_token = metadata.block_idx_last_scheduled_token - block_idx_last_computed_token = metadata.block_idx_last_computed_token - block_idx_last_scheduled_token_prev_step = ( - metadata.block_idx_last_scheduled_token_prev_step - ) write_pos_d = metadata.write_pos_d is_flush_d = metadata.is_flush_d bc_pre_scratch = metadata.bc_pre_scratch @@ -824,44 +637,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): 1 # pad with 1st slot index ) - if self.vllm_config.cache_config.mamba_cache_mode == "all": - assert block_idx_last_scheduled_token is not None - assert block_idx_last_computed_token is not None - self.block_idx_last_scheduled_token[: metadata.num_decodes].copy_( - block_idx_last_scheduled_token[: metadata.num_decodes], - non_blocking=True, - ) - block_idx_last_scheduled_token = self.block_idx_last_scheduled_token[ - :padded_bs - ] - block_idx_last_scheduled_token[metadata.num_decodes :] = 0 - - self.block_idx_last_computed_token[: metadata.num_decodes].copy_( - block_idx_last_computed_token[: metadata.num_decodes], - non_blocking=True, - ) - block_idx_last_computed_token = self.block_idx_last_computed_token[ - :padded_bs - ] - block_idx_last_computed_token[metadata.num_decodes :] = 0 - - if ( - self.use_spec_decode - and block_idx_last_scheduled_token_prev_step is not None - ): - self.block_idx_last_scheduled_token_prev_step[ - : metadata.num_decodes - ].copy_( - block_idx_last_scheduled_token_prev_step[ - : metadata.num_decodes - ], - non_blocking=True, - ) - block_idx_last_scheduled_token_prev_step = ( - self.block_idx_last_scheduled_token_prev_step[:padded_bs] - ) - block_idx_last_scheduled_token_prev_step[metadata.num_decodes :] = 0 - if self.use_replayssm and not self.use_flashinfer_replayssm: assert write_pos_d is not None assert is_flush_d is not None @@ -914,11 +689,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): bc_pre_scratch=bc_pre_scratch, replayssm_scratch=replayssm_scratch, replayssm_state_indices_d=replayssm_state_indices_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token, - block_idx_last_computed_token=block_idx_last_computed_token, - block_idx_last_scheduled_token_prev_step=( - block_idx_last_scheduled_token_prev_step - ), ) def update_block_table( @@ -950,11 +720,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): [metadata.num_decodes, metadata.num_prefills], dim=0, ) - if self.vllm_config.cache_config.mamba_cache_mode != "all": - state_indices_tensor_d = state_indices_tensor_d[ - :, : 1 + self.num_spec_tokens - ] - state_indices_tensor_p = state_indices_tensor_p[:, 0] + state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens] + state_indices_tensor_p = state_indices_tensor_p[:, 0] new_metadata = replace( metadata, diff --git a/vllm/v1/attention/backends/short_conv_attn.py b/vllm/v1/attention/backends/short_conv_attn.py index f2cc3dae99..24bbee228d 100644 --- a/vllm/v1/attention/backends/short_conv_attn.py +++ b/vllm/v1/attention/backends/short_conv_attn.py @@ -283,7 +283,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): has_initial_states_p = None has_initial_states_d = None - num_computed_tokens_p = None # Per-request classification by mask, NOT by position. With # spec-decode, the front decode group can contain both spec-decode @@ -402,10 +401,9 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): assert non_spec_query_start_loc is not None query_start_loc_d = non_spec_query_start_loc[: num_decodes + 1] if num_prefills > 0: - num_computed_tokens_p = num_computed_tokens[ - num_decodes : num_decodes + num_prefills - ] - has_initial_states_p = num_computed_tokens_p > 0 + has_initial_states_p = ( + num_computed_tokens[num_decodes : num_decodes + num_prefills] > 0 + ) assert non_spec_query_start_loc is not None query_start_loc_p = ( non_spec_query_start_loc[num_decodes:] - num_decode_tokens @@ -478,11 +476,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): query_start_loc_d=query_start_loc_d, state_indices_tensor_p=state_indices_tensor_p, state_indices_tensor_d=state_indices_tensor_d, - num_computed_tokens_p=num_computed_tokens_p, - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, seq_lens=m.seq_lens, ) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index c97483912d..eb28fd1e78 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -1162,11 +1162,6 @@ def mamba_get_block_table_tensor( """Get the block table tensor for mamba kernels from the input common_attn_metadata.block_table_tensor given different mamba cache modes. - - "all": input (#requests, cdiv(max_model_len, block_size) - + num_speculative_blocks); - output (#requests, cdiv(max_model_len, block_size) - + num_speculative_blocks). - - "none": input (#requests, 1 + num_speculative_blocks); output (#requests, 1 + num_speculative_blocks). @@ -1174,7 +1169,7 @@ def mamba_get_block_table_tensor( output (#requests, 1 + num_speculative_blocks), which are the last 1 + num_speculative_blocks of each request. """ - if mamba_cache_mode in ("all", "none"): + if mamba_cache_mode == "none": return block_table else: assert isinstance(kv_cache_spec, MambaSpec) diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index cabbbfadc3..248f03dd02 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -1079,12 +1079,7 @@ class MambaSpec(KVCacheSpec): return None def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - if vllm_config.cache_config.mamba_cache_mode == "all": - max_model_len = vllm_config.model_config.max_model_len - return ( - cdiv(max_model_len, self.block_size) + self.num_speculative_blocks - ) * self.page_size_bytes - elif vllm_config.cache_config.mamba_cache_mode == "align": + if vllm_config.cache_config.mamba_cache_mode == "align": return self.page_size_bytes * ( 2 + self.num_speculative_blocks + self.num_prefill_checkpoint_blocks ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 4ccdec2f21..9a0a4eeda9 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -967,11 +967,6 @@ class GPUModelRunner( self.mamba_state_idx: dict[str, int] = {} self._mamba_bufs: mamba_utils.MambaBuffers | None = None self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None - self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None - if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0: - self.mamba_prev_last_scheduled_idx = self._make_buffer( - self.max_num_reqs, dtype=torch.int32 - ) self.layerwise_nvtx_hooks_registered = False def update_max_model_len(self, max_model_len: int) -> None: @@ -1608,17 +1603,6 @@ class GPUModelRunner( assert self.num_accepted_tokens_event is not None self.num_accepted_tokens_event.record() - if self.cache_config.mamba_cache_mode == "all": - mamba_utils.postprocess_mamba_all( - scheduler_output, - self.kv_cache_config, - self.input_batch, - self.requests, - self.mamba_state_idx, - self.num_spec_tokens, - num_reqs, - ) - def _update_streaming_request( self, req_id: str, new_req_data: NewRequestData ) -> CachedRequestState: @@ -2127,15 +2111,6 @@ class GPUModelRunner( self.num_accepted_tokens.np.fill(1) self.num_accepted_tokens.gpu.fill_(1) - if self.mamba_prev_last_scheduled_idx is not None: - mamba_utils.preprocess_mamba_all_specdec( - scheduler_output, - self.input_batch, - self.mamba_state_idx, - num_reqs, - self.mamba_prev_last_scheduled_idx, - ) - # Update num_computed_tokens on GPU. In async spec decode, # CPU values are optimistic (all drafts accepted). The kernel # corrects on GPU using the previous step's @@ -2511,13 +2486,6 @@ class GPUModelRunner( :num_reqs_padded ], ) - if ( - isinstance(builder, Mamba2AttentionMetadataBuilder) - and self.mamba_prev_last_scheduled_idx is not None - ): - extra_attn_metadata_args["prev_last_scheduled_idx"] = ( - self.mamba_prev_last_scheduled_idx.gpu[:num_reqs_padded] - ) if for_cudagraph_capture: attn_metadata_i = builder.build_for_cudagraph_capture( diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index c2e0120e46..33a2b8b480 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -1535,53 +1535,6 @@ def preprocess_mamba( do_mamba_copy_block(copy_bufs) -def postprocess_mamba_all( - scheduler_output: SchedulerOutput, - kv_cache_config: KVCacheConfig, - input_batch: InputBatch, - requests: dict[str, CachedRequestState], - mamba_state_idx: dict[str, int], - num_spec_tokens: int, - num_reqs: int, -): - """All-mode postprocess (only meaningful with num_spec_tokens > 0): - record per-request the block index of the last token scheduled this - step, so the next step can anchor its in-place writes when accepted - drafts leave the sequence at a non-block-aligned position. - """ - if num_spec_tokens <= 0: - return - mamba_groups = get_mamba_groups(kv_cache_config) - block_sizes = {mamba_spec.block_size for mamba_spec in mamba_groups} - assert len(block_sizes) == 1, "all mamba groups must share block_size" - block_size = next(iter(block_sizes)) - full_decode_len = 1 + num_spec_tokens - scheduled = scheduler_output.num_scheduled_tokens - for req_id in input_batch.req_ids[:num_reqs]: - num_query = scheduled.get(req_id, 0) - if num_query == full_decode_len: - req = requests[req_id] - seq_len = req.num_computed_tokens + num_query - mamba_state_idx[req_id] = max(0, (seq_len - 1) // block_size) - else: - mamba_state_idx.pop(req_id, None) - - -def preprocess_mamba_all_specdec( - scheduler_output: SchedulerOutput, - input_batch: InputBatch, - mamba_state_idx: dict[str, int], - num_reqs: int, - prev_last_scheduled_idx_buf: CpuGpuBuffer, -) -> None: - cleanup_mamba_state_idx(scheduler_output, mamba_state_idx) - np_view = prev_last_scheduled_idx_buf.np - for i, req_id in enumerate(input_batch.req_ids[:num_reqs]): - np_view[i] = mamba_state_idx.get(req_id, -1) - np_view[num_reqs:].fill(-1) - prev_last_scheduled_idx_buf.copy_to_gpu() - - def postprocess_mamba_align_gpu( *, bufs: "MambaBuffers",