[Core] Remove deprecated mamba_cache_mode "all" (#58997)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
Jiangyun Zhu
2026-10-01 02:59:37 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Thomas Parnell
parent 08d77cad7b
commit cb6458c5ce
45 changed files with 201 additions and 1476 deletions
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
num_accepted_tokens=None,
query_start_loc_d=None,
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([5]),
)
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
state_indices_tensor_d=state_indices_tensor_d,
num_accepted_tokens=None,
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([1, 1]),
)
-5
View File
@@ -568,15 +568,10 @@ def test_replayssm_mixer_selects_mtp_layout(
num_reqs=2,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
+6 -3
View File
@@ -270,9 +270,6 @@ def test_kda_recoverssm_derivation_is_revalidated():
with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"):
VllmConfig.validate_mamba_cached_kernel(config)
config.use_v2_model_runner = True
config.cache_config.mamba_cache_mode = "all"
with pytest.raises(ValueError, match="only none and align"):
VllmConfig.validate_mamba_cached_kernel(config)
config.cache_config.mamba_cache_mode = "none"
config.model_config.architecture = "NemotronHForCausalLM"
@@ -286,6 +283,12 @@ def test_kda_recoverssm_derivation_is_revalidated():
VllmConfig.validate_mamba_cached_kernel(config)
def test_mamba_cache_mode_all_is_rejected():
"""The removed 'all' mode must fail validation instead of being ignored."""
with pytest.raises(ValidationError, match="mamba_cache_mode"):
CacheConfig(mamba_cache_mode="all")
def test_per_request_spec_decode_metrics_requires_spec_decode():
# The flag only makes sense with speculative decoding configured; enabling
# it without --speculative-config should fail fast rather than silently
@@ -0,0 +1,58 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for BaseMambaAttentionMetadataBuilder batch classification."""
from types import SimpleNamespace
from tests.v1.attention.utils import MockMambaBuilder
from vllm.config.compilation import CUDAGraphMode
def _make_vllm_config(
max_model_len: int,
max_num_seqs: int,
num_speculative_tokens: int = 0,
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection."""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=False,
)
if num_speculative_tokens > 0
else None
)
return SimpleNamespace(
cache_config=SimpleNamespace(
block_size=block_size,
mamba_cache_mode="none",
use_replayssm=False,
replayssm_buffer_len=16,
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.FULL,
max_cudagraph_capture_size=None,
),
speculative_config=speculative_config,
num_speculative_tokens=num_speculative_tokens,
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
model_config=SimpleNamespace(max_model_len=max_model_len),
)
def test_mamba_single_token_prompt_runs_as_prefill():
seq_lens = [8, 9, 1]
config = _make_vllm_config(256, len(seq_lens), block_size=16)
metadata = MockMambaBuilder.build_mamba_metadata(
config,
seq_lens=seq_lens,
query_lens=[1] * len(seq_lens),
is_prefilling=[False, False, True],
)
assert metadata.num_decodes == 2
assert metadata.num_prefills == 1
assert metadata.has_initial_states_p.tolist() == [False]
@@ -1,466 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
When multiple KV cache groups share the same MambaSpec (as in Nemotron
hybrid models), the metadata caching optimization reuses metadata from
an earlier group via update_block_table(). In 'all' mode with CUDA graphs,
update_block_table() must copy block_idx_last_scheduled_token and
block_idx_last_computed_token to the *current* builder's persistent
buffers, otherwise CUDA graph replay reads stale values from uninitialized
buffers.
"""
from types import SimpleNamespace
import torch
from tests.v1.attention.utils import MockMambaBuilder
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadata
from vllm.v1.kv_cache_interface import MambaSpec
def _make_vllm_config(
max_model_len: int,
max_num_seqs: int,
num_speculative_tokens: int = 0,
block_size: int | None = None,
):
"""Create a minimal mock VllmConfig with only the fields the builder
accesses, avoiding any model download / HF config inspection."""
speculative_config = (
SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=False,
)
if num_speculative_tokens > 0
else None
)
return SimpleNamespace(
cache_config=SimpleNamespace(
block_size=block_size,
mamba_cache_mode="all",
use_replayssm=False,
replayssm_buffer_len=16,
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.FULL,
max_cudagraph_capture_size=None,
),
speculative_config=speculative_config,
num_speculative_tokens=num_speculative_tokens,
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
model_config=SimpleNamespace(max_model_len=max_model_len),
)
def test_mamba_single_token_prompt_runs_as_prefill():
seq_lens = [8, 9, 1]
config = _make_vllm_config(256, len(seq_lens), block_size=16)
metadata = MockMambaBuilder.build_mamba_metadata(
config,
seq_lens=seq_lens,
query_lens=[1] * len(seq_lens),
is_prefilling=[False, False, True],
)
assert metadata.num_decodes == 2
assert metadata.num_prefills == 1
assert metadata.has_initial_states_p.tolist() == [False]
def test_update_block_table_copies_block_idx_to_persistent_buffers():
"""update_block_table() must write block_idx tensors to the current
builder's persistent buffers, not leave them pointing to a different
builder's buffers."""
block_size = 16
max_model_len = 256
num_reqs = 4
device = torch.device("cpu")
vllm_config = _make_vllm_config(max_model_len, num_reqs)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
)
# Two builders simulating two KV cache groups with the same MambaSpec.
builder_a = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
builder_b = MockMambaBuilder(spec, ["layer1"], vllm_config, device)
# Sanity: each builder has its own persistent buffer.
assert (
builder_a.block_idx_last_scheduled_token.data_ptr()
!= builder_b.block_idx_last_scheduled_token.data_ptr()
)
# Construct decode-only metadata as if builder_a.build() produced it.
max_blocks = max_model_len // block_size
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
block_idx_vals = (seq_lens - 1) // block_size # [3, 3, 3, 3]
builder_a.block_idx_last_scheduled_token[:num_reqs].copy_(block_idx_vals)
builder_a.block_idx_last_computed_token[:num_reqs].copy_(block_idx_vals)
metadata_a = BaseMambaAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_reqs,
num_decode_tokens=num_reqs,
num_reqs=num_reqs,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
query_start_loc_d=None,
num_accepted_tokens=None,
state_indices_tensor_d=builder_a.state_indices_tensor_d[:num_reqs],
block_idx_last_scheduled_token=(
builder_a.block_idx_last_scheduled_token[:num_reqs]
),
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=(
builder_a.block_idx_last_computed_token[:num_reqs]
),
block_idx_last_scheduled_token_prev_step=None,
seq_lens=seq_lens,
)
# Call update_block_table on builder_b (simulates the metadata caching
# optimization reusing metadata from builder_a's group).
blk_table = torch.randint(
0, 100, (num_reqs, max_blocks), dtype=torch.int32, device=device
)
slot_mapping = torch.zeros(num_reqs, dtype=torch.int64, device=device)
metadata_b = builder_b.update_block_table(metadata_a, blk_table, slot_mapping)
# block_idx tensors must live in builder_b's persistent buffers.
def shares_storage(tensor, buffer):
return (
tensor.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr()
)
assert shares_storage(
metadata_b.block_idx_last_scheduled_token,
builder_b.block_idx_last_scheduled_token,
), "block_idx_last_scheduled_token not in builder_b's persistent buffer"
assert shares_storage(
metadata_b.block_idx_last_computed_token,
builder_b.block_idx_last_computed_token,
), "block_idx_last_computed_token not in builder_b's persistent buffer"
# Must NOT point to builder_a's buffers.
assert not shares_storage(
metadata_b.block_idx_last_scheduled_token,
builder_a.block_idx_last_scheduled_token,
), "block_idx_last_scheduled_token still points to builder_a's buffer"
# Values must be correct (copied from metadata_a).
torch.testing.assert_close(
metadata_b.block_idx_last_scheduled_token,
block_idx_vals,
)
torch.testing.assert_close(
metadata_b.block_idx_last_computed_token,
block_idx_vals,
)
def test_state_indices_tensor_d_includes_num_speculative_blocks():
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
the cudagraph buffer for state_indices_tensor_d must allocate the same
per-request column count as the runtime block table, which includes
num_speculative_blocks trailing scratch columns."""
block_size = 16
max_model_len = 256
max_num_seqs = 4
num_speculative_tokens = 1
num_speculative_blocks = 2
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=num_speculative_blocks,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
expected_cols = (max_model_len // block_size) + num_speculative_blocks
assert builder.state_indices_tensor_d.shape == (max_num_seqs, expected_cols)
def test_block_idx_cudagraph_capture_padded_by_num_reqs():
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
Past num_decodes, the slice must be zero-filled."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
num_speculative_tokens = 1
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=2,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
builder.block_idx_last_scheduled_token.fill_(-1)
builder.block_idx_last_computed_token.fill_(-1)
num_decodes = 2
num_reqs = 3
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
state_indices_d = torch.zeros(
(num_decodes, builder.state_indices_tensor_d.shape[1]),
dtype=torch.int32,
device=device,
)
query_start_loc_d = torch.arange(
num_decodes + 1, dtype=torch.int32, device=device
) * (1 + num_speculative_tokens)
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
metadata = BaseMambaAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_reqs=num_reqs,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=state_indices_d,
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=num_accepted_tokens,
block_idx_last_scheduled_token=block_idx_vals,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=block_idx_vals,
block_idx_last_scheduled_token_prev_step=None,
seq_lens=seq_lens,
)
out = builder._update_metadata_for_cudagraph_capture(metadata)
assert out.block_idx_last_scheduled_token.shape == (num_reqs,)
assert out.block_idx_last_computed_token.shape == (num_reqs,)
torch.testing.assert_close(
out.block_idx_last_scheduled_token[:num_decodes], block_idx_vals
)
torch.testing.assert_close(
out.block_idx_last_computed_token[:num_decodes], block_idx_vals
)
assert torch.all(out.block_idx_last_scheduled_token[num_decodes:] == 0)
assert torch.all(out.block_idx_last_computed_token[num_decodes:] == 0)
def test_block_idx_prev_step_persistent_buffer_allocated():
"""With mamba_cache_mode='all' + spec decode, the builder must allocate
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
same shape as the existing block_idx_last_{scheduled,computed}_token
buffers, so cudagraph capture records a stable pointer for the prev-step
input anchor consumed by mamba_mixer2's input gather."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
num_speculative_tokens = 1
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=2,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
assert hasattr(builder, "block_idx_last_scheduled_token_prev_step")
assert builder.block_idx_last_scheduled_token_prev_step.shape == (max_num_seqs,)
assert builder.block_idx_last_scheduled_token_prev_step.dtype == torch.int32
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
"""Without spec decode, the prev-step buffer is unused and must not be
allocated — the input anchor reduces to last_computed_token."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len, max_num_seqs, num_speculative_tokens=0
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
assert not hasattr(builder, "block_idx_last_scheduled_token_prev_step")
def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
into the builder's persistent buffer (so cudagraph replay reads from the
same underlying memory), pad past num_decodes with zero, and return a
slice of the persistent buffer in the metadata."""
block_size = 16
max_model_len = 256
max_num_seqs = 8
num_speculative_tokens = 1
device = torch.device("cpu")
vllm_config = _make_vllm_config(
max_model_len,
max_num_seqs,
num_speculative_tokens=num_speculative_tokens,
)
spec = MambaSpec(
block_size=block_size,
shapes=((1,), (1,)),
dtypes=(torch.float32,),
mamba_cache_mode="all",
num_speculative_blocks=2,
)
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
builder.block_idx_last_scheduled_token.fill_(-1)
builder.block_idx_last_computed_token.fill_(-1)
builder.block_idx_last_scheduled_token_prev_step.fill_(-1)
num_decodes = 2
num_reqs = 3
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
prev_step_vals = torch.tensor([2, 4], dtype=torch.int32, device=device)
state_indices_d = torch.zeros(
(num_decodes, builder.state_indices_tensor_d.shape[1]),
dtype=torch.int32,
device=device,
)
query_start_loc_d = torch.arange(
num_decodes + 1, dtype=torch.int32, device=device
) * (1 + num_speculative_tokens)
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
metadata = BaseMambaAttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_reqs=num_reqs,
has_initial_states_p=None,
query_start_loc_p=None,
num_computed_tokens_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=state_indices_d,
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=num_accepted_tokens,
block_idx_last_scheduled_token=block_idx_vals,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=block_idx_vals,
block_idx_last_scheduled_token_prev_step=prev_step_vals,
seq_lens=seq_lens,
)
out = builder._update_metadata_for_cudagraph_capture(metadata)
# Output field exists and is identity-shared with the persistent buffer.
assert out.block_idx_last_scheduled_token_prev_step is not None
assert (
out.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
== builder.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
), (
"prev-step buffer must live in the builder's persistent buffer, not "
"in the caller-provided tensor"
)
# Padded by num_reqs (not num_decode_tokens) — same fix as bug 2 for the
# other block_idx_* fields.
assert out.block_idx_last_scheduled_token_prev_step.shape == (num_reqs,)
# First num_decodes values: input values copied through.
torch.testing.assert_close(
out.block_idx_last_scheduled_token_prev_step[:num_decodes],
prev_step_vals,
)
# Tail values past num_decodes: zero-filled padding for cudagraph capture.
assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0)
def test_num_state_writes_p_matches_block_idx_tensors():
"""The prefill state save sizes its batched copy from
num_state_writes_p instead of reading the block index tensors back from
the GPU, so the host count must equal what those tensors say."""
# (block_size, seq_lens, query_lens, is_prefilling)
cases = [
# block-aligned and unaligned computed tokens, and a chunked prefill
(16, [64, 33, 17], [48, 17, 1], [True, True, True]),
(32, [200, 100, 64], [200, 36, 64], [True, True, True]),
# decode rows ahead of the prefills
(16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]),
# prefills that stay inside their first block: nothing to write
(16, [65, 66, 67], [1, 2, 3], [True, True, True]),
]
for block_size, seq_lens, query_lens, is_prefilling in cases:
metadata = MockMambaBuilder.build_mamba_metadata(
_make_vllm_config(4096, len(seq_lens), block_size=block_size),
seq_lens=seq_lens,
query_lens=query_lens,
is_prefilling=is_prefilling,
)
num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills
block_idx_last_p = metadata.block_idx_last_scheduled_token[
num_reqs - num_prefills : num_reqs
]
expected = int(
(block_idx_last_p - metadata.block_idx_first_scheduled_token_p)
.clamp(min=0)
.sum()
)
assert metadata.num_state_writes_p == expected, (
f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}"
)
+38 -38
View File
@@ -39,6 +39,7 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.utils.hashing import sha256
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
from vllm.v1.core.sched.diffusion_scheduler import (
DiffusionAsyncScheduler,
@@ -6751,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler(
num_blocks: int = 100,
supports_divergent_hits: bool = True,
) -> Scheduler:
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
"""FA + Mamba ("align" cache mode) scheduler with a MockKVConnector."""
model_config = ModelConfig(
model="facebook/opt-125m",
trust_remote_code=True,
@@ -6772,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler(
cache_config=CacheConfig(
block_size=block_size,
enable_prefix_caching=True,
mamba_cache_mode="all",
mamba_cache_mode="align",
),
kv_transfer_config=KVTransferConfig(
kv_connector="MockKVConnector",
@@ -6804,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
mamba_cache_mode="align",
),
),
],
@@ -6820,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler(
)
def _seed_hybrid_prefix(
manager: KVCacheManager, num_blocks: int, block_size: int
) -> tuple[list[int], list[int]]:
"""Prefill a prefix one block per step so that align-mode Mamba caches
the state at every block boundary. Returns the FA and Mamba block ids."""
[fill] = create_requests(
num_requests=1,
num_tokens=num_blocks * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
assert num_computed == 0
mamba_ids = []
for i in range(num_blocks):
manager.allocate_slots(
fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None
)
fill.num_computed_tokens += block_size
fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id)
mamba_ids.append(group_mamba_ids[i])
manager.free(fill)
return fa_ids, mamba_ids
@pytest.mark.parametrize(
"matched_tokens,expected_num_computed",
[
@@ -6845,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
manager = scheduler.kv_cache_manager
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix so both groups cache all four boundaries
# (mamba cache mode "all" caches every block's state densely).
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
fa_ids = [b.block_id for b in blocks.blocks[0]]
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
# Seed a 4-block prefix so both groups cache all four boundaries.
fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
# Evict the FA tail and the middle mamba states; block 0 (both groups)
# and the deep mamba state at block 3 survive.
@@ -6885,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
[new_req] = output.scheduled_new_reqs
assert new_req.num_computed_tokens == expected_num_computed
@pytest.mark.parametrize(
@@ -6921,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
# Seed a 4-block prefix in both groups.
[fill] = create_requests(
num_requests=1,
num_tokens=4 * block_size,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["fill"],
)
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
blocks = manager.allocate_slots(
fill, fill.num_tokens, num_computed, computed_blocks
)
mamba_ids = [b.block_id for b in blocks.blocks[1]]
manager.free(fill)
_, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
@@ -6955,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
scheduler.add_request(replay)
output = scheduler.schedule()
num_scheduled = output.num_scheduled_tokens[replay.request_id]
assert replay.num_tokens - num_scheduled == expected_num_computed
[new_req] = output.scheduled_new_reqs
assert new_req.num_computed_tokens == expected_num_computed
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
@@ -60,7 +60,7 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -78,7 +78,7 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -67,7 +67,6 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
self._mamba_group_ids: list[int] = []
self._attn_group_ids: list[int] = [0]
self._num_ssm_scratch_blocks = 0
self._ssm_state_slots_are_positional = False
self._is_hma_required = False
self.kv_cache_config = SimpleNamespace(
kv_cache_groups=[None, None],
@@ -130,7 +129,7 @@ def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0):
block_size=16,
shapes=shapes,
dtypes=(torch.float32,) * num_states,
mamba_cache_mode="all",
mamba_cache_mode="align",
num_speculative_blocks=num_speculative_blocks,
)
@@ -146,7 +145,7 @@ def _gate_vllm_config(
block_size: int = 16,
num_lookahead_tokens: int = 0,
num_speculative_tokens: int = 0,
mamba_cache_mode: str = "all",
mamba_cache_mode: str = "align",
dcp_size: int = 1,
):
return SimpleNamespace(
@@ -193,7 +192,7 @@ def test_split_block_groups_ignores_transfer_disabled_group():
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
mamba_cache_mode="align",
)
config = SimpleNamespace(
kv_cache_groups=[
@@ -361,7 +360,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
block_size=4,
num_lookahead_tokens=7,
num_speculative_tokens=7,
mamba_cache_mode="align",
dcp_size=dcp_size,
)
scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config)
@@ -398,21 +396,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
)
def test_scheduler_rejects_dspark_with_positional_mamba_cache():
"""Positional mode counts lookahead in the Mamba allocation, so the
one-block recurrent-state tail bound no longer holds."""
with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(
speculative_config=_spec_config("dspark"),
block_size=4,
mamba_cache_mode="all",
),
"engine",
_dspark_wrapped_attention_config(),
)
def test_scheduler_rejects_hybrid_write():
config = SimpleNamespace(
kv_cache_groups=[
@@ -441,20 +424,7 @@ def test_split_block_groups_accepts_empty_abort_payload():
assert sched.split_block_groups(()) == ([], [])
def test_split_block_groups_keeps_positional_slots_in_all_mode():
# mamba_cache_mode="all" keeps a state per block position.
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_ssm_state_slots_are_positional=True,
)
attn, mamba = sched.split_block_groups(([1], [40, 41, 42]))
assert attn == [1]
assert mamba == [[40, 41, 42]]
def test_split_block_groups_keeps_only_running_state_outside_all_mode():
def test_split_block_groups_keeps_only_running_state():
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
@@ -464,24 +434,21 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode():
@pytest.mark.parametrize(
"positional,blocks,expected",
"blocks,expected",
[
# Trailing scratch slots go in either cache mode.
(True, [40, 41, 42], [40]),
# Outside "all" the running state is picked only after they are gone.
(False, [40, 41, 42, 43], [41]),
# The running state is picked only after the scratch slots are gone.
([40, 41, 42, 43], [41]),
# Never empty: a list shorter than the scratch count keeps one state.
(True, [40], [40]),
(False, [], []),
([40], [40]),
([], []),
],
)
def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected):
def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected):
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_num_ssm_scratch_blocks=2,
_ssm_state_slots_are_positional=positional,
)
assert sched.split_block_groups(([1], blocks)) == ([1], [expected])
@@ -555,7 +522,6 @@ def _make_read_scheduler():
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_ssm_state_slots_are_positional=True,
request_id_to_transfer_id={},
transfer_id_to_request_id={},
_reqs_need_recv={},
@@ -578,17 +544,17 @@ def _make_read_request(remote_block_ids):
def test_update_state_drops_decode_recompute_tail_block():
sched = _make_read_scheduler()
request = _make_read_request([[10, 11], [90, 91]])
request = _make_read_request([[10, 11], [91]])
blocks = _FakeBlocks(
all_groups=([100, 101, 102], [200, 201, 202]),
)
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]]
assert sched._reqs_need_recv["req"][1] == [[100, 101], [202]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [
[10, 11],
[90, 91],
[91],
]
@@ -609,7 +575,6 @@ def test_update_state_pairs_trimmed_mamba_state_with_remote_state():
"""The DSpark shape end to end: scratch slots are stripped, the running
state is then selected, and the attention list loses its lookahead tail."""
sched = _make_read_scheduler()
sched._ssm_state_slots_are_positional = False
sched._num_ssm_scratch_blocks = 2
sched._max_decode_tail_blocks = 2
request = _make_read_request([[10, 11], [90]])
@@ -856,28 +856,6 @@ def test_apply_prefix_caching_mamba_hybrid(
[[6, 7, 8, 9], [99]],
id="fa_prefix_hit_and_ssm_trim",
),
# Multi-slot SSM ("all" mode): a local prefix hit leaves fewer local
# slots; the earlier remote slots are covered locally → remote tail.
pytest.param(
10,
10,
[list(range(10)), [5, 6]],
[list(range(10)), [1, 2, 3]],
[list(range(10)), [5, 6]],
[list(range(10)), [2, 3]],
id="ssm_multi_block_local_hit_tail",
),
# Multi-slot SSM ("all" mode): the one trailing local position holds
# the token D recomputes itself → local head-clip.
pytest.param(
10,
10,
[list(range(10)), [4, 5, 6]],
[list(range(10)), [8, 9]],
[list(range(10)), [4, 5]],
[list(range(10)), [8, 9]],
id="ssm_multi_block_local_extra_head_clip",
),
],
)
def test_apply_prefix_caching_ssm_prefix_cache_hit(
@@ -2034,13 +2012,12 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_ssm_single_state():
"""In single-state cache modes, SSM lists are reduced to the running
state slot: speculative scratch slots, null placeholders and the previous
step's state carry nothing. Attention groups pass through untouched."""
"""SSM lists are reduced to the running state slot: speculative scratch
slots, null placeholders and the previous step's state carry nothing.
Attention groups pass through untouched."""
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
sched.blocks_per_sw = [0, 0]
sched._ssm_spec_blocks = [None, 2]
sched._ssm_state_slots_are_positional = False
# Align-mode list: null placeholders, state block, 2 speculative slots.
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 0, 7, 8, 9]))
@@ -2069,19 +2046,6 @@ def test_exchange_clipped_blocks_ssm_single_state():
assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],)
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_ssm_positional_states():
"""In "all" mode every position holds a state, so only the speculative
slots go; placeholders stay to keep the list position-indexed."""
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
sched.blocks_per_sw = [0, 0]
sched._ssm_spec_blocks = [None, 2]
sched._ssm_state_slots_are_positional = True
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 5, 6, 7, 8, 9]))
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
@@ -566,10 +566,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
("state-spaces/mamba-1.4b-hf", 16, 1),
],
)
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
def test_mamba_cpu_offload_boundary(
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
):
def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int):
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
@@ -586,7 +583,7 @@ def test_mamba_cpu_offload_boundary(
kv_transfer_config=kv_transfer_config,
language_model_only=True,
enable_prefix_caching=True,
mamba_cache_mode=mamba_cache_mode,
mamba_cache_mode="align",
# Use lossless state storage so this exact-equality regression isolates
# offload boundary selection from low-precision Mamba checkpointing.
mamba_cache_dtype="float32",
+1 -1
View File
@@ -474,7 +474,7 @@ def make_kv_cache_config(
mamba_enabled: bool = False,
sw_size: int = 128,
num_blocks: int = 100,
mamba_cache_mode: Literal["all", "align", "none"] = "none",
mamba_cache_mode: Literal["align", "none"] = "none",
) -> KVCacheConfig:
kv_cache_groups = [
KVCacheGroupSpec(
+1 -50
View File
@@ -2335,7 +2335,7 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
# of the hash block, so it cannot exceed the smallest of them.
hash_block_size=BLOCK_SIZE,
dcp_world_size=dcp_world_size,
mamba_cache_mode="all",
mamba_cache_mode="none",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
@@ -2747,55 +2747,6 @@ def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None:
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
def test_eager_store_scans_mamba_all_blocks_positionally() -> None:
"""Mamba all-mode retains positional identity and needs no handoff."""
block_size = 4 * BLOCK_SIZE
fix = _make_hybrid_attention_mamba_scheduler(
num_cpu_blocks=16,
num_gpu_blocks=24,
block_size=block_size,
mamba_cache_mode="all",
)
sched = fix.scheduler
gpu_pool = fix.gpu_block_pool
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
assert attention_manager.has_positionally_stable_blocks
assert mamba_manager.has_positionally_stable_blocks
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
attention_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
)
mamba_blocks = _allocate_cp_gpu_blocks(
gpu_pool, req, 2, virtual_block_size=block_size, group_id=1
)
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
# build_connector_meta runs before the scheduler commits this step's
# tokens, so the positional scan picks the blocks up on the next step.
first = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 2 * block_size},
new_reqs={req.request_id: kv_blocks.get_block_ids()},
)
)
assert first.store_gpu_blocks == []
req.num_computed_tokens = 2 * block_size
meta = sched.build_connector_meta(
make_scheduler_output(
{req.request_id: 1},
cached_req_new_blocks={req.request_id: None},
)
)
assert set(meta.store_gpu_blocks) == {
*(block.block_id for block in attention_blocks),
*(block.block_id for block in mamba_blocks),
}
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2]
@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"])
def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None:
"""Handoffs for a request leaving this step must not be read.
+2 -10
View File
@@ -189,7 +189,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
@pytest.mark.parametrize("mamba_cache_mode", ["none", "all", "align"])
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
"""Mamba groups hold the running-state block plus the speculative tail.
@@ -227,14 +227,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
"""Attention, circular, and Mamba groups exercise every reservation branch.
"none" and "all" reach the same branch of both `_reserved_block_count` and
`MambaManager`, which tests only for "align". They are still both listed:
the mode is a spec-level input, and having the real manager confirm the
prediction for each is what keeps a future divergence between them from
landing unnoticed.
"""
"""Attention, circular, and Mamba groups exercise every reservation branch."""
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[],
@@ -242,7 +235,6 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
_attention_group(),
_circular_group(),
_mamba_group("none"),
_mamba_group("all"),
_mamba_group("align"),
],
)
+1 -1
View File
@@ -398,7 +398,7 @@ def _make_kv_cache_config(cfg: _TestConfig, layer_names: list[str]) -> KVCacheCo
(cfg.temporal_state_dim,),
),
dtypes=(cfg.dtype, cfg.dtype),
mamba_cache_mode="all",
mamba_cache_mode="align",
)
group = KVCacheGroupSpec(
layer_names=layer_names,
+1 -18
View File
@@ -59,7 +59,7 @@ CacheDType = Literal[
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
MambaCacheMode = Literal["all", "align", "none"]
MambaCacheMode = Literal["align", "none"]
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
KVOffloadingBackend = Literal["native", "lmcache"]
@@ -75,8 +75,6 @@ class CacheConfig:
Accepts None (meaning "use default"). After construction, always int."""
user_specified_block_size: bool = field(default=False, init=False)
"""Whether block_size was explicitly provided. Derived automatically."""
user_specified_mamba_block_size: bool = field(default=False, init=False)
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
kv_cache_layout: str | None = field(default=None, init=False)
"""Resolved physical KV cache layout name (a ``KVCacheLayout`` member).
@@ -191,7 +189,6 @@ class CacheConfig:
"""The cache strategy for Mamba layers:
- "none": set when prefix caching is disabled.
- "all": cache the mamba state of all tokens at position i * block_size.
- "align": only cache the mamba state of the last token of each scheduler step and
when the token is at position i * block_size. This is the default when prefix
caching is enabled.
@@ -293,7 +290,6 @@ class CacheConfig:
"mamba_page_size_padded",
"skip_page_size_padded",
"user_specified_block_size",
"user_specified_mamba_block_size",
"_block_size_resolved",
# Post-init/derived counters
"num_gpu_blocks",
@@ -337,21 +333,8 @@ class CacheConfig:
self.block_size = self.DEFAULT_BLOCK_SIZE
else:
self.user_specified_block_size = True
if self.mamba_block_size is not None:
self.user_specified_mamba_block_size = True
return self
@field_validator("mamba_cache_mode", mode="after")
@classmethod
def _validate_mamba_cache_mode(cls, mode: MambaCacheMode) -> MambaCacheMode:
if mode == "all":
logger.warning_once(
"Mamba cache mode 'all' is deprecated and will be removed in an "
"upcoming release. If this is a problem, please open an issue "
"at https://github.com/vllm-project/vllm/issues."
)
return mode
@field_validator("cache_dtype", mode="after")
@classmethod
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
-12
View File
@@ -3091,9 +3091,6 @@ class VllmConfig:
if self.parallel_config.use_ubatching:
unsupported.extend(self._get_dbo_unsupported_features())
if self.cache_config.mamba_cache_mode == "all":
unsupported.append("mamba cache mode 'all'")
return unsupported
def _get_v1_model_runner_unsupported_features(self) -> list[str]:
@@ -3499,10 +3496,6 @@ class VllmConfig:
"SSM state caches, not --enable-mamba-cache-stochastic-"
"rounding, which requires an explicit float16 cache"
)
if self.cache_config.mamba_cache_mode not in ("none", "align"):
raise ValueError(
"RecoverSSM supports only none and align Mamba cache modes"
)
if (
self.cache_config.mamba_cache_mode == "align"
and not self.use_v2_model_runner
@@ -3535,11 +3528,6 @@ class VllmConfig:
"Mamba2 ReplaySSM speculative decoding requires "
"--mamba-backend flashinfer"
)
elif self.cache_config.mamba_cache_mode == "all":
raise ValueError(
"--use-replayssm supports prefix caching only in align mode; "
"pass --mamba-cache-mode align"
)
elif self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
if self.cache_config.mamba_cache_mode == "align":
raise ValueError(
@@ -33,16 +33,14 @@ EngineId = str
BlockIds = tuple[list[int], ...] | list[list[int]]
def clip_ssm_state_blocks(
blocks: list[int], num_spec_blocks: int, positional: bool
) -> list[int]:
"""Drop speculative scratch slots, retaining position-indexed or running state."""
def clip_ssm_state_blocks(blocks: list[int], num_spec_blocks: int) -> list[int]:
"""Drop speculative scratch slots and keep only the running-state slot."""
if not blocks:
return blocks
# Keep at least the running-state slot, even before scratch allocation.
if num_scratch := min(num_spec_blocks, len(blocks) - 1):
blocks = blocks[:-num_scratch]
return blocks if positional else blocks[-1:]
return blocks[-1:]
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
@@ -450,13 +450,6 @@ def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
f"method={speculative_config.method!r}"
)
if vllm_config.cache_config.mamba_cache_mode == "all":
# Positional allocation counts lookahead, so its local tail can exceed
# the single running-state block supported by hybrid READ.
raise MoRIIOError(
"MoRIIO hybrid READ does not support DSpark with "
"mamba_cache_mode='all'; use 'align'"
)
def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
@@ -521,11 +514,6 @@ class MoRIIOConnectorScheduler:
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
"mode only; set kv_connector_extra_config.read_mode=true"
)
# Only "all" mode keeps a state per block position; the other modes keep
# a single running state in the last slot.
self._ssm_state_slots_are_positional = (
vllm_config.cache_config.mamba_cache_mode == "all"
)
self._num_ssm_scratch_blocks = (
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
)
@@ -1288,8 +1276,8 @@ class MoRIIOConnectorScheduler:
"""Select transferable attention and Mamba block groups.
The wire payload stores the attention group first, followed by every
Mamba group in transfer-group order. Outside ``mamba_cache_mode="all"``,
only each group's running-state slot is transferred.
Mamba group in transfer-group order. Only each group's running-state
slot is transferred.
"""
if not block_ids:
return [], []
@@ -1301,7 +1289,6 @@ class MoRIIOConnectorScheduler:
clip_ssm_state_blocks(
list(transfer_block_ids[group_id]),
self._num_ssm_scratch_blocks,
self._ssm_state_slots_are_positional,
)
for group_id in self._mamba_group_ids
]
@@ -154,11 +154,6 @@ class NixlBaseConnectorScheduler:
else None
for g in kv_cache_config.transfer_groups
]
# Only "all" mode keeps a state per block position; the other modes
# keep a single running state in the last non-speculative slot.
self._ssm_state_slots_are_positional = (
vllm_config.cache_config.mamba_cache_mode == "all"
)
# Threshold to decide whether to compute kv cache locally
# or pull from a remote node: minimum number of remote
@@ -258,11 +253,9 @@ class NixlBaseConnectorScheduler:
out-of-window blocks only prior to the `request_finished_all_groups`
hook.
SSM groups keep only their state-bearing slots: the trailing
speculative scratch slots always go, and in single-state cache modes
so does everything before the running state (null placeholders and
the previous step's superseded state). "all" mode keeps its remaining
slots, which the worker pairs position-wise.
SSM groups keep only their state-bearing slot: the trailing
speculative scratch slots and everything before the running state
(null placeholders and the previous step's superseded state) go.
Use this at every block-id exchange point. Pass ``clip_ssm=False``
for per-step partial lists (host-buffer save), where the SSM strip
@@ -289,9 +282,7 @@ class NixlBaseConnectorScheduler:
and blocks
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
):
blocks = clip_ssm_state_blocks(
blocks, n_spec_blocks, self._ssm_state_slots_are_positional
)
blocks = clip_ssm_state_blocks(blocks, n_spec_blocks)
clipped.append(blocks)
return tuple(clipped)
@@ -3510,13 +3510,12 @@ class NixlBaseConnectorWorker:
if _is_ssm_spec(self._group_spec_types[i]):
if num_decode_blocks == num_prefill_blocks:
continue
# Only state-bearing slots reach here, single-state modes
# just one (see get_exchange_clipped_blocks), so differing
# counts mean position-indexed "all"-mode lists. A longer
# prefill list carries earlier positions the decode side
# already has (prefix hit) -> take its tail; a longer decode
# list holds the position D recomputes itself, which gets
# no prefill state.
# Only state-bearing slots reach here, normally just one
# (see get_exchange_clipped_blocks). A longer prefill list
# carries earlier positions the decode side already has
# (prefix hit) -> take its tail; a longer decode list holds
# the position D recomputes itself, which gets no prefill
# state.
assert num_decode_blocks - num_prefill_blocks <= 1, (
f"Group {i}: unpairable SSM state slots, "
f"decode={num_decode_blocks} prefill={num_prefill_blocks}"
@@ -156,7 +156,7 @@ def resolve_mamba_align_size(
"""Scan all KV cache groups in *spec* and return the single mamba alignment
size, or None if no group requires mamba alignment.
For MambaSpec groups in "align" or "all" cache mode the hit window must be
For MambaSpec groups in "align" cache mode the hit window must be
rounded down to a multiple of the offloaded chunk size. Asserts that all
such groups agree on the same value.
"""
@@ -164,10 +164,7 @@ def resolve_mamba_align_size(
for group in spec.config.groups:
tokens_per_block = group.tokens_per_block
kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in (
"align",
"all",
):
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
mamba_align_size = tokens_per_chunk
@@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer):
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
has_prefill = num_prefill_tokens > 0
has_decode = num_decode_tokens > 0
num_actual_tokens = num_prefill_tokens + num_decode_tokens
@@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer):
gate_p = prefill_decode_split.gate_p
gate_d = prefill_decode_split.gate_d
if is_mamba_cache_all:
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_d = None
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
num_computed_tokens_p = None
ssm_outputs = []
if has_prefill:
@@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer):
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
query_start_loc=query_start_loc_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
)
# 3. State Space Model sequence transformations.
@@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer):
cache_indices=state_indices_tensor_p,
has_initial_state=has_initial_states_p,
query_start_loc=query_start_loc_p,
block_size=mamba_block_size,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
cu_chunk_seqlen=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
)
@@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer):
# state_indices_tensor_d is assigned when attn_metadata is not None,
# and has_decode is only True when attn_metadata is not None
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
# 2. Convolution sequence transformation
conv_out_d = causal_conv1d_update(
hidden_states_BC_d.transpose(0, 1),
@@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
).transpose(0, 1)
# 3. State Space Model sequence transformation.
@@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer):
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=scan_outputs_d,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
+17 -182
View File
@@ -549,12 +549,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
self.register_buffer(
"_decode_state_offsets",
torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0),
persistent=False,
)
# Pre-compute sizes for forward pass
self.tped_intermediate_size = self.intermediate_size // self.tp_size
@@ -732,8 +726,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata_raw = forward_context.attn_metadata
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
use_spec_decode = self.num_spec > 0
ring_start = prev_num_accepted = prev_query_len = None
@@ -803,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
dim=0,
)
if is_mamba_cache_all:
# If prefix caching is enabled, retrieve the relevant variables
# for prefill and decode
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
torch.split(
attn_metadata.block_idx_last_computed_token,
[num_decodes, num_prefills],
dim=0,
)
)
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
torch.split(
attn_metadata.block_idx_last_scheduled_token,
[num_decodes, num_prefills],
dim=0,
)
)
if attn_metadata.block_idx_last_scheduled_token_prev_step is not None:
block_idx_last_scheduled_token_prev_step_d, _ = torch.split(
attn_metadata.block_idx_last_scheduled_token_prev_step,
[num_decodes, num_prefills],
dim=0,
)
else:
block_idx_last_scheduled_token_prev_step_d = None
# Prefill-only variables:
block_idx_first_scheduled_token_p = (
attn_metadata.block_idx_first_scheduled_token_p
)
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
else:
block_idx_last_computed_token_p = None
block_idx_last_scheduled_token_p = None
block_idx_first_scheduled_token_p = None
block_idx_last_scheduled_token_d = None
block_idx_last_computed_token_d = None
block_idx_last_scheduled_token_prev_step_d = None
num_computed_tokens_p = None
preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split(
output[:num_actual_tokens],
[num_decode_tokens, num_prefill_tokens],
@@ -858,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# In addition, "block_idx_first_scheduled_token_p" and
# "block_idx_last_scheduled_token_p"
# are provided (which are pointers into
# "state_indices_tensor_p"), it will write additional cache
# states aligned at "block_size_to_align".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -874,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
initial_state_idx=block_idx_last_computed_token_p,
num_computed_tokens=num_computed_tokens_p,
block_size_to_align=mamba_block_size,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
@@ -891,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
initial_states = None
if has_initial_states_p is not None and prep_initial_states:
assert state_indices_tensor_p is not None
kernel_ssm_indices = state_indices_tensor_p
if is_mamba_cache_all:
kernel_ssm_indices = state_indices_tensor_p.gather(
1, block_idx_last_computed_token_p.unsqueeze(1)
).squeeze(1)
initial_states = torch.where(
has_initial_states_p[:, None, None, None],
ssm_state[kernel_ssm_indices],
ssm_state[state_indices_tensor_p],
0,
)
@@ -921,130 +859,29 @@ class MambaMixer2(MambaBase, PluggableLayer):
cu_chunk_seqlens=cu_chunk_seqlen_p,
last_chunk_indices=last_chunk_indices_p,
initial_states=initial_states,
return_intermediate_states=is_mamba_cache_all,
dt_softplus=True,
dt_limit=(0.0, float("inf")),
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
state_dtype=ssm_state.dtype,
)
if is_mamba_cache_all:
assert mamba_block_size is not None
assert state_indices_tensor_p is not None
assert block_idx_first_scheduled_token_p is not None
assert block_idx_last_scheduled_token_p is not None
assert last_chunk_indices_p is not None
assert num_computed_tokens_p is not None
# The chunk_stride is the number of chunks per mamba block
# e.g., if mamba_block_size = 512 and chunk_size = 256,
# then chunk_stride = 2
chunk_stride = mamba_block_size // chunk_size
# Save state for sequences with more than just final state,
# batched over sequences so that the number of kernel launches
# does not scale with the number of prefills.
n_blocks_to_fill = (
block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p
).clamp(min=0)
total_fill = attn_metadata.num_state_writes_p
if total_fill > 0:
first_chunk = torch.zeros_like(last_chunk_indices_p)
first_chunk[1:] = last_chunk_indices_p[:-1] + 1
# The chunk that completes this sequence's first mamba block.
# A block spans chunk_stride chunks. When the sequence resumed
# mid-block, the chunks before this step are not in varlen_states,
# so subtract them: the boundary sits earlier by their count.
num_unaligned_computed_tokens = (
num_computed_tokens_p % mamba_block_size
)
unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size
first_aligned_chunk = (
first_chunk + (chunk_stride - 1) - unaligned_chunk_shift
)
# Flatten all writes into one list: fill_counts=[2,1,2]
# -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1]
fill_counts = n_blocks_to_fill.long()
# output_size, or repeat_interleave reads the size off the device
seq_ids = torch.repeat_interleave(
torch.arange(num_prefills, device=ssm_state.device),
fill_counts,
output_size=total_fill,
)
write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts
write_offsets = torch.arange(
total_fill, device=ssm_state.device
) - torch.repeat_interleave(
write_starts, fill_counts, output_size=total_fill
)
cache_blocks_to_fill = state_indices_tensor_p[
seq_ids,
block_idx_first_scheduled_token_p[seq_ids] + write_offsets,
]
# Write the states. Each row goes to its own cache block,
# so the indexed copy has no duplicate destinations.
ssm_state[cache_blocks_to_fill] = varlen_states[
first_aligned_chunk[seq_ids] + write_offsets * chunk_stride
]
# For all seqs, store the last state (note: might be partial):
assert state_indices_tensor_p is not None
ssm_state[
state_indices_tensor_p.gather(
1, block_idx_last_scheduled_token_p.unsqueeze(1)
).squeeze(1)
] = varlen_states[last_chunk_indices_p]
else:
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# update ssm states
# - varlen state is a (num_prefills, nheads, headdim, dstate)
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
# Process decode requests
if has_decode:
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
if use_spec_decode:
assert block_idx_last_scheduled_token_prev_step_d is not None
input_indices = (
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
+ self._decode_state_offsets
)
output_indices = (
block_idx_last_scheduled_token_d.unsqueeze(1)
+ self._decode_state_offsets
)
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, input_indices
)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, output_indices
)
else:
state_indices_tensor_d_input = state_indices_tensor_d.gather(
1, block_idx_last_computed_token_d.unsqueeze(1)
).squeeze(1)
state_indices_tensor_d_output = state_indices_tensor_d.gather(
1, block_idx_last_scheduled_token_d.unsqueeze(1)
).squeeze(1)
else:
# Without caching, read and write in-place to the same blocks:
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
if (
self.use_replayssm
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
@@ -1081,8 +918,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices_tensor_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
initial_state_idx=block_idx_last_computed_token_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
# ReplaySSM keeps one physical state block while a speculative
@@ -1207,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
@@ -1227,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
D_d,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
state_batch_indices=state_indices_tensor_d,
dst_state_batch_indices=state_indices_tensor_d,
out=preallocated_ssm_out_d,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
+3 -14
View File
@@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig):
cache_config.mamba_cache_mode,
model_config.architecture,
)
if (
cache_config.mamba_cache_mode == "all"
and not model_config.supports_mamba_prefix_caching
):
cache_config.mamba_cache_mode = "align"
logger.warning(
"Hybrid or mamba-based model detected without support "
"for prefix caching with Mamba cache 'all' mode: "
"falling back to 'align' mode."
)
if cache_config.mamba_cache_mode == "align":
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
)
# By default, mamba block size will be set to max_model_len (see
# below). When enabling prefix caching, we align mamba block size
# to the block size as the basic granularity for prefix caching.
@@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase):
self.quant_config = vllm_config.quant_config
self.scheduler_config = vllm_config.scheduler_config
if vllm_config.cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Intern-S2-Mobius does not support 'all' prefix caching; "
"use '--mamba-cache-mode=align' instead."
)
nn.Module.__init__(self)
self.config = config
self.model = InternS2MobiusModel(
-7
View File
@@ -458,13 +458,6 @@ class Lfm2ForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
self.model = Lfm2Model(
-7
View File
@@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Lfm2Moe currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
super().__init__()
self.config = config
-6
View File
@@ -329,14 +329,8 @@ class Qwen3_5ForCausalLMBase(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3.5 currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -227,13 +227,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3_5MTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
-6
View File
@@ -828,14 +828,8 @@ class Qwen3NextForCausalLM(
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
scheduler_config = vllm_config.scheduler_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3Next currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
@@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen3NextMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
+1 -1
View File
@@ -104,7 +104,7 @@ def _mamba_get_block_table_tensor(
kv_cache_spec: MambaSpec,
mamba_cache_mode: str,
) -> torch.Tensor:
if mamba_cache_mode in ("all", "none"):
if mamba_cache_mode == "none":
return block_table
assert block_table.is_cuda and seq_lens.is_cuda
-5
View File
@@ -638,11 +638,6 @@ class Qwen4ExpForCausalLM(
self.quant_config = vllm_config.quant_config
self.config = config
self.scheduler_config = vllm_config.scheduler_config
if vllm_config.cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen4Exp currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.model = Qwen4ExpModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
-7
View File
@@ -395,13 +395,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
-5
View File
@@ -669,11 +669,6 @@ class Qwen4ExpForCausalLM(
self.quant_config = vllm_config.quant_config
self.config = config
self.scheduler_config = vllm_config.scheduler_config
if vllm_config.cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen4Exp currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.model = Qwen4ExpModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
-7
View File
@@ -375,13 +375,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
cache_config = vllm_config.cache_config
if cache_config.mamba_cache_mode == "all":
raise NotImplementedError(
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
"please use '--mamba-cache-mode=align' instead"
)
self.quant_config = vllm_config.quant_config
super().__init__()
+9 -28
View File
@@ -939,13 +939,6 @@ class Platform:
if mamba_page_size == 0:
return
# mamba_block_size here should either be user specified value or None
mamba_block_size = (
cache_config.mamba_block_size
if cache_config.user_specified_mamba_block_size
else None
)
# Get kernel block alignment from the backend's supported sizes
with set_current_vllm_config(vllm_config):
kernel_block_alignment_size = max(
@@ -962,27 +955,15 @@ class Platform:
# multiple of 128 so split kernel blocks keep that invariant.
kernel_block_alignment_size = max(kernel_block_alignment_size, 128)
if cache_config.mamba_cache_mode == "all":
# With prefix caching, align to mamba chunk size for kernel perf
# TODO(tdoublep): this constraint can be relaxed fairly
# easily by changing the way we layout chunks in the
# mamba2 kernels.
base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size()
assert base_chunk_size is not None
attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token)
chunk_size = lcm(base_chunk_size, kernel_block_alignment_size)
attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size)
cache_config.mamba_block_size = attn_block_size
else:
# Without prefix caching, use minimum block size that satisfies
# both backend alignment and mamba page size compatibility
attn_block_size = kernel_block_alignment_size * cdiv(
mamba_page_size,
kernel_block_alignment_size * attn_page_size_1_token,
)
indexer_align = cls._get_indexer_block_alignment(vllm_config)
if indexer_align:
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
# Use minimum block size that satisfies both backend alignment and
# mamba page size compatibility
attn_block_size = kernel_block_alignment_size * cdiv(
mamba_page_size,
kernel_block_alignment_size * attn_page_size_1_token,
)
indexer_align = cls._get_indexer_block_alignment(vllm_config)
if indexer_align:
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
if cache_config.block_size < attn_block_size:
cache_config.block_size = attn_block_size
+2 -21
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, replace
from dataclasses import dataclass
from typing import Any
from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata
@@ -42,23 +42,4 @@ class Mamba1AttentionMetadataBuilder(
fast_build: bool = False,
**kwargs: Any,
) -> Mamba1AttentionMetadata:
common = self._compute_common_metadata(common_attn_metadata)
if (
common.num_prefills > 0
and self.vllm_config.cache_config.mamba_cache_mode == "all"
):
cu_chunk_seqlen_p, _, last_chunk_indices_p = (
self._build_chunk_metadata_tensors(
self.kv_cache_spec.block_size,
common,
common_attn_metadata,
)
)
return replace(
common,
cu_chunk_seqlen_p=cu_chunk_seqlen_p,
last_chunk_indices_p=last_chunk_indices_p,
)
return common
return self._compute_common_metadata(common_attn_metadata)
@@ -139,7 +139,6 @@ class Mamba2AttentionMetadataBuilder(
common = self._compute_common_metadata(
common_attn_metadata,
num_accepted_tokens=kwargs.get("num_accepted_tokens"),
prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"),
num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"),
)
+17 -250
View File
@@ -39,7 +39,6 @@ class BaseMambaAttentionMetadata:
# the batch has no prefill requests.
has_initial_states_p: torch.Tensor | None
query_start_loc_p: torch.Tensor | None
num_computed_tokens_p: torch.Tensor | None
state_indices_tensor_p: torch.Tensor | None
# The following tensors are used for decode requests and
@@ -52,13 +51,6 @@ class BaseMambaAttentionMetadata:
# Includes the bonus token (so minimum is 1)
num_accepted_tokens: torch.Tensor | None # shape: [batch,]
# The following tensors are only used for prefix caching in all mode and
# are None if disabled
block_idx_last_scheduled_token: torch.Tensor | None
block_idx_first_scheduled_token_p: torch.Tensor | None
block_idx_last_computed_token: torch.Tensor | None
block_idx_last_scheduled_token_prev_step: torch.Tensor | None
# The following tensor is only used for prefix caching in align mode
seq_lens: torch.Tensor
@@ -70,9 +62,6 @@ class BaseMambaAttentionMetadata:
# last_chunk_indices_p is a tensor of shape (batch,) that contains the
# index of the last chunk for every sequence in the (prefill) batch.
last_chunk_indices_p: torch.Tensor | None = None
# Number of block-aligned SSM states the prefill path writes, i.e.
# sum(block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p).
num_state_writes_p: int = 0
# The following attributes are for triton implementation of causal_conv1d
nums_dict: dict | None = None
@@ -130,47 +119,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
self.compilation_config.max_cudagraph_capture_size,
)
if self.vllm_config.cache_config.mamba_cache_mode == "all":
max_num_blocks = (
cdiv(
self.vllm_config.model_config.max_model_len,
kv_cache_spec.block_size,
)
+ kv_cache_spec.num_speculative_blocks
)
# TODO: reduce this size as needed for decode-only cudagraph capture
self.state_indices_tensor_d: torch.Tensor = torch.empty(
(
self.decode_cudagraph_max_bs,
max_num_blocks,
),
dtype=torch.int32,
device=device,
)
self.block_idx_last_scheduled_token: torch.Tensor = torch.empty(
(self.decode_cudagraph_max_bs,),
dtype=torch.int32,
device=device,
)
self.block_idx_last_computed_token: torch.Tensor = torch.empty(
(self.decode_cudagraph_max_bs,),
dtype=torch.int32,
device=device,
)
if self.use_spec_decode:
self.block_idx_last_scheduled_token_prev_step: torch.Tensor = (
torch.empty(
(self.decode_cudagraph_max_bs,),
dtype=torch.int32,
device=device,
)
)
else:
self.state_indices_tensor_d = torch.empty(
(self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens),
dtype=torch.int32,
device=device,
)
self.state_indices_tensor_d = torch.empty(
(self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens),
dtype=torch.int32,
device=device,
)
# For speculative decoding, we need to store the following buffers
# for CUDA graph capture during decode
@@ -259,23 +212,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
if self.num_spec_tokens > 0:
num_accepted_tokens = torch.diff(m.query_start_loc)
prev_last_scheduled_idx = None
if (
self.use_spec_decode
and self.vllm_config.cache_config.mamba_cache_mode == "all"
):
prev_last_scheduled_idx = torch.zeros(
(m.num_reqs,),
dtype=torch.int32,
device=m.query_start_loc.device,
)
return self.build(
0,
m,
num_accepted_tokens=num_accepted_tokens,
prev_last_scheduled_idx=prev_last_scheduled_idx,
)
return self.build(0, m, num_accepted_tokens=num_accepted_tokens)
def build(
self,
@@ -284,7 +221,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
fast_build: bool = False,
*,
num_accepted_tokens: torch.Tensor | None = None,
prev_last_scheduled_idx: torch.Tensor | None = None,
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
**kwargs: Any,
) -> M:
@@ -294,7 +230,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
return self._compute_common_metadata(
common_attn_metadata,
num_accepted_tokens=num_accepted_tokens,
prev_last_scheduled_idx=prev_last_scheduled_idx,
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
)
@@ -423,56 +358,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
)
return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p
def _compute_prefix_caching_block_indices(
self,
common_attn_metadata: CommonAttentionMetadata,
mamba_block_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
# Block index of the last computed token
block_idx_last_computed_token = (
torch.div(
num_computed_tokens + mamba_block_size - 1,
mamba_block_size,
rounding_mode="floor",
)
- 1
)
# which is <= block index for the first scheduled token
block_idx_first_scheduled_token = (
torch.div(
num_computed_tokens + mamba_block_size,
mamba_block_size,
rounding_mode="floor",
)
- 1
)
# which is <= block index of the last scheduled token
block_idx_last_scheduled_token = (
torch.div(
common_attn_metadata.seq_lens + mamba_block_size - 1,
mamba_block_size,
rounding_mode="floor",
)
- 1
)
# -1 in case it's non-computed and causes later issues with indexing
block_idx_last_computed_token.clamp_(min=0)
# -1 in the case we have a padded request (0 seq-len)
block_idx_last_scheduled_token.clamp_(min=0)
return (
block_idx_last_computed_token,
block_idx_first_scheduled_token,
block_idx_last_scheduled_token,
)
def _compute_common_metadata(
self,
common_attn_metadata: CommonAttentionMetadata,
*,
num_accepted_tokens: torch.Tensor | None = None,
prev_last_scheduled_idx: torch.Tensor | None = None,
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
) -> M:
"""Compute metadata common to both Mamba1 and Mamba2."""
@@ -540,16 +430,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
has_initial_states_p = None
query_start_loc_p = None
query_start_loc_d = None
num_computed_tokens = None
num_computed_tokens_p = None
# for prefix caching
block_idx_first_scheduled_token = None
block_idx_first_scheduled_token_p = None
block_idx_last_computed_token = None
block_idx_last_scheduled_token = None
block_idx_last_scheduled_token_prev_step = None
num_state_writes_p = 0
# for causal_conv1d
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
@@ -557,35 +437,12 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
is_flush_d = None
replayssm_scratch = None
if self.vllm_config.cache_config.mamba_cache_mode == "all":
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
# Return a tensor of shape (#requests, #max blocks)
state_indices_tensor = common_attn_metadata.block_table_tensor
# Additional cache-related variables:
mamba_block_size = self.kv_cache_spec.block_size
(
block_idx_last_computed_token,
block_idx_first_scheduled_token,
block_idx_last_scheduled_token,
) = self._compute_prefix_caching_block_indices(
common_attn_metadata, mamba_block_size
)
if self.use_spec_decode and prev_last_scheduled_idx is not None:
fallback = (num_computed_tokens - 1) // mamba_block_size
fallback.clamp_(min=0)
block_idx_last_scheduled_token_prev_step = torch.where(
prev_last_scheduled_idx >= 0,
prev_last_scheduled_idx,
fallback,
)
else:
state_indices_tensor = mamba_get_block_table_tensor(
common_attn_metadata.block_table_tensor,
common_attn_metadata.seq_lens,
self.kv_cache_spec,
self.vllm_config.cache_config.mamba_cache_mode,
)
state_indices_tensor = mamba_get_block_table_tensor(
common_attn_metadata.block_table_tensor,
common_attn_metadata.seq_lens,
self.kv_cache_spec,
self.vllm_config.cache_config.mamba_cache_mode,
)
if state_indices_tensor.dim() == 1:
state_indices_tensor = state_indices_tensor.unsqueeze(-1)
@@ -595,11 +452,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
[num_decodes, num_prefills],
dim=0,
)
if self.vllm_config.cache_config.mamba_cache_mode != "all":
state_indices_tensor_d = state_indices_tensor_d[
:, : 1 + self.num_spec_tokens
]
state_indices_tensor_p = state_indices_tensor_p[:, 0]
state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens]
state_indices_tensor_p = state_indices_tensor_p[:, 0]
if num_decodes > 0 and self.use_spec_decode:
query_start_loc_d = common_attn_metadata.query_start_loc[: num_decodes + 1]
@@ -613,8 +467,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
num_accepted_tokens = num_accepted_tokens[:num_decodes]
if num_prefills > 0:
if num_computed_tokens is None:
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
query_start_loc_p = (
common_attn_metadata.query_start_loc[-num_prefills - 1 :]
@@ -636,33 +489,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
)
)
if self.vllm_config.cache_config.mamba_cache_mode == "all":
assert num_computed_tokens is not None
num_computed_tokens_p = num_computed_tokens[
num_reqs - num_prefills : num_reqs
]
assert block_idx_first_scheduled_token is not None
block_idx_first_scheduled_token_p = block_idx_first_scheduled_token[
num_reqs - num_prefills : num_reqs
]
# How many block-aligned states prefill will write: the same
# (block_idx_last - block_idx_first) as above, from CPU data so
# the mixer needs no D2H read. The -1 in both indices cancels.
block_size = self.kv_cache_spec.block_size
seq_lens_p_cpu = seq_lens_cpu[num_reqs - num_prefills : num_reqs]
num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata(
common_attn_metadata, num_reqs, num_prefills, num_decode_tokens
)
num_state_writes_p = int(
(
(seq_lens_p_cpu + block_size - 1) // block_size
- (num_computed_tokens_p_cpu + block_size) // block_size
)
.clamp(min=0)
.sum()
)
if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0:
decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
@@ -764,14 +590,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
replayssm_scratch=replayssm_scratch,
num_accepted_tokens=num_accepted_tokens,
query_start_loc_d=query_start_loc_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token,
block_idx_first_scheduled_token_p=block_idx_first_scheduled_token_p,
num_state_writes_p=num_state_writes_p,
block_idx_last_computed_token=block_idx_last_computed_token,
block_idx_last_scheduled_token_prev_step=(
block_idx_last_scheduled_token_prev_step
),
num_computed_tokens_p=num_computed_tokens_p,
num_reqs=num_reqs,
seq_lens=common_attn_metadata.seq_lens,
nums_dict=nums_dict,
@@ -791,11 +609,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
state_indices_tensor_d = metadata.state_indices_tensor_d
query_start_loc_d = metadata.query_start_loc_d
num_accepted_tokens = metadata.num_accepted_tokens
block_idx_last_scheduled_token = metadata.block_idx_last_scheduled_token
block_idx_last_computed_token = metadata.block_idx_last_computed_token
block_idx_last_scheduled_token_prev_step = (
metadata.block_idx_last_scheduled_token_prev_step
)
write_pos_d = metadata.write_pos_d
is_flush_d = metadata.is_flush_d
bc_pre_scratch = metadata.bc_pre_scratch
@@ -824,44 +637,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
1 # pad with 1st slot index
)
if self.vllm_config.cache_config.mamba_cache_mode == "all":
assert block_idx_last_scheduled_token is not None
assert block_idx_last_computed_token is not None
self.block_idx_last_scheduled_token[: metadata.num_decodes].copy_(
block_idx_last_scheduled_token[: metadata.num_decodes],
non_blocking=True,
)
block_idx_last_scheduled_token = self.block_idx_last_scheduled_token[
:padded_bs
]
block_idx_last_scheduled_token[metadata.num_decodes :] = 0
self.block_idx_last_computed_token[: metadata.num_decodes].copy_(
block_idx_last_computed_token[: metadata.num_decodes],
non_blocking=True,
)
block_idx_last_computed_token = self.block_idx_last_computed_token[
:padded_bs
]
block_idx_last_computed_token[metadata.num_decodes :] = 0
if (
self.use_spec_decode
and block_idx_last_scheduled_token_prev_step is not None
):
self.block_idx_last_scheduled_token_prev_step[
: metadata.num_decodes
].copy_(
block_idx_last_scheduled_token_prev_step[
: metadata.num_decodes
],
non_blocking=True,
)
block_idx_last_scheduled_token_prev_step = (
self.block_idx_last_scheduled_token_prev_step[:padded_bs]
)
block_idx_last_scheduled_token_prev_step[metadata.num_decodes :] = 0
if self.use_replayssm and not self.use_flashinfer_replayssm:
assert write_pos_d is not None
assert is_flush_d is not None
@@ -914,11 +689,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
bc_pre_scratch=bc_pre_scratch,
replayssm_scratch=replayssm_scratch,
replayssm_state_indices_d=replayssm_state_indices_d,
block_idx_last_scheduled_token=block_idx_last_scheduled_token,
block_idx_last_computed_token=block_idx_last_computed_token,
block_idx_last_scheduled_token_prev_step=(
block_idx_last_scheduled_token_prev_step
),
)
def update_block_table(
@@ -950,11 +720,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
[metadata.num_decodes, metadata.num_prefills],
dim=0,
)
if self.vllm_config.cache_config.mamba_cache_mode != "all":
state_indices_tensor_d = state_indices_tensor_d[
:, : 1 + self.num_spec_tokens
]
state_indices_tensor_p = state_indices_tensor_p[:, 0]
state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens]
state_indices_tensor_p = state_indices_tensor_p[:, 0]
new_metadata = replace(
metadata,
+3 -10
View File
@@ -283,7 +283,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
has_initial_states_p = None
has_initial_states_d = None
num_computed_tokens_p = None
# Per-request classification by mask, NOT by position. With
# spec-decode, the front decode group can contain both spec-decode
@@ -402,10 +401,9 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
assert non_spec_query_start_loc is not None
query_start_loc_d = non_spec_query_start_loc[: num_decodes + 1]
if num_prefills > 0:
num_computed_tokens_p = num_computed_tokens[
num_decodes : num_decodes + num_prefills
]
has_initial_states_p = num_computed_tokens_p > 0
has_initial_states_p = (
num_computed_tokens[num_decodes : num_decodes + num_prefills] > 0
)
assert non_spec_query_start_loc is not None
query_start_loc_p = (
non_spec_query_start_loc[num_decodes:] - num_decode_tokens
@@ -478,11 +476,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
query_start_loc_d=query_start_loc_d,
state_indices_tensor_p=state_indices_tensor_p,
state_indices_tensor_d=state_indices_tensor_d,
num_computed_tokens_p=num_computed_tokens_p,
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
seq_lens=m.seq_lens,
)
+1 -6
View File
@@ -1162,11 +1162,6 @@ def mamba_get_block_table_tensor(
"""Get the block table tensor for mamba kernels from the input
common_attn_metadata.block_table_tensor given different mamba cache modes.
- "all": input (#requests, cdiv(max_model_len, block_size)
+ num_speculative_blocks);
output (#requests, cdiv(max_model_len, block_size)
+ num_speculative_blocks).
- "none": input (#requests, 1 + num_speculative_blocks);
output (#requests, 1 + num_speculative_blocks).
@@ -1174,7 +1169,7 @@ def mamba_get_block_table_tensor(
output (#requests, 1 + num_speculative_blocks), which are the last
1 + num_speculative_blocks of each request.
"""
if mamba_cache_mode in ("all", "none"):
if mamba_cache_mode == "none":
return block_table
else:
assert isinstance(kv_cache_spec, MambaSpec)
+1 -6
View File
@@ -1079,12 +1079,7 @@ class MambaSpec(KVCacheSpec):
return None
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
if vllm_config.cache_config.mamba_cache_mode == "all":
max_model_len = vllm_config.model_config.max_model_len
return (
cdiv(max_model_len, self.block_size) + self.num_speculative_blocks
) * self.page_size_bytes
elif vllm_config.cache_config.mamba_cache_mode == "align":
if vllm_config.cache_config.mamba_cache_mode == "align":
return self.page_size_bytes * (
2 + self.num_speculative_blocks + self.num_prefill_checkpoint_blocks
)
-32
View File
@@ -967,11 +967,6 @@ class GPUModelRunner(
self.mamba_state_idx: dict[str, int] = {}
self._mamba_bufs: mamba_utils.MambaBuffers | None = None
self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None
self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None
if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0:
self.mamba_prev_last_scheduled_idx = self._make_buffer(
self.max_num_reqs, dtype=torch.int32
)
self.layerwise_nvtx_hooks_registered = False
def update_max_model_len(self, max_model_len: int) -> None:
@@ -1608,17 +1603,6 @@ class GPUModelRunner(
assert self.num_accepted_tokens_event is not None
self.num_accepted_tokens_event.record()
if self.cache_config.mamba_cache_mode == "all":
mamba_utils.postprocess_mamba_all(
scheduler_output,
self.kv_cache_config,
self.input_batch,
self.requests,
self.mamba_state_idx,
self.num_spec_tokens,
num_reqs,
)
def _update_streaming_request(
self, req_id: str, new_req_data: NewRequestData
) -> CachedRequestState:
@@ -2127,15 +2111,6 @@ class GPUModelRunner(
self.num_accepted_tokens.np.fill(1)
self.num_accepted_tokens.gpu.fill_(1)
if self.mamba_prev_last_scheduled_idx is not None:
mamba_utils.preprocess_mamba_all_specdec(
scheduler_output,
self.input_batch,
self.mamba_state_idx,
num_reqs,
self.mamba_prev_last_scheduled_idx,
)
# Update num_computed_tokens on GPU. In async spec decode,
# CPU values are optimistic (all drafts accepted). The kernel
# corrects on GPU using the previous step's
@@ -2511,13 +2486,6 @@ class GPUModelRunner(
:num_reqs_padded
],
)
if (
isinstance(builder, Mamba2AttentionMetadataBuilder)
and self.mamba_prev_last_scheduled_idx is not None
):
extra_attn_metadata_args["prev_last_scheduled_idx"] = (
self.mamba_prev_last_scheduled_idx.gpu[:num_reqs_padded]
)
if for_cudagraph_capture:
attn_metadata_i = builder.build_for_cudagraph_capture(
-47
View File
@@ -1535,53 +1535,6 @@ def preprocess_mamba(
do_mamba_copy_block(copy_bufs)
def postprocess_mamba_all(
scheduler_output: SchedulerOutput,
kv_cache_config: KVCacheConfig,
input_batch: InputBatch,
requests: dict[str, CachedRequestState],
mamba_state_idx: dict[str, int],
num_spec_tokens: int,
num_reqs: int,
):
"""All-mode postprocess (only meaningful with num_spec_tokens > 0):
record per-request the block index of the last token scheduled this
step, so the next step can anchor its in-place writes when accepted
drafts leave the sequence at a non-block-aligned position.
"""
if num_spec_tokens <= 0:
return
mamba_groups = get_mamba_groups(kv_cache_config)
block_sizes = {mamba_spec.block_size for mamba_spec in mamba_groups}
assert len(block_sizes) == 1, "all mamba groups must share block_size"
block_size = next(iter(block_sizes))
full_decode_len = 1 + num_spec_tokens
scheduled = scheduler_output.num_scheduled_tokens
for req_id in input_batch.req_ids[:num_reqs]:
num_query = scheduled.get(req_id, 0)
if num_query == full_decode_len:
req = requests[req_id]
seq_len = req.num_computed_tokens + num_query
mamba_state_idx[req_id] = max(0, (seq_len - 1) // block_size)
else:
mamba_state_idx.pop(req_id, None)
def preprocess_mamba_all_specdec(
scheduler_output: SchedulerOutput,
input_batch: InputBatch,
mamba_state_idx: dict[str, int],
num_reqs: int,
prev_last_scheduled_idx_buf: CpuGpuBuffer,
) -> None:
cleanup_mamba_state_idx(scheduler_output, mamba_state_idx)
np_view = prev_last_scheduled_idx_buf.np
for i, req_id in enumerate(input_batch.req_ids[:num_reqs]):
np_view[i] = mamba_state_idx.get(req_id, -1)
np_view[num_reqs:].fill(-1)
prev_last_scheduled_idx_buf.copy_to_gpu()
def postprocess_mamba_align_gpu(
*,
bufs: "MambaBuffers",