[Core] Remove deprecated mamba_cache_mode "all" (#58997)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Thomas Parnell
parent
08d77cad7b
commit
cb6458c5ce
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
|
||||
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=None,
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([5]),
|
||||
)
|
||||
|
||||
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([1, 1]),
|
||||
)
|
||||
|
||||
|
||||
@@ -568,15 +568,10 @@ def test_replayssm_mixer_selects_mtp_layout(
|
||||
num_reqs=2,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
|
||||
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
|
||||
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
|
||||
|
||||
@@ -270,9 +270,6 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.use_v2_model_runner = True
|
||||
config.cache_config.mamba_cache_mode = "all"
|
||||
with pytest.raises(ValueError, match="only none and align"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
|
||||
config.model_config.architecture = "NemotronHForCausalLM"
|
||||
@@ -286,6 +283,12 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
|
||||
|
||||
def test_mamba_cache_mode_all_is_rejected():
|
||||
"""The removed 'all' mode must fail validation instead of being ignored."""
|
||||
with pytest.raises(ValidationError, match="mamba_cache_mode"):
|
||||
CacheConfig(mamba_cache_mode="all")
|
||||
|
||||
|
||||
def test_per_request_spec_decode_metrics_requires_spec_decode():
|
||||
# The flag only makes sense with speculative decoding configured; enabling
|
||||
# it without --speculative-config should fail fast rather than silently
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for BaseMambaAttentionMetadataBuilder batch classification."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.v1.attention.utils import MockMambaBuilder
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
num_speculative_tokens: int = 0,
|
||||
block_size: int | None = None,
|
||||
):
|
||||
"""Create a minimal mock VllmConfig with only the fields the builder
|
||||
accesses, avoiding any model download / HF config inspection."""
|
||||
speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
if num_speculative_tokens > 0
|
||||
else None
|
||||
)
|
||||
return SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="none",
|
||||
use_replayssm=False,
|
||||
replayssm_buffer_len=16,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
max_cudagraph_capture_size=None,
|
||||
),
|
||||
speculative_config=speculative_config,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
model_config=SimpleNamespace(max_model_len=max_model_len),
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_single_token_prompt_runs_as_prefill():
|
||||
seq_lens = [8, 9, 1]
|
||||
config = _make_vllm_config(256, len(seq_lens), block_size=16)
|
||||
metadata = MockMambaBuilder.build_mamba_metadata(
|
||||
config,
|
||||
seq_lens=seq_lens,
|
||||
query_lens=[1] * len(seq_lens),
|
||||
is_prefilling=[False, False, True],
|
||||
)
|
||||
|
||||
assert metadata.num_decodes == 2
|
||||
assert metadata.num_prefills == 1
|
||||
assert metadata.has_initial_states_p.tolist() == [False]
|
||||
@@ -1,466 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
|
||||
|
||||
When multiple KV cache groups share the same MambaSpec (as in Nemotron
|
||||
hybrid models), the metadata caching optimization reuses metadata from
|
||||
an earlier group via update_block_table(). In 'all' mode with CUDA graphs,
|
||||
update_block_table() must copy block_idx_last_scheduled_token and
|
||||
block_idx_last_computed_token to the *current* builder's persistent
|
||||
buffers, otherwise CUDA graph replay reads stale values from uninitialized
|
||||
buffers.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import MockMambaBuilder
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
num_speculative_tokens: int = 0,
|
||||
block_size: int | None = None,
|
||||
):
|
||||
"""Create a minimal mock VllmConfig with only the fields the builder
|
||||
accesses, avoiding any model download / HF config inspection."""
|
||||
speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
if num_speculative_tokens > 0
|
||||
else None
|
||||
)
|
||||
return SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="all",
|
||||
use_replayssm=False,
|
||||
replayssm_buffer_len=16,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
max_cudagraph_capture_size=None,
|
||||
),
|
||||
speculative_config=speculative_config,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
model_config=SimpleNamespace(max_model_len=max_model_len),
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_single_token_prompt_runs_as_prefill():
|
||||
seq_lens = [8, 9, 1]
|
||||
config = _make_vllm_config(256, len(seq_lens), block_size=16)
|
||||
metadata = MockMambaBuilder.build_mamba_metadata(
|
||||
config,
|
||||
seq_lens=seq_lens,
|
||||
query_lens=[1] * len(seq_lens),
|
||||
is_prefilling=[False, False, True],
|
||||
)
|
||||
|
||||
assert metadata.num_decodes == 2
|
||||
assert metadata.num_prefills == 1
|
||||
assert metadata.has_initial_states_p.tolist() == [False]
|
||||
|
||||
|
||||
def test_update_block_table_copies_block_idx_to_persistent_buffers():
|
||||
"""update_block_table() must write block_idx tensors to the current
|
||||
builder's persistent buffers, not leave them pointing to a different
|
||||
builder's buffers."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
num_reqs = 4
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(max_model_len, num_reqs)
|
||||
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
|
||||
# Two builders simulating two KV cache groups with the same MambaSpec.
|
||||
builder_a = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
builder_b = MockMambaBuilder(spec, ["layer1"], vllm_config, device)
|
||||
|
||||
# Sanity: each builder has its own persistent buffer.
|
||||
assert (
|
||||
builder_a.block_idx_last_scheduled_token.data_ptr()
|
||||
!= builder_b.block_idx_last_scheduled_token.data_ptr()
|
||||
)
|
||||
|
||||
# Construct decode-only metadata as if builder_a.build() produced it.
|
||||
max_blocks = max_model_len // block_size
|
||||
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
|
||||
block_idx_vals = (seq_lens - 1) // block_size # [3, 3, 3, 3]
|
||||
|
||||
builder_a.block_idx_last_scheduled_token[:num_reqs].copy_(block_idx_vals)
|
||||
builder_a.block_idx_last_computed_token[:num_reqs].copy_(block_idx_vals)
|
||||
|
||||
metadata_a = BaseMambaAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_reqs,
|
||||
num_decode_tokens=num_reqs,
|
||||
num_reqs=num_reqs,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
query_start_loc_d=None,
|
||||
num_accepted_tokens=None,
|
||||
state_indices_tensor_d=builder_a.state_indices_tensor_d[:num_reqs],
|
||||
block_idx_last_scheduled_token=(
|
||||
builder_a.block_idx_last_scheduled_token[:num_reqs]
|
||||
),
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=(
|
||||
builder_a.block_idx_last_computed_token[:num_reqs]
|
||||
),
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
|
||||
# Call update_block_table on builder_b (simulates the metadata caching
|
||||
# optimization reusing metadata from builder_a's group).
|
||||
blk_table = torch.randint(
|
||||
0, 100, (num_reqs, max_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
slot_mapping = torch.zeros(num_reqs, dtype=torch.int64, device=device)
|
||||
|
||||
metadata_b = builder_b.update_block_table(metadata_a, blk_table, slot_mapping)
|
||||
|
||||
# block_idx tensors must live in builder_b's persistent buffers.
|
||||
def shares_storage(tensor, buffer):
|
||||
return (
|
||||
tensor.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
assert shares_storage(
|
||||
metadata_b.block_idx_last_scheduled_token,
|
||||
builder_b.block_idx_last_scheduled_token,
|
||||
), "block_idx_last_scheduled_token not in builder_b's persistent buffer"
|
||||
|
||||
assert shares_storage(
|
||||
metadata_b.block_idx_last_computed_token,
|
||||
builder_b.block_idx_last_computed_token,
|
||||
), "block_idx_last_computed_token not in builder_b's persistent buffer"
|
||||
|
||||
# Must NOT point to builder_a's buffers.
|
||||
assert not shares_storage(
|
||||
metadata_b.block_idx_last_scheduled_token,
|
||||
builder_a.block_idx_last_scheduled_token,
|
||||
), "block_idx_last_scheduled_token still points to builder_a's buffer"
|
||||
|
||||
# Values must be correct (copied from metadata_a).
|
||||
torch.testing.assert_close(
|
||||
metadata_b.block_idx_last_scheduled_token,
|
||||
block_idx_vals,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata_b.block_idx_last_computed_token,
|
||||
block_idx_vals,
|
||||
)
|
||||
|
||||
|
||||
def test_state_indices_tensor_d_includes_num_speculative_blocks():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
|
||||
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
|
||||
the cudagraph buffer for state_indices_tensor_d must allocate the same
|
||||
per-request column count as the runtime block table, which includes
|
||||
num_speculative_blocks trailing scratch columns."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 4
|
||||
num_speculative_tokens = 1
|
||||
num_speculative_blocks = 2
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
expected_cols = (max_model_len // block_size) + num_speculative_blocks
|
||||
assert builder.state_indices_tensor_d.shape == (max_num_seqs, expected_cols)
|
||||
|
||||
|
||||
def test_block_idx_cudagraph_capture_padded_by_num_reqs():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
|
||||
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
|
||||
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
|
||||
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
|
||||
Past num_decodes, the slice must be zero-filled."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
num_speculative_tokens = 1
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
builder.block_idx_last_scheduled_token.fill_(-1)
|
||||
builder.block_idx_last_computed_token.fill_(-1)
|
||||
|
||||
num_decodes = 2
|
||||
num_reqs = 3
|
||||
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
|
||||
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
|
||||
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
|
||||
state_indices_d = torch.zeros(
|
||||
(num_decodes, builder.state_indices_tensor_d.shape[1]),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
query_start_loc_d = torch.arange(
|
||||
num_decodes + 1, dtype=torch.int32, device=device
|
||||
) * (1 + num_speculative_tokens)
|
||||
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
|
||||
|
||||
metadata = BaseMambaAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=num_reqs,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=state_indices_d,
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
block_idx_last_scheduled_token=block_idx_vals,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=block_idx_vals,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
|
||||
out = builder._update_metadata_for_cudagraph_capture(metadata)
|
||||
|
||||
assert out.block_idx_last_scheduled_token.shape == (num_reqs,)
|
||||
assert out.block_idx_last_computed_token.shape == (num_reqs,)
|
||||
torch.testing.assert_close(
|
||||
out.block_idx_last_scheduled_token[:num_decodes], block_idx_vals
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
out.block_idx_last_computed_token[:num_decodes], block_idx_vals
|
||||
)
|
||||
assert torch.all(out.block_idx_last_scheduled_token[num_decodes:] == 0)
|
||||
assert torch.all(out.block_idx_last_computed_token[num_decodes:] == 0)
|
||||
|
||||
|
||||
def test_block_idx_prev_step_persistent_buffer_allocated():
|
||||
"""With mamba_cache_mode='all' + spec decode, the builder must allocate
|
||||
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
|
||||
same shape as the existing block_idx_last_{scheduled,computed}_token
|
||||
buffers, so cudagraph capture records a stable pointer for the prev-step
|
||||
input anchor consumed by mamba_mixer2's input gather."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
num_speculative_tokens = 1
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
assert hasattr(builder, "block_idx_last_scheduled_token_prev_step")
|
||||
assert builder.block_idx_last_scheduled_token_prev_step.shape == (max_num_seqs,)
|
||||
assert builder.block_idx_last_scheduled_token_prev_step.dtype == torch.int32
|
||||
|
||||
|
||||
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
|
||||
"""Without spec decode, the prev-step buffer is unused and must not be
|
||||
allocated — the input anchor reduces to last_computed_token."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len, max_num_seqs, num_speculative_tokens=0
|
||||
)
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
assert not hasattr(builder, "block_idx_last_scheduled_token_prev_step")
|
||||
|
||||
|
||||
def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
|
||||
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
|
||||
into the builder's persistent buffer (so cudagraph replay reads from the
|
||||
same underlying memory), pad past num_decodes with zero, and return a
|
||||
slice of the persistent buffer in the metadata."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
num_speculative_tokens = 1
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
builder.block_idx_last_scheduled_token.fill_(-1)
|
||||
builder.block_idx_last_computed_token.fill_(-1)
|
||||
builder.block_idx_last_scheduled_token_prev_step.fill_(-1)
|
||||
|
||||
num_decodes = 2
|
||||
num_reqs = 3
|
||||
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
|
||||
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
|
||||
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
|
||||
prev_step_vals = torch.tensor([2, 4], dtype=torch.int32, device=device)
|
||||
state_indices_d = torch.zeros(
|
||||
(num_decodes, builder.state_indices_tensor_d.shape[1]),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
query_start_loc_d = torch.arange(
|
||||
num_decodes + 1, dtype=torch.int32, device=device
|
||||
) * (1 + num_speculative_tokens)
|
||||
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
|
||||
|
||||
metadata = BaseMambaAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=num_reqs,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=state_indices_d,
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
block_idx_last_scheduled_token=block_idx_vals,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=block_idx_vals,
|
||||
block_idx_last_scheduled_token_prev_step=prev_step_vals,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
|
||||
out = builder._update_metadata_for_cudagraph_capture(metadata)
|
||||
|
||||
# Output field exists and is identity-shared with the persistent buffer.
|
||||
assert out.block_idx_last_scheduled_token_prev_step is not None
|
||||
assert (
|
||||
out.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
|
||||
== builder.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
|
||||
), (
|
||||
"prev-step buffer must live in the builder's persistent buffer, not "
|
||||
"in the caller-provided tensor"
|
||||
)
|
||||
|
||||
# Padded by num_reqs (not num_decode_tokens) — same fix as bug 2 for the
|
||||
# other block_idx_* fields.
|
||||
assert out.block_idx_last_scheduled_token_prev_step.shape == (num_reqs,)
|
||||
|
||||
# First num_decodes values: input values copied through.
|
||||
torch.testing.assert_close(
|
||||
out.block_idx_last_scheduled_token_prev_step[:num_decodes],
|
||||
prev_step_vals,
|
||||
)
|
||||
|
||||
# Tail values past num_decodes: zero-filled padding for cudagraph capture.
|
||||
assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0)
|
||||
|
||||
|
||||
def test_num_state_writes_p_matches_block_idx_tensors():
|
||||
"""The prefill state save sizes its batched copy from
|
||||
num_state_writes_p instead of reading the block index tensors back from
|
||||
the GPU, so the host count must equal what those tensors say."""
|
||||
# (block_size, seq_lens, query_lens, is_prefilling)
|
||||
cases = [
|
||||
# block-aligned and unaligned computed tokens, and a chunked prefill
|
||||
(16, [64, 33, 17], [48, 17, 1], [True, True, True]),
|
||||
(32, [200, 100, 64], [200, 36, 64], [True, True, True]),
|
||||
# decode rows ahead of the prefills
|
||||
(16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]),
|
||||
# prefills that stay inside their first block: nothing to write
|
||||
(16, [65, 66, 67], [1, 2, 3], [True, True, True]),
|
||||
]
|
||||
|
||||
for block_size, seq_lens, query_lens, is_prefilling in cases:
|
||||
metadata = MockMambaBuilder.build_mamba_metadata(
|
||||
_make_vllm_config(4096, len(seq_lens), block_size=block_size),
|
||||
seq_lens=seq_lens,
|
||||
query_lens=query_lens,
|
||||
is_prefilling=is_prefilling,
|
||||
)
|
||||
num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills
|
||||
block_idx_last_p = metadata.block_idx_last_scheduled_token[
|
||||
num_reqs - num_prefills : num_reqs
|
||||
]
|
||||
expected = int(
|
||||
(block_idx_last_p - metadata.block_idx_first_scheduled_token_p)
|
||||
.clamp(min=0)
|
||||
.sum()
|
||||
)
|
||||
|
||||
assert metadata.num_state_writes_p == expected, (
|
||||
f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}"
|
||||
)
|
||||
@@ -39,6 +39,7 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.diffusion_scheduler import (
|
||||
DiffusionAsyncScheduler,
|
||||
@@ -6751,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
num_blocks: int = 100,
|
||||
supports_divergent_hits: bool = True,
|
||||
) -> Scheduler:
|
||||
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
|
||||
"""FA + Mamba ("align" cache mode) scheduler with a MockKVConnector."""
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
trust_remote_code=True,
|
||||
@@ -6772,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="MockKVConnector",
|
||||
@@ -6804,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -6820,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
)
|
||||
|
||||
|
||||
def _seed_hybrid_prefix(
|
||||
manager: KVCacheManager, num_blocks: int, block_size: int
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Prefill a prefix one block per step so that align-mode Mamba caches
|
||||
the state at every block boundary. Returns the FA and Mamba block ids."""
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=num_blocks * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
assert num_computed == 0
|
||||
mamba_ids = []
|
||||
for i in range(num_blocks):
|
||||
manager.allocate_slots(
|
||||
fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None
|
||||
)
|
||||
fill.num_computed_tokens += block_size
|
||||
fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id)
|
||||
mamba_ids.append(group_mamba_ids[i])
|
||||
manager.free(fill)
|
||||
return fa_ids, mamba_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"matched_tokens,expected_num_computed",
|
||||
[
|
||||
@@ -6845,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries
|
||||
# (mamba cache mode "all" caches every block's state densely).
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
fa_ids = [b.block_id for b in blocks.blocks[0]]
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries.
|
||||
fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
|
||||
|
||||
# Evict the FA tail and the middle mamba states; block 0 (both groups)
|
||||
# and the deep mamba state at block 3 survive.
|
||||
@@ -6885,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
[new_req] = output.scheduled_new_reqs
|
||||
assert new_req.num_computed_tokens == expected_num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -6921,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix in both groups.
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
_, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
|
||||
|
||||
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
|
||||
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
|
||||
@@ -6955,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
[new_req] = output.scheduled_new_reqs
|
||||
assert new_req.num_computed_tokens == expected_num_computed
|
||||
|
||||
|
||||
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
|
||||
|
||||
@@ -60,7 +60,7 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode all \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
@@ -78,7 +78,7 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode all \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
|
||||
@@ -67,7 +67,6 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._attn_group_ids: list[int] = [0]
|
||||
self._num_ssm_scratch_blocks = 0
|
||||
self._ssm_state_slots_are_positional = False
|
||||
self._is_hma_required = False
|
||||
self.kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[None, None],
|
||||
@@ -130,7 +129,7 @@ def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0):
|
||||
block_size=16,
|
||||
shapes=shapes,
|
||||
dtypes=(torch.float32,) * num_states,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
@@ -146,7 +145,7 @@ def _gate_vllm_config(
|
||||
block_size: int = 16,
|
||||
num_lookahead_tokens: int = 0,
|
||||
num_speculative_tokens: int = 0,
|
||||
mamba_cache_mode: str = "all",
|
||||
mamba_cache_mode: str = "align",
|
||||
dcp_size: int = 1,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
@@ -193,7 +192,7 @@ def test_split_block_groups_ignores_transfer_disabled_group():
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
@@ -361,7 +360,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
block_size=4,
|
||||
num_lookahead_tokens=7,
|
||||
num_speculative_tokens=7,
|
||||
mamba_cache_mode="align",
|
||||
dcp_size=dcp_size,
|
||||
)
|
||||
scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config)
|
||||
@@ -398,21 +396,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_dspark_with_positional_mamba_cache():
|
||||
"""Positional mode counts lookahead in the Mamba allocation, so the
|
||||
one-block recurrent-state tail bound no longer holds."""
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(
|
||||
speculative_config=_spec_config("dspark"),
|
||||
block_size=4,
|
||||
mamba_cache_mode="all",
|
||||
),
|
||||
"engine",
|
||||
_dspark_wrapped_attention_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_hybrid_write():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
@@ -441,20 +424,7 @@ def test_split_block_groups_accepts_empty_abort_payload():
|
||||
assert sched.split_block_groups(()) == ([], [])
|
||||
|
||||
|
||||
def test_split_block_groups_keeps_positional_slots_in_all_mode():
|
||||
# mamba_cache_mode="all" keeps a state per block position.
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_ssm_state_slots_are_positional=True,
|
||||
)
|
||||
attn, mamba = sched.split_block_groups(([1], [40, 41, 42]))
|
||||
assert attn == [1]
|
||||
assert mamba == [[40, 41, 42]]
|
||||
|
||||
|
||||
def test_split_block_groups_keeps_only_running_state_outside_all_mode():
|
||||
def test_split_block_groups_keeps_only_running_state():
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
@@ -464,24 +434,21 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"positional,blocks,expected",
|
||||
"blocks,expected",
|
||||
[
|
||||
# Trailing scratch slots go in either cache mode.
|
||||
(True, [40, 41, 42], [40]),
|
||||
# Outside "all" the running state is picked only after they are gone.
|
||||
(False, [40, 41, 42, 43], [41]),
|
||||
# The running state is picked only after the scratch slots are gone.
|
||||
([40, 41, 42, 43], [41]),
|
||||
# Never empty: a list shorter than the scratch count keeps one state.
|
||||
(True, [40], [40]),
|
||||
(False, [], []),
|
||||
([40], [40]),
|
||||
([], []),
|
||||
],
|
||||
)
|
||||
def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected):
|
||||
def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected):
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_num_ssm_scratch_blocks=2,
|
||||
_ssm_state_slots_are_positional=positional,
|
||||
)
|
||||
|
||||
assert sched.split_block_groups(([1], blocks)) == ([1], [expected])
|
||||
@@ -555,7 +522,6 @@ def _make_read_scheduler():
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_ssm_state_slots_are_positional=True,
|
||||
request_id_to_transfer_id={},
|
||||
transfer_id_to_request_id={},
|
||||
_reqs_need_recv={},
|
||||
@@ -578,17 +544,17 @@ def _make_read_request(remote_block_ids):
|
||||
|
||||
def test_update_state_drops_decode_recompute_tail_block():
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10, 11], [90, 91]])
|
||||
request = _make_read_request([[10, 11], [91]])
|
||||
blocks = _FakeBlocks(
|
||||
all_groups=([100, 101, 102], [200, 201, 202]),
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]]
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [202]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [
|
||||
[10, 11],
|
||||
[90, 91],
|
||||
[91],
|
||||
]
|
||||
|
||||
|
||||
@@ -609,7 +575,6 @@ def test_update_state_pairs_trimmed_mamba_state_with_remote_state():
|
||||
"""The DSpark shape end to end: scratch slots are stripped, the running
|
||||
state is then selected, and the attention list loses its lookahead tail."""
|
||||
sched = _make_read_scheduler()
|
||||
sched._ssm_state_slots_are_positional = False
|
||||
sched._num_ssm_scratch_blocks = 2
|
||||
sched._max_decode_tail_blocks = 2
|
||||
request = _make_read_request([[10, 11], [90]])
|
||||
|
||||
@@ -856,28 +856,6 @@ def test_apply_prefix_caching_mamba_hybrid(
|
||||
[[6, 7, 8, 9], [99]],
|
||||
id="fa_prefix_hit_and_ssm_trim",
|
||||
),
|
||||
# Multi-slot SSM ("all" mode): a local prefix hit leaves fewer local
|
||||
# slots; the earlier remote slots are covered locally → remote tail.
|
||||
pytest.param(
|
||||
10,
|
||||
10,
|
||||
[list(range(10)), [5, 6]],
|
||||
[list(range(10)), [1, 2, 3]],
|
||||
[list(range(10)), [5, 6]],
|
||||
[list(range(10)), [2, 3]],
|
||||
id="ssm_multi_block_local_hit_tail",
|
||||
),
|
||||
# Multi-slot SSM ("all" mode): the one trailing local position holds
|
||||
# the token D recomputes itself → local head-clip.
|
||||
pytest.param(
|
||||
10,
|
||||
10,
|
||||
[list(range(10)), [4, 5, 6]],
|
||||
[list(range(10)), [8, 9]],
|
||||
[list(range(10)), [4, 5]],
|
||||
[list(range(10)), [8, 9]],
|
||||
id="ssm_multi_block_local_extra_head_clip",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_apply_prefix_caching_ssm_prefix_cache_hit(
|
||||
@@ -2034,13 +2012,12 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_ssm_single_state():
|
||||
"""In single-state cache modes, SSM lists are reduced to the running
|
||||
state slot: speculative scratch slots, null placeholders and the previous
|
||||
step's state carry nothing. Attention groups pass through untouched."""
|
||||
"""SSM lists are reduced to the running state slot: speculative scratch
|
||||
slots, null placeholders and the previous step's state carry nothing.
|
||||
Attention groups pass through untouched."""
|
||||
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
|
||||
sched.blocks_per_sw = [0, 0]
|
||||
sched._ssm_spec_blocks = [None, 2]
|
||||
sched._ssm_state_slots_are_positional = False
|
||||
|
||||
# Align-mode list: null placeholders, state block, 2 speculative slots.
|
||||
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 0, 7, 8, 9]))
|
||||
@@ -2069,19 +2046,6 @@ def test_exchange_clipped_blocks_ssm_single_state():
|
||||
assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_ssm_positional_states():
|
||||
"""In "all" mode every position holds a state, so only the speculative
|
||||
slots go; placeholders stay to keep the list position-indexed."""
|
||||
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
|
||||
sched.blocks_per_sw = [0, 0]
|
||||
sched._ssm_spec_blocks = [None, 2]
|
||||
sched._ssm_state_slots_are_positional = True
|
||||
|
||||
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 5, 6, 7, 8, 9]))
|
||||
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
|
||||
|
||||
|
||||
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -566,10 +566,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
|
||||
def test_mamba_cpu_offload_boundary(
|
||||
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
|
||||
):
|
||||
def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
@@ -586,7 +583,7 @@ def test_mamba_cpu_offload_boundary(
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
language_model_only=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
mamba_cache_mode="align",
|
||||
# Use lossless state storage so this exact-equality regression isolates
|
||||
# offload boundary selection from low-precision Mamba checkpointing.
|
||||
mamba_cache_dtype="float32",
|
||||
|
||||
@@ -474,7 +474,7 @@ def make_kv_cache_config(
|
||||
mamba_enabled: bool = False,
|
||||
sw_size: int = 128,
|
||||
num_blocks: int = 100,
|
||||
mamba_cache_mode: Literal["all", "align", "none"] = "none",
|
||||
mamba_cache_mode: Literal["align", "none"] = "none",
|
||||
) -> KVCacheConfig:
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
|
||||
@@ -2335,7 +2335,7 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
# of the hash block, so it cannot exceed the smallest of them.
|
||||
hash_block_size=BLOCK_SIZE,
|
||||
dcp_world_size=dcp_world_size,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
@@ -2747,55 +2747,6 @@ def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None:
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
|
||||
|
||||
|
||||
def test_eager_store_scans_mamba_all_blocks_positionally() -> None:
|
||||
"""Mamba all-mode retains positional identity and needs no handoff."""
|
||||
block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=16,
|
||||
num_gpu_blocks=24,
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
|
||||
assert attention_manager.has_positionally_stable_blocks
|
||||
assert mamba_manager.has_positionally_stable_blocks
|
||||
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
|
||||
attention_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
|
||||
)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=1
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
# build_connector_meta runs before the scheduler commits this step's
|
||||
# tokens, so the positional scan picks the blocks up on the next step.
|
||||
first = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 2 * block_size},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
assert first.store_gpu_blocks == []
|
||||
req.num_computed_tokens = 2 * block_size
|
||||
meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 1},
|
||||
cached_req_new_blocks={req.request_id: None},
|
||||
)
|
||||
)
|
||||
|
||||
assert set(meta.store_gpu_blocks) == {
|
||||
*(block.block_id for block in attention_blocks),
|
||||
*(block.block_id for block in mamba_blocks),
|
||||
}
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"])
|
||||
def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None:
|
||||
"""Handoffs for a request leaving this step must not be read.
|
||||
|
||||
@@ -189,7 +189,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
|
||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "all", "align"])
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
|
||||
"""Mamba groups hold the running-state block plus the speculative tail.
|
||||
|
||||
@@ -227,14 +227,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
|
||||
|
||||
|
||||
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
"""Attention, circular, and Mamba groups exercise every reservation branch.
|
||||
|
||||
"none" and "all" reach the same branch of both `_reserved_block_count` and
|
||||
`MambaManager`, which tests only for "align". They are still both listed:
|
||||
the mode is a spec-level input, and having the real manager confirm the
|
||||
prediction for each is what keeps a future divergence between them from
|
||||
landing unnoticed.
|
||||
"""
|
||||
"""Attention, circular, and Mamba groups exercise every reservation branch."""
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
@@ -242,7 +235,6 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
_attention_group(),
|
||||
_circular_group(),
|
||||
_mamba_group("none"),
|
||||
_mamba_group("all"),
|
||||
_mamba_group("align"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -398,7 +398,7 @@ def _make_kv_cache_config(cfg: _TestConfig, layer_names: list[str]) -> KVCacheCo
|
||||
(cfg.temporal_state_dim,),
|
||||
),
|
||||
dtypes=(cfg.dtype, cfg.dtype),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
group = KVCacheGroupSpec(
|
||||
layer_names=layer_names,
|
||||
|
||||
+1
-18
@@ -59,7 +59,7 @@ CacheDType = Literal[
|
||||
|
||||
|
||||
MambaDType = Literal["auto", "float32", "float16", "bfloat16"]
|
||||
MambaCacheMode = Literal["all", "align", "none"]
|
||||
MambaCacheMode = Literal["align", "none"]
|
||||
PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"]
|
||||
KVOffloadingBackend = Literal["native", "lmcache"]
|
||||
|
||||
@@ -75,8 +75,6 @@ class CacheConfig:
|
||||
Accepts None (meaning "use default"). After construction, always int."""
|
||||
user_specified_block_size: bool = field(default=False, init=False)
|
||||
"""Whether block_size was explicitly provided. Derived automatically."""
|
||||
user_specified_mamba_block_size: bool = field(default=False, init=False)
|
||||
"""Whether mamba_block_size was explicitly provided. Derived automatically."""
|
||||
kv_cache_layout: str | None = field(default=None, init=False)
|
||||
"""Resolved physical KV cache layout name (a ``KVCacheLayout`` member).
|
||||
|
||||
@@ -191,7 +189,6 @@ class CacheConfig:
|
||||
"""The cache strategy for Mamba layers:
|
||||
|
||||
- "none": set when prefix caching is disabled.
|
||||
- "all": cache the mamba state of all tokens at position i * block_size.
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size. This is the default when prefix
|
||||
caching is enabled.
|
||||
@@ -293,7 +290,6 @@ class CacheConfig:
|
||||
"mamba_page_size_padded",
|
||||
"skip_page_size_padded",
|
||||
"user_specified_block_size",
|
||||
"user_specified_mamba_block_size",
|
||||
"_block_size_resolved",
|
||||
# Post-init/derived counters
|
||||
"num_gpu_blocks",
|
||||
@@ -337,21 +333,8 @@ class CacheConfig:
|
||||
self.block_size = self.DEFAULT_BLOCK_SIZE
|
||||
else:
|
||||
self.user_specified_block_size = True
|
||||
if self.mamba_block_size is not None:
|
||||
self.user_specified_mamba_block_size = True
|
||||
return self
|
||||
|
||||
@field_validator("mamba_cache_mode", mode="after")
|
||||
@classmethod
|
||||
def _validate_mamba_cache_mode(cls, mode: MambaCacheMode) -> MambaCacheMode:
|
||||
if mode == "all":
|
||||
logger.warning_once(
|
||||
"Mamba cache mode 'all' is deprecated and will be removed in an "
|
||||
"upcoming release. If this is a problem, please open an issue "
|
||||
"at https://github.com/vllm-project/vllm/issues."
|
||||
)
|
||||
return mode
|
||||
|
||||
@field_validator("cache_dtype", mode="after")
|
||||
@classmethod
|
||||
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
|
||||
|
||||
@@ -3091,9 +3091,6 @@ class VllmConfig:
|
||||
if self.parallel_config.use_ubatching:
|
||||
unsupported.extend(self._get_dbo_unsupported_features())
|
||||
|
||||
if self.cache_config.mamba_cache_mode == "all":
|
||||
unsupported.append("mamba cache mode 'all'")
|
||||
|
||||
return unsupported
|
||||
|
||||
def _get_v1_model_runner_unsupported_features(self) -> list[str]:
|
||||
@@ -3499,10 +3496,6 @@ class VllmConfig:
|
||||
"SSM state caches, not --enable-mamba-cache-stochastic-"
|
||||
"rounding, which requires an explicit float16 cache"
|
||||
)
|
||||
if self.cache_config.mamba_cache_mode not in ("none", "align"):
|
||||
raise ValueError(
|
||||
"RecoverSSM supports only none and align Mamba cache modes"
|
||||
)
|
||||
if (
|
||||
self.cache_config.mamba_cache_mode == "align"
|
||||
and not self.use_v2_model_runner
|
||||
@@ -3535,11 +3528,6 @@ class VllmConfig:
|
||||
"Mamba2 ReplaySSM speculative decoding requires "
|
||||
"--mamba-backend flashinfer"
|
||||
)
|
||||
elif self.cache_config.mamba_cache_mode == "all":
|
||||
raise ValueError(
|
||||
"--use-replayssm supports prefix caching only in align mode; "
|
||||
"pass --mamba-cache-mode align"
|
||||
)
|
||||
elif self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
raise ValueError(
|
||||
|
||||
@@ -33,16 +33,14 @@ EngineId = str
|
||||
BlockIds = tuple[list[int], ...] | list[list[int]]
|
||||
|
||||
|
||||
def clip_ssm_state_blocks(
|
||||
blocks: list[int], num_spec_blocks: int, positional: bool
|
||||
) -> list[int]:
|
||||
"""Drop speculative scratch slots, retaining position-indexed or running state."""
|
||||
def clip_ssm_state_blocks(blocks: list[int], num_spec_blocks: int) -> list[int]:
|
||||
"""Drop speculative scratch slots and keep only the running-state slot."""
|
||||
if not blocks:
|
||||
return blocks
|
||||
# Keep at least the running-state slot, even before scratch allocation.
|
||||
if num_scratch := min(num_spec_blocks, len(blocks) - 1):
|
||||
blocks = blocks[:-num_scratch]
|
||||
return blocks if positional else blocks[-1:]
|
||||
return blocks[-1:]
|
||||
|
||||
|
||||
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
|
||||
@@ -450,13 +450,6 @@ def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
|
||||
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
|
||||
f"method={speculative_config.method!r}"
|
||||
)
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
# Positional allocation counts lookahead, so its local tail can exceed
|
||||
# the single running-state block supported by hybrid READ.
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ does not support DSpark with "
|
||||
"mamba_cache_mode='all'; use 'align'"
|
||||
)
|
||||
|
||||
|
||||
def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
|
||||
@@ -521,11 +514,6 @@ class MoRIIOConnectorScheduler:
|
||||
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
|
||||
"mode only; set kv_connector_extra_config.read_mode=true"
|
||||
)
|
||||
# Only "all" mode keeps a state per block position; the other modes keep
|
||||
# a single running state in the last slot.
|
||||
self._ssm_state_slots_are_positional = (
|
||||
vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
)
|
||||
self._num_ssm_scratch_blocks = (
|
||||
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
|
||||
)
|
||||
@@ -1288,8 +1276,8 @@ class MoRIIOConnectorScheduler:
|
||||
"""Select transferable attention and Mamba block groups.
|
||||
|
||||
The wire payload stores the attention group first, followed by every
|
||||
Mamba group in transfer-group order. Outside ``mamba_cache_mode="all"``,
|
||||
only each group's running-state slot is transferred.
|
||||
Mamba group in transfer-group order. Only each group's running-state
|
||||
slot is transferred.
|
||||
"""
|
||||
if not block_ids:
|
||||
return [], []
|
||||
@@ -1301,7 +1289,6 @@ class MoRIIOConnectorScheduler:
|
||||
clip_ssm_state_blocks(
|
||||
list(transfer_block_ids[group_id]),
|
||||
self._num_ssm_scratch_blocks,
|
||||
self._ssm_state_slots_are_positional,
|
||||
)
|
||||
for group_id in self._mamba_group_ids
|
||||
]
|
||||
|
||||
@@ -154,11 +154,6 @@ class NixlBaseConnectorScheduler:
|
||||
else None
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# Only "all" mode keeps a state per block position; the other modes
|
||||
# keep a single running state in the last non-speculative slot.
|
||||
self._ssm_state_slots_are_positional = (
|
||||
vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
)
|
||||
|
||||
# Threshold to decide whether to compute kv cache locally
|
||||
# or pull from a remote node: minimum number of remote
|
||||
@@ -258,11 +253,9 @@ class NixlBaseConnectorScheduler:
|
||||
out-of-window blocks only prior to the `request_finished_all_groups`
|
||||
hook.
|
||||
|
||||
SSM groups keep only their state-bearing slots: the trailing
|
||||
speculative scratch slots always go, and in single-state cache modes
|
||||
so does everything before the running state (null placeholders and
|
||||
the previous step's superseded state). "all" mode keeps its remaining
|
||||
slots, which the worker pairs position-wise.
|
||||
SSM groups keep only their state-bearing slot: the trailing
|
||||
speculative scratch slots and everything before the running state
|
||||
(null placeholders and the previous step's superseded state) go.
|
||||
|
||||
Use this at every block-id exchange point. Pass ``clip_ssm=False``
|
||||
for per-step partial lists (host-buffer save), where the SSM strip
|
||||
@@ -289,9 +282,7 @@ class NixlBaseConnectorScheduler:
|
||||
and blocks
|
||||
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
|
||||
):
|
||||
blocks = clip_ssm_state_blocks(
|
||||
blocks, n_spec_blocks, self._ssm_state_slots_are_positional
|
||||
)
|
||||
blocks = clip_ssm_state_blocks(blocks, n_spec_blocks)
|
||||
clipped.append(blocks)
|
||||
return tuple(clipped)
|
||||
|
||||
|
||||
@@ -3510,13 +3510,12 @@ class NixlBaseConnectorWorker:
|
||||
if _is_ssm_spec(self._group_spec_types[i]):
|
||||
if num_decode_blocks == num_prefill_blocks:
|
||||
continue
|
||||
# Only state-bearing slots reach here, single-state modes
|
||||
# just one (see get_exchange_clipped_blocks), so differing
|
||||
# counts mean position-indexed "all"-mode lists. A longer
|
||||
# prefill list carries earlier positions the decode side
|
||||
# already has (prefix hit) -> take its tail; a longer decode
|
||||
# list holds the position D recomputes itself, which gets
|
||||
# no prefill state.
|
||||
# Only state-bearing slots reach here, normally just one
|
||||
# (see get_exchange_clipped_blocks). A longer prefill list
|
||||
# carries earlier positions the decode side already has
|
||||
# (prefix hit) -> take its tail; a longer decode list holds
|
||||
# the position D recomputes itself, which gets no prefill
|
||||
# state.
|
||||
assert num_decode_blocks - num_prefill_blocks <= 1, (
|
||||
f"Group {i}: unpairable SSM state slots, "
|
||||
f"decode={num_decode_blocks} prefill={num_prefill_blocks}"
|
||||
|
||||
@@ -156,7 +156,7 @@ def resolve_mamba_align_size(
|
||||
"""Scan all KV cache groups in *spec* and return the single mamba alignment
|
||||
size, or None if no group requires mamba alignment.
|
||||
|
||||
For MambaSpec groups in "align" or "all" cache mode the hit window must be
|
||||
For MambaSpec groups in "align" cache mode the hit window must be
|
||||
rounded down to a multiple of the offloaded chunk size. Asserts that all
|
||||
such groups agree on the same value.
|
||||
"""
|
||||
@@ -164,10 +164,7 @@ def resolve_mamba_align_size(
|
||||
for group in spec.config.groups:
|
||||
tokens_per_block = group.tokens_per_block
|
||||
kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in (
|
||||
"align",
|
||||
"all",
|
||||
):
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
|
||||
tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
|
||||
assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
|
||||
mamba_align_size = tokens_per_chunk
|
||||
|
||||
@@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
assert self.cache_config is not None
|
||||
mamba_block_size = self.cache_config.mamba_block_size
|
||||
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
|
||||
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
@@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
num_prefills = attn_metadata.num_prefills # request count
|
||||
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
|
||||
has_prefill = num_prefill_tokens > 0
|
||||
has_decode = num_decode_tokens > 0
|
||||
num_actual_tokens = num_prefill_tokens + num_decode_tokens
|
||||
@@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
gate_p = prefill_decode_split.gate_p
|
||||
gate_d = prefill_decode_split.gate_d
|
||||
|
||||
if is_mamba_cache_all:
|
||||
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
|
||||
torch.split(
|
||||
attn_metadata.block_idx_last_computed_token,
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
)
|
||||
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
|
||||
torch.split(
|
||||
attn_metadata.block_idx_last_scheduled_token,
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
)
|
||||
|
||||
block_idx_first_scheduled_token_p = (
|
||||
attn_metadata.block_idx_first_scheduled_token_p
|
||||
)
|
||||
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
|
||||
else:
|
||||
block_idx_last_computed_token_d = None
|
||||
block_idx_last_computed_token_p = None
|
||||
block_idx_last_scheduled_token_d = None
|
||||
block_idx_last_scheduled_token_p = None
|
||||
block_idx_first_scheduled_token_p = None
|
||||
num_computed_tokens_p = None
|
||||
|
||||
ssm_outputs = []
|
||||
|
||||
if has_prefill:
|
||||
@@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
query_start_loc=query_start_loc_p,
|
||||
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
|
||||
initial_state_idx=block_idx_last_computed_token_p,
|
||||
num_computed_tokens=num_computed_tokens_p,
|
||||
block_size_to_align=mamba_block_size,
|
||||
metadata=attn_metadata,
|
||||
)
|
||||
# 3. State Space Model sequence transformations.
|
||||
@@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
cache_indices=state_indices_tensor_p,
|
||||
has_initial_state=has_initial_states_p,
|
||||
query_start_loc=query_start_loc_p,
|
||||
block_size=mamba_block_size,
|
||||
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
|
||||
initial_state_idx=block_idx_last_computed_token_p,
|
||||
cu_chunk_seqlen=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
)
|
||||
@@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
# state_indices_tensor_d is assigned when attn_metadata is not None,
|
||||
# and has_decode is only True when attn_metadata is not None
|
||||
assert state_indices_tensor_d is not None
|
||||
if is_mamba_cache_all:
|
||||
state_indices_tensor_d_input = state_indices_tensor_d.gather(
|
||||
1, block_idx_last_computed_token_d.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
state_indices_tensor_d_output = state_indices_tensor_d.gather(
|
||||
1, block_idx_last_scheduled_token_d.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
else:
|
||||
state_indices_tensor_d_input = state_indices_tensor_d
|
||||
state_indices_tensor_d_output = state_indices_tensor_d
|
||||
# 2. Convolution sequence transformation
|
||||
conv_out_d = causal_conv1d_update(
|
||||
hidden_states_BC_d.transpose(0, 1),
|
||||
@@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
|
||||
initial_state_idx=block_idx_last_computed_token_d,
|
||||
).transpose(0, 1)
|
||||
|
||||
# 3. State Space Model sequence transformation.
|
||||
@@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
time_proj_bias,
|
||||
z=gate_d.transpose(0, 1),
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
dst_state_batch_indices=state_indices_tensor_d_output,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
dst_state_batch_indices=state_indices_tensor_d,
|
||||
out=scan_outputs_d,
|
||||
)
|
||||
scan_outputs_d = scan_outputs_d.transpose(0, 1)
|
||||
|
||||
@@ -549,12 +549,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
self._updates_replayssm_trackers = True
|
||||
|
||||
self.num_spec = vllm_config.num_speculative_tokens
|
||||
if self.num_spec > 0:
|
||||
self.register_buffer(
|
||||
"_decode_state_offsets",
|
||||
torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
# Pre-compute sizes for forward pass
|
||||
self.tped_intermediate_size = self.intermediate_size // self.tp_size
|
||||
@@ -732,8 +726,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
|
||||
assert self.cache_config is not None
|
||||
mamba_block_size = self.cache_config.mamba_block_size
|
||||
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
|
||||
use_spec_decode = self.num_spec > 0
|
||||
ring_start = prev_num_accepted = prev_query_len = None
|
||||
|
||||
@@ -803,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
dim=0,
|
||||
)
|
||||
|
||||
if is_mamba_cache_all:
|
||||
# If prefix caching is enabled, retrieve the relevant variables
|
||||
# for prefill and decode
|
||||
block_idx_last_computed_token_d, block_idx_last_computed_token_p = (
|
||||
torch.split(
|
||||
attn_metadata.block_idx_last_computed_token,
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
)
|
||||
block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = (
|
||||
torch.split(
|
||||
attn_metadata.block_idx_last_scheduled_token,
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
)
|
||||
if attn_metadata.block_idx_last_scheduled_token_prev_step is not None:
|
||||
block_idx_last_scheduled_token_prev_step_d, _ = torch.split(
|
||||
attn_metadata.block_idx_last_scheduled_token_prev_step,
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
else:
|
||||
block_idx_last_scheduled_token_prev_step_d = None
|
||||
# Prefill-only variables:
|
||||
block_idx_first_scheduled_token_p = (
|
||||
attn_metadata.block_idx_first_scheduled_token_p
|
||||
)
|
||||
num_computed_tokens_p = attn_metadata.num_computed_tokens_p
|
||||
else:
|
||||
block_idx_last_computed_token_p = None
|
||||
block_idx_last_scheduled_token_p = None
|
||||
block_idx_first_scheduled_token_p = None
|
||||
block_idx_last_scheduled_token_d = None
|
||||
block_idx_last_computed_token_d = None
|
||||
block_idx_last_scheduled_token_prev_step_d = None
|
||||
num_computed_tokens_p = None
|
||||
|
||||
preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split(
|
||||
output[:num_actual_tokens],
|
||||
[num_decode_tokens, num_prefill_tokens],
|
||||
@@ -858,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# to by "state_indices_tensor_p".
|
||||
# In particular, it will always write the state at the
|
||||
# sequence end.
|
||||
# In addition, "block_idx_first_scheduled_token_p" and
|
||||
# "block_idx_last_scheduled_token_p"
|
||||
# are provided (which are pointers into
|
||||
# "state_indices_tensor_p"), it will write additional cache
|
||||
# states aligned at "block_size_to_align".
|
||||
x = hidden_states_B_C_p.transpose(
|
||||
0, 1
|
||||
) # this is the form that causal-conv see
|
||||
@@ -874,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
block_idx_first_scheduled_token=block_idx_first_scheduled_token_p,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token_p,
|
||||
initial_state_idx=block_idx_last_computed_token_p,
|
||||
num_computed_tokens=num_computed_tokens_p,
|
||||
block_size_to_align=mamba_block_size,
|
||||
metadata=attn_metadata,
|
||||
query_start_loc=query_start_loc_p,
|
||||
).transpose(0, 1)[:num_prefill_tokens]
|
||||
@@ -891,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
initial_states = None
|
||||
if has_initial_states_p is not None and prep_initial_states:
|
||||
assert state_indices_tensor_p is not None
|
||||
kernel_ssm_indices = state_indices_tensor_p
|
||||
if is_mamba_cache_all:
|
||||
kernel_ssm_indices = state_indices_tensor_p.gather(
|
||||
1, block_idx_last_computed_token_p.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
initial_states = torch.where(
|
||||
has_initial_states_p[:, None, None, None],
|
||||
ssm_state[kernel_ssm_indices],
|
||||
ssm_state[state_indices_tensor_p],
|
||||
0,
|
||||
)
|
||||
|
||||
@@ -921,130 +859,29 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
cu_chunk_seqlens=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
initial_states=initial_states,
|
||||
return_intermediate_states=is_mamba_cache_all,
|
||||
dt_softplus=True,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
|
||||
state_dtype=ssm_state.dtype,
|
||||
)
|
||||
|
||||
if is_mamba_cache_all:
|
||||
assert mamba_block_size is not None
|
||||
assert state_indices_tensor_p is not None
|
||||
assert block_idx_first_scheduled_token_p is not None
|
||||
assert block_idx_last_scheduled_token_p is not None
|
||||
assert last_chunk_indices_p is not None
|
||||
assert num_computed_tokens_p is not None
|
||||
|
||||
# The chunk_stride is the number of chunks per mamba block
|
||||
# e.g., if mamba_block_size = 512 and chunk_size = 256,
|
||||
# then chunk_stride = 2
|
||||
chunk_stride = mamba_block_size // chunk_size
|
||||
|
||||
# Save state for sequences with more than just final state,
|
||||
# batched over sequences so that the number of kernel launches
|
||||
# does not scale with the number of prefills.
|
||||
n_blocks_to_fill = (
|
||||
block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p
|
||||
).clamp(min=0)
|
||||
total_fill = attn_metadata.num_state_writes_p
|
||||
if total_fill > 0:
|
||||
first_chunk = torch.zeros_like(last_chunk_indices_p)
|
||||
first_chunk[1:] = last_chunk_indices_p[:-1] + 1
|
||||
|
||||
# The chunk that completes this sequence's first mamba block.
|
||||
# A block spans chunk_stride chunks. When the sequence resumed
|
||||
# mid-block, the chunks before this step are not in varlen_states,
|
||||
# so subtract them: the boundary sits earlier by their count.
|
||||
num_unaligned_computed_tokens = (
|
||||
num_computed_tokens_p % mamba_block_size
|
||||
)
|
||||
unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size
|
||||
first_aligned_chunk = (
|
||||
first_chunk + (chunk_stride - 1) - unaligned_chunk_shift
|
||||
)
|
||||
|
||||
# Flatten all writes into one list: fill_counts=[2,1,2]
|
||||
# -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1]
|
||||
fill_counts = n_blocks_to_fill.long()
|
||||
# output_size, or repeat_interleave reads the size off the device
|
||||
seq_ids = torch.repeat_interleave(
|
||||
torch.arange(num_prefills, device=ssm_state.device),
|
||||
fill_counts,
|
||||
output_size=total_fill,
|
||||
)
|
||||
write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts
|
||||
write_offsets = torch.arange(
|
||||
total_fill, device=ssm_state.device
|
||||
) - torch.repeat_interleave(
|
||||
write_starts, fill_counts, output_size=total_fill
|
||||
)
|
||||
|
||||
cache_blocks_to_fill = state_indices_tensor_p[
|
||||
seq_ids,
|
||||
block_idx_first_scheduled_token_p[seq_ids] + write_offsets,
|
||||
]
|
||||
# Write the states. Each row goes to its own cache block,
|
||||
# so the indexed copy has no duplicate destinations.
|
||||
ssm_state[cache_blocks_to_fill] = varlen_states[
|
||||
first_aligned_chunk[seq_ids] + write_offsets * chunk_stride
|
||||
]
|
||||
|
||||
# For all seqs, store the last state (note: might be partial):
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[
|
||||
state_indices_tensor_p.gather(
|
||||
1, block_idx_last_scheduled_token_p.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
] = varlen_states[last_chunk_indices_p]
|
||||
|
||||
else:
|
||||
# update ssm states
|
||||
# - varlen state is a (num_prefills, nheads, headdim, dstate)
|
||||
# tensor
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[state_indices_tensor_p] = varlen_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
assert prev_num_accepted is not None
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_indices_tensor_p,
|
||||
)
|
||||
# update ssm states
|
||||
# - varlen state is a (num_prefills, nheads, headdim, dstate)
|
||||
# tensor
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[state_indices_tensor_p] = varlen_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
assert prev_num_accepted is not None
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_indices_tensor_p,
|
||||
)
|
||||
|
||||
# Process decode requests
|
||||
if has_decode:
|
||||
assert state_indices_tensor_d is not None
|
||||
if is_mamba_cache_all:
|
||||
if use_spec_decode:
|
||||
assert block_idx_last_scheduled_token_prev_step_d is not None
|
||||
input_indices = (
|
||||
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
|
||||
+ self._decode_state_offsets
|
||||
)
|
||||
output_indices = (
|
||||
block_idx_last_scheduled_token_d.unsqueeze(1)
|
||||
+ self._decode_state_offsets
|
||||
)
|
||||
state_indices_tensor_d_input = state_indices_tensor_d.gather(
|
||||
1, input_indices
|
||||
)
|
||||
state_indices_tensor_d_output = state_indices_tensor_d.gather(
|
||||
1, output_indices
|
||||
)
|
||||
else:
|
||||
state_indices_tensor_d_input = state_indices_tensor_d.gather(
|
||||
1, block_idx_last_computed_token_d.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
state_indices_tensor_d_output = state_indices_tensor_d.gather(
|
||||
1, block_idx_last_scheduled_token_d.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
else:
|
||||
# Without caching, read and write in-place to the same blocks:
|
||||
state_indices_tensor_d_input = state_indices_tensor_d
|
||||
state_indices_tensor_d_output = state_indices_tensor_d
|
||||
|
||||
if (
|
||||
self.use_replayssm
|
||||
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
|
||||
@@ -1081,8 +918,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
self.conv1d.bias,
|
||||
self.activation,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token_d,
|
||||
initial_state_idx=block_idx_last_computed_token_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc_d,
|
||||
# ReplaySSM keeps one physical state block while a speculative
|
||||
@@ -1207,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
write_pos=attn_metadata.write_pos_d,
|
||||
is_flush=attn_metadata.is_flush_d,
|
||||
max_cache_len=self.replayssm_buffer_len,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
out=preallocated_ssm_out_d,
|
||||
enable_stochastic_rounding=(
|
||||
self.mamba_config.enable_stochastic_rounding
|
||||
@@ -1227,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
D_d,
|
||||
dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
dst_state_batch_indices=state_indices_tensor_d_output,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
dst_state_batch_indices=state_indices_tensor_d,
|
||||
out=preallocated_ssm_out_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cu_seqlens=query_start_loc_d,
|
||||
|
||||
@@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
cache_config.mamba_cache_mode,
|
||||
model_config.architecture,
|
||||
)
|
||||
if (
|
||||
cache_config.mamba_cache_mode == "all"
|
||||
and not model_config.supports_mamba_prefix_caching
|
||||
):
|
||||
cache_config.mamba_cache_mode = "align"
|
||||
logger.warning(
|
||||
"Hybrid or mamba-based model detected without support "
|
||||
"for prefix caching with Mamba cache 'all' mode: "
|
||||
"falling back to 'align' mode."
|
||||
)
|
||||
if cache_config.mamba_cache_mode == "align":
|
||||
assert vllm_config.scheduler_config.enable_chunked_prefill, (
|
||||
"Chunked prefill is required for mamba cache mode 'align'."
|
||||
)
|
||||
assert vllm_config.scheduler_config.enable_chunked_prefill, (
|
||||
"Chunked prefill is required for mamba cache mode 'align'."
|
||||
)
|
||||
# By default, mamba block size will be set to max_model_len (see
|
||||
# below). When enabling prefix caching, we align mamba block size
|
||||
# to the block size as the basic granularity for prefix caching.
|
||||
|
||||
@@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase):
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Intern-S2-Mobius does not support 'all' prefix caching; "
|
||||
"use '--mamba-cache-mode=align' instead."
|
||||
)
|
||||
|
||||
nn.Module.__init__(self)
|
||||
self.config = config
|
||||
self.model = InternS2MobiusModel(
|
||||
|
||||
@@ -458,13 +458,6 @@ class Lfm2ForCausalLM(
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Lfm2 currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.model = Lfm2Model(
|
||||
|
||||
@@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM(
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Lfm2Moe currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
@@ -329,14 +329,8 @@ class Qwen3_5ForCausalLMBase(
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen3.5 currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
|
||||
@@ -227,13 +227,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen3_5MTP currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
|
||||
@@ -828,14 +828,8 @@ class Qwen3NextForCausalLM(
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen3Next currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
|
||||
@@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.vllm_config = vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen3NextMTP currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
|
||||
@@ -104,7 +104,7 @@ def _mamba_get_block_table_tensor(
|
||||
kv_cache_spec: MambaSpec,
|
||||
mamba_cache_mode: str,
|
||||
) -> torch.Tensor:
|
||||
if mamba_cache_mode in ("all", "none"):
|
||||
if mamba_cache_mode == "none":
|
||||
return block_table
|
||||
|
||||
assert block_table.is_cuda and seq_lens.is_cuda
|
||||
|
||||
@@ -638,11 +638,6 @@ class Qwen4ExpForCausalLM(
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
self.model = Qwen4ExpModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
@@ -395,13 +395,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
|
||||
@@ -669,11 +669,6 @@ class Qwen4ExpForCausalLM(
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
self.model = Qwen4ExpModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
@@ -375,13 +375,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
raise NotImplementedError(
|
||||
"Qwen4ExpMTP currently does not support 'all' prefix caching, "
|
||||
"please use '--mamba-cache-mode=align' instead"
|
||||
)
|
||||
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
super().__init__()
|
||||
|
||||
@@ -939,13 +939,6 @@ class Platform:
|
||||
if mamba_page_size == 0:
|
||||
return
|
||||
|
||||
# mamba_block_size here should either be user specified value or None
|
||||
mamba_block_size = (
|
||||
cache_config.mamba_block_size
|
||||
if cache_config.user_specified_mamba_block_size
|
||||
else None
|
||||
)
|
||||
|
||||
# Get kernel block alignment from the backend's supported sizes
|
||||
with set_current_vllm_config(vllm_config):
|
||||
kernel_block_alignment_size = max(
|
||||
@@ -962,27 +955,15 @@ class Platform:
|
||||
# multiple of 128 so split kernel blocks keep that invariant.
|
||||
kernel_block_alignment_size = max(kernel_block_alignment_size, 128)
|
||||
|
||||
if cache_config.mamba_cache_mode == "all":
|
||||
# With prefix caching, align to mamba chunk size for kernel perf
|
||||
# TODO(tdoublep): this constraint can be relaxed fairly
|
||||
# easily by changing the way we layout chunks in the
|
||||
# mamba2 kernels.
|
||||
base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size()
|
||||
assert base_chunk_size is not None
|
||||
attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token)
|
||||
chunk_size = lcm(base_chunk_size, kernel_block_alignment_size)
|
||||
attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size)
|
||||
cache_config.mamba_block_size = attn_block_size
|
||||
else:
|
||||
# Without prefix caching, use minimum block size that satisfies
|
||||
# both backend alignment and mamba page size compatibility
|
||||
attn_block_size = kernel_block_alignment_size * cdiv(
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
# Use minimum block size that satisfies both backend alignment and
|
||||
# mamba page size compatibility
|
||||
attn_block_size = kernel_block_alignment_size * cdiv(
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata
|
||||
@@ -42,23 +42,4 @@ class Mamba1AttentionMetadataBuilder(
|
||||
fast_build: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> Mamba1AttentionMetadata:
|
||||
common = self._compute_common_metadata(common_attn_metadata)
|
||||
|
||||
if (
|
||||
common.num_prefills > 0
|
||||
and self.vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
):
|
||||
cu_chunk_seqlen_p, _, last_chunk_indices_p = (
|
||||
self._build_chunk_metadata_tensors(
|
||||
self.kv_cache_spec.block_size,
|
||||
common,
|
||||
common_attn_metadata,
|
||||
)
|
||||
)
|
||||
return replace(
|
||||
common,
|
||||
cu_chunk_seqlen_p=cu_chunk_seqlen_p,
|
||||
last_chunk_indices_p=last_chunk_indices_p,
|
||||
)
|
||||
|
||||
return common
|
||||
return self._compute_common_metadata(common_attn_metadata)
|
||||
|
||||
@@ -139,7 +139,6 @@ class Mamba2AttentionMetadataBuilder(
|
||||
common = self._compute_common_metadata(
|
||||
common_attn_metadata,
|
||||
num_accepted_tokens=kwargs.get("num_accepted_tokens"),
|
||||
prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"),
|
||||
num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"),
|
||||
)
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ class BaseMambaAttentionMetadata:
|
||||
# the batch has no prefill requests.
|
||||
has_initial_states_p: torch.Tensor | None
|
||||
query_start_loc_p: torch.Tensor | None
|
||||
num_computed_tokens_p: torch.Tensor | None
|
||||
state_indices_tensor_p: torch.Tensor | None
|
||||
|
||||
# The following tensors are used for decode requests and
|
||||
@@ -52,13 +51,6 @@ class BaseMambaAttentionMetadata:
|
||||
# Includes the bonus token (so minimum is 1)
|
||||
num_accepted_tokens: torch.Tensor | None # shape: [batch,]
|
||||
|
||||
# The following tensors are only used for prefix caching in all mode and
|
||||
# are None if disabled
|
||||
block_idx_last_scheduled_token: torch.Tensor | None
|
||||
block_idx_first_scheduled_token_p: torch.Tensor | None
|
||||
block_idx_last_computed_token: torch.Tensor | None
|
||||
block_idx_last_scheduled_token_prev_step: torch.Tensor | None
|
||||
|
||||
# The following tensor is only used for prefix caching in align mode
|
||||
seq_lens: torch.Tensor
|
||||
|
||||
@@ -70,9 +62,6 @@ class BaseMambaAttentionMetadata:
|
||||
# last_chunk_indices_p is a tensor of shape (batch,) that contains the
|
||||
# index of the last chunk for every sequence in the (prefill) batch.
|
||||
last_chunk_indices_p: torch.Tensor | None = None
|
||||
# Number of block-aligned SSM states the prefill path writes, i.e.
|
||||
# sum(block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p).
|
||||
num_state_writes_p: int = 0
|
||||
|
||||
# The following attributes are for triton implementation of causal_conv1d
|
||||
nums_dict: dict | None = None
|
||||
@@ -130,47 +119,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
self.compilation_config.max_cudagraph_capture_size,
|
||||
)
|
||||
|
||||
if self.vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
max_num_blocks = (
|
||||
cdiv(
|
||||
self.vllm_config.model_config.max_model_len,
|
||||
kv_cache_spec.block_size,
|
||||
)
|
||||
+ kv_cache_spec.num_speculative_blocks
|
||||
)
|
||||
# TODO: reduce this size as needed for decode-only cudagraph capture
|
||||
self.state_indices_tensor_d: torch.Tensor = torch.empty(
|
||||
(
|
||||
self.decode_cudagraph_max_bs,
|
||||
max_num_blocks,
|
||||
),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.block_idx_last_scheduled_token: torch.Tensor = torch.empty(
|
||||
(self.decode_cudagraph_max_bs,),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.block_idx_last_computed_token: torch.Tensor = torch.empty(
|
||||
(self.decode_cudagraph_max_bs,),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
if self.use_spec_decode:
|
||||
self.block_idx_last_scheduled_token_prev_step: torch.Tensor = (
|
||||
torch.empty(
|
||||
(self.decode_cudagraph_max_bs,),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.state_indices_tensor_d = torch.empty(
|
||||
(self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.state_indices_tensor_d = torch.empty(
|
||||
(self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# For speculative decoding, we need to store the following buffers
|
||||
# for CUDA graph capture during decode
|
||||
@@ -259,23 +212,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
if self.num_spec_tokens > 0:
|
||||
num_accepted_tokens = torch.diff(m.query_start_loc)
|
||||
|
||||
prev_last_scheduled_idx = None
|
||||
if (
|
||||
self.use_spec_decode
|
||||
and self.vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
):
|
||||
prev_last_scheduled_idx = torch.zeros(
|
||||
(m.num_reqs,),
|
||||
dtype=torch.int32,
|
||||
device=m.query_start_loc.device,
|
||||
)
|
||||
|
||||
return self.build(
|
||||
0,
|
||||
m,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
prev_last_scheduled_idx=prev_last_scheduled_idx,
|
||||
)
|
||||
return self.build(0, m, num_accepted_tokens=num_accepted_tokens)
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -284,7 +221,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
fast_build: bool = False,
|
||||
*,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
prev_last_scheduled_idx: torch.Tensor | None = None,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||
**kwargs: Any,
|
||||
) -> M:
|
||||
@@ -294,7 +230,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
return self._compute_common_metadata(
|
||||
common_attn_metadata,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
prev_last_scheduled_idx=prev_last_scheduled_idx,
|
||||
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||
)
|
||||
|
||||
@@ -423,56 +358,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
)
|
||||
return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p
|
||||
|
||||
def _compute_prefix_caching_block_indices(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
mamba_block_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
|
||||
# Block index of the last computed token
|
||||
block_idx_last_computed_token = (
|
||||
torch.div(
|
||||
num_computed_tokens + mamba_block_size - 1,
|
||||
mamba_block_size,
|
||||
rounding_mode="floor",
|
||||
)
|
||||
- 1
|
||||
)
|
||||
# which is <= block index for the first scheduled token
|
||||
block_idx_first_scheduled_token = (
|
||||
torch.div(
|
||||
num_computed_tokens + mamba_block_size,
|
||||
mamba_block_size,
|
||||
rounding_mode="floor",
|
||||
)
|
||||
- 1
|
||||
)
|
||||
# which is <= block index of the last scheduled token
|
||||
block_idx_last_scheduled_token = (
|
||||
torch.div(
|
||||
common_attn_metadata.seq_lens + mamba_block_size - 1,
|
||||
mamba_block_size,
|
||||
rounding_mode="floor",
|
||||
)
|
||||
- 1
|
||||
)
|
||||
# -1 in case it's non-computed and causes later issues with indexing
|
||||
block_idx_last_computed_token.clamp_(min=0)
|
||||
# -1 in the case we have a padded request (0 seq-len)
|
||||
block_idx_last_scheduled_token.clamp_(min=0)
|
||||
|
||||
return (
|
||||
block_idx_last_computed_token,
|
||||
block_idx_first_scheduled_token,
|
||||
block_idx_last_scheduled_token,
|
||||
)
|
||||
|
||||
def _compute_common_metadata(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
*,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
prev_last_scheduled_idx: torch.Tensor | None = None,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||
) -> M:
|
||||
"""Compute metadata common to both Mamba1 and Mamba2."""
|
||||
@@ -540,16 +430,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
has_initial_states_p = None
|
||||
query_start_loc_p = None
|
||||
query_start_loc_d = None
|
||||
num_computed_tokens = None
|
||||
num_computed_tokens_p = None
|
||||
|
||||
# for prefix caching
|
||||
block_idx_first_scheduled_token = None
|
||||
block_idx_first_scheduled_token_p = None
|
||||
block_idx_last_computed_token = None
|
||||
block_idx_last_scheduled_token = None
|
||||
block_idx_last_scheduled_token_prev_step = None
|
||||
num_state_writes_p = 0
|
||||
|
||||
# for causal_conv1d
|
||||
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
|
||||
@@ -557,35 +437,12 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
is_flush_d = None
|
||||
replayssm_scratch = None
|
||||
|
||||
if self.vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
|
||||
|
||||
# Return a tensor of shape (#requests, #max blocks)
|
||||
state_indices_tensor = common_attn_metadata.block_table_tensor
|
||||
# Additional cache-related variables:
|
||||
mamba_block_size = self.kv_cache_spec.block_size
|
||||
(
|
||||
block_idx_last_computed_token,
|
||||
block_idx_first_scheduled_token,
|
||||
block_idx_last_scheduled_token,
|
||||
) = self._compute_prefix_caching_block_indices(
|
||||
common_attn_metadata, mamba_block_size
|
||||
)
|
||||
if self.use_spec_decode and prev_last_scheduled_idx is not None:
|
||||
fallback = (num_computed_tokens - 1) // mamba_block_size
|
||||
fallback.clamp_(min=0)
|
||||
block_idx_last_scheduled_token_prev_step = torch.where(
|
||||
prev_last_scheduled_idx >= 0,
|
||||
prev_last_scheduled_idx,
|
||||
fallback,
|
||||
)
|
||||
else:
|
||||
state_indices_tensor = mamba_get_block_table_tensor(
|
||||
common_attn_metadata.block_table_tensor,
|
||||
common_attn_metadata.seq_lens,
|
||||
self.kv_cache_spec,
|
||||
self.vllm_config.cache_config.mamba_cache_mode,
|
||||
)
|
||||
state_indices_tensor = mamba_get_block_table_tensor(
|
||||
common_attn_metadata.block_table_tensor,
|
||||
common_attn_metadata.seq_lens,
|
||||
self.kv_cache_spec,
|
||||
self.vllm_config.cache_config.mamba_cache_mode,
|
||||
)
|
||||
|
||||
if state_indices_tensor.dim() == 1:
|
||||
state_indices_tensor = state_indices_tensor.unsqueeze(-1)
|
||||
@@ -595,11 +452,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
[num_decodes, num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
if self.vllm_config.cache_config.mamba_cache_mode != "all":
|
||||
state_indices_tensor_d = state_indices_tensor_d[
|
||||
:, : 1 + self.num_spec_tokens
|
||||
]
|
||||
state_indices_tensor_p = state_indices_tensor_p[:, 0]
|
||||
state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens]
|
||||
state_indices_tensor_p = state_indices_tensor_p[:, 0]
|
||||
|
||||
if num_decodes > 0 and self.use_spec_decode:
|
||||
query_start_loc_d = common_attn_metadata.query_start_loc[: num_decodes + 1]
|
||||
@@ -613,8 +467,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
num_accepted_tokens = num_accepted_tokens[:num_decodes]
|
||||
|
||||
if num_prefills > 0:
|
||||
if num_computed_tokens is None:
|
||||
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
|
||||
num_computed_tokens = common_attn_metadata.compute_num_computed_tokens()
|
||||
|
||||
query_start_loc_p = (
|
||||
common_attn_metadata.query_start_loc[-num_prefills - 1 :]
|
||||
@@ -636,33 +489,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
)
|
||||
)
|
||||
|
||||
if self.vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
assert num_computed_tokens is not None
|
||||
num_computed_tokens_p = num_computed_tokens[
|
||||
num_reqs - num_prefills : num_reqs
|
||||
]
|
||||
assert block_idx_first_scheduled_token is not None
|
||||
block_idx_first_scheduled_token_p = block_idx_first_scheduled_token[
|
||||
num_reqs - num_prefills : num_reqs
|
||||
]
|
||||
|
||||
# How many block-aligned states prefill will write: the same
|
||||
# (block_idx_last - block_idx_first) as above, from CPU data so
|
||||
# the mixer needs no D2H read. The -1 in both indices cancels.
|
||||
block_size = self.kv_cache_spec.block_size
|
||||
seq_lens_p_cpu = seq_lens_cpu[num_reqs - num_prefills : num_reqs]
|
||||
num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata(
|
||||
common_attn_metadata, num_reqs, num_prefills, num_decode_tokens
|
||||
)
|
||||
num_state_writes_p = int(
|
||||
(
|
||||
(seq_lens_p_cpu + block_size - 1) // block_size
|
||||
- (num_computed_tokens_p_cpu + block_size) // block_size
|
||||
)
|
||||
.clamp(min=0)
|
||||
.sum()
|
||||
)
|
||||
|
||||
if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0:
|
||||
decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
@@ -764,14 +590,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
replayssm_scratch=replayssm_scratch,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token,
|
||||
block_idx_first_scheduled_token_p=block_idx_first_scheduled_token_p,
|
||||
num_state_writes_p=num_state_writes_p,
|
||||
block_idx_last_computed_token=block_idx_last_computed_token,
|
||||
block_idx_last_scheduled_token_prev_step=(
|
||||
block_idx_last_scheduled_token_prev_step
|
||||
),
|
||||
num_computed_tokens_p=num_computed_tokens_p,
|
||||
num_reqs=num_reqs,
|
||||
seq_lens=common_attn_metadata.seq_lens,
|
||||
nums_dict=nums_dict,
|
||||
@@ -791,11 +609,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
state_indices_tensor_d = metadata.state_indices_tensor_d
|
||||
query_start_loc_d = metadata.query_start_loc_d
|
||||
num_accepted_tokens = metadata.num_accepted_tokens
|
||||
block_idx_last_scheduled_token = metadata.block_idx_last_scheduled_token
|
||||
block_idx_last_computed_token = metadata.block_idx_last_computed_token
|
||||
block_idx_last_scheduled_token_prev_step = (
|
||||
metadata.block_idx_last_scheduled_token_prev_step
|
||||
)
|
||||
write_pos_d = metadata.write_pos_d
|
||||
is_flush_d = metadata.is_flush_d
|
||||
bc_pre_scratch = metadata.bc_pre_scratch
|
||||
@@ -824,44 +637,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
1 # pad with 1st slot index
|
||||
)
|
||||
|
||||
if self.vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
assert block_idx_last_scheduled_token is not None
|
||||
assert block_idx_last_computed_token is not None
|
||||
self.block_idx_last_scheduled_token[: metadata.num_decodes].copy_(
|
||||
block_idx_last_scheduled_token[: metadata.num_decodes],
|
||||
non_blocking=True,
|
||||
)
|
||||
block_idx_last_scheduled_token = self.block_idx_last_scheduled_token[
|
||||
:padded_bs
|
||||
]
|
||||
block_idx_last_scheduled_token[metadata.num_decodes :] = 0
|
||||
|
||||
self.block_idx_last_computed_token[: metadata.num_decodes].copy_(
|
||||
block_idx_last_computed_token[: metadata.num_decodes],
|
||||
non_blocking=True,
|
||||
)
|
||||
block_idx_last_computed_token = self.block_idx_last_computed_token[
|
||||
:padded_bs
|
||||
]
|
||||
block_idx_last_computed_token[metadata.num_decodes :] = 0
|
||||
|
||||
if (
|
||||
self.use_spec_decode
|
||||
and block_idx_last_scheduled_token_prev_step is not None
|
||||
):
|
||||
self.block_idx_last_scheduled_token_prev_step[
|
||||
: metadata.num_decodes
|
||||
].copy_(
|
||||
block_idx_last_scheduled_token_prev_step[
|
||||
: metadata.num_decodes
|
||||
],
|
||||
non_blocking=True,
|
||||
)
|
||||
block_idx_last_scheduled_token_prev_step = (
|
||||
self.block_idx_last_scheduled_token_prev_step[:padded_bs]
|
||||
)
|
||||
block_idx_last_scheduled_token_prev_step[metadata.num_decodes :] = 0
|
||||
|
||||
if self.use_replayssm and not self.use_flashinfer_replayssm:
|
||||
assert write_pos_d is not None
|
||||
assert is_flush_d is not None
|
||||
@@ -914,11 +689,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
bc_pre_scratch=bc_pre_scratch,
|
||||
replayssm_scratch=replayssm_scratch,
|
||||
replayssm_state_indices_d=replayssm_state_indices_d,
|
||||
block_idx_last_scheduled_token=block_idx_last_scheduled_token,
|
||||
block_idx_last_computed_token=block_idx_last_computed_token,
|
||||
block_idx_last_scheduled_token_prev_step=(
|
||||
block_idx_last_scheduled_token_prev_step
|
||||
),
|
||||
)
|
||||
|
||||
def update_block_table(
|
||||
@@ -950,11 +720,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
[metadata.num_decodes, metadata.num_prefills],
|
||||
dim=0,
|
||||
)
|
||||
if self.vllm_config.cache_config.mamba_cache_mode != "all":
|
||||
state_indices_tensor_d = state_indices_tensor_d[
|
||||
:, : 1 + self.num_spec_tokens
|
||||
]
|
||||
state_indices_tensor_p = state_indices_tensor_p[:, 0]
|
||||
state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens]
|
||||
state_indices_tensor_p = state_indices_tensor_p[:, 0]
|
||||
|
||||
new_metadata = replace(
|
||||
metadata,
|
||||
|
||||
@@ -283,7 +283,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
|
||||
|
||||
has_initial_states_p = None
|
||||
has_initial_states_d = None
|
||||
num_computed_tokens_p = None
|
||||
|
||||
# Per-request classification by mask, NOT by position. With
|
||||
# spec-decode, the front decode group can contain both spec-decode
|
||||
@@ -402,10 +401,9 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
|
||||
assert non_spec_query_start_loc is not None
|
||||
query_start_loc_d = non_spec_query_start_loc[: num_decodes + 1]
|
||||
if num_prefills > 0:
|
||||
num_computed_tokens_p = num_computed_tokens[
|
||||
num_decodes : num_decodes + num_prefills
|
||||
]
|
||||
has_initial_states_p = num_computed_tokens_p > 0
|
||||
has_initial_states_p = (
|
||||
num_computed_tokens[num_decodes : num_decodes + num_prefills] > 0
|
||||
)
|
||||
assert non_spec_query_start_loc is not None
|
||||
query_start_loc_p = (
|
||||
non_spec_query_start_loc[num_decodes:] - num_decode_tokens
|
||||
@@ -478,11 +476,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder):
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
state_indices_tensor_p=state_indices_tensor_p,
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_computed_tokens_p=num_computed_tokens_p,
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=m.seq_lens,
|
||||
)
|
||||
|
||||
|
||||
@@ -1162,11 +1162,6 @@ def mamba_get_block_table_tensor(
|
||||
"""Get the block table tensor for mamba kernels from the input
|
||||
common_attn_metadata.block_table_tensor given different mamba cache modes.
|
||||
|
||||
- "all": input (#requests, cdiv(max_model_len, block_size)
|
||||
+ num_speculative_blocks);
|
||||
output (#requests, cdiv(max_model_len, block_size)
|
||||
+ num_speculative_blocks).
|
||||
|
||||
- "none": input (#requests, 1 + num_speculative_blocks);
|
||||
output (#requests, 1 + num_speculative_blocks).
|
||||
|
||||
@@ -1174,7 +1169,7 @@ def mamba_get_block_table_tensor(
|
||||
output (#requests, 1 + num_speculative_blocks), which are the last
|
||||
1 + num_speculative_blocks of each request.
|
||||
"""
|
||||
if mamba_cache_mode in ("all", "none"):
|
||||
if mamba_cache_mode == "none":
|
||||
return block_table
|
||||
else:
|
||||
assert isinstance(kv_cache_spec, MambaSpec)
|
||||
|
||||
@@ -1079,12 +1079,7 @@ class MambaSpec(KVCacheSpec):
|
||||
return None
|
||||
|
||||
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
return (
|
||||
cdiv(max_model_len, self.block_size) + self.num_speculative_blocks
|
||||
) * self.page_size_bytes
|
||||
elif vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
if vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
return self.page_size_bytes * (
|
||||
2 + self.num_speculative_blocks + self.num_prefill_checkpoint_blocks
|
||||
)
|
||||
|
||||
@@ -967,11 +967,6 @@ class GPUModelRunner(
|
||||
self.mamba_state_idx: dict[str, int] = {}
|
||||
self._mamba_bufs: mamba_utils.MambaBuffers | None = None
|
||||
self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None
|
||||
self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None
|
||||
if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0:
|
||||
self.mamba_prev_last_scheduled_idx = self._make_buffer(
|
||||
self.max_num_reqs, dtype=torch.int32
|
||||
)
|
||||
self.layerwise_nvtx_hooks_registered = False
|
||||
|
||||
def update_max_model_len(self, max_model_len: int) -> None:
|
||||
@@ -1608,17 +1603,6 @@ class GPUModelRunner(
|
||||
assert self.num_accepted_tokens_event is not None
|
||||
self.num_accepted_tokens_event.record()
|
||||
|
||||
if self.cache_config.mamba_cache_mode == "all":
|
||||
mamba_utils.postprocess_mamba_all(
|
||||
scheduler_output,
|
||||
self.kv_cache_config,
|
||||
self.input_batch,
|
||||
self.requests,
|
||||
self.mamba_state_idx,
|
||||
self.num_spec_tokens,
|
||||
num_reqs,
|
||||
)
|
||||
|
||||
def _update_streaming_request(
|
||||
self, req_id: str, new_req_data: NewRequestData
|
||||
) -> CachedRequestState:
|
||||
@@ -2127,15 +2111,6 @@ class GPUModelRunner(
|
||||
self.num_accepted_tokens.np.fill(1)
|
||||
self.num_accepted_tokens.gpu.fill_(1)
|
||||
|
||||
if self.mamba_prev_last_scheduled_idx is not None:
|
||||
mamba_utils.preprocess_mamba_all_specdec(
|
||||
scheduler_output,
|
||||
self.input_batch,
|
||||
self.mamba_state_idx,
|
||||
num_reqs,
|
||||
self.mamba_prev_last_scheduled_idx,
|
||||
)
|
||||
|
||||
# Update num_computed_tokens on GPU. In async spec decode,
|
||||
# CPU values are optimistic (all drafts accepted). The kernel
|
||||
# corrects on GPU using the previous step's
|
||||
@@ -2511,13 +2486,6 @@ class GPUModelRunner(
|
||||
:num_reqs_padded
|
||||
],
|
||||
)
|
||||
if (
|
||||
isinstance(builder, Mamba2AttentionMetadataBuilder)
|
||||
and self.mamba_prev_last_scheduled_idx is not None
|
||||
):
|
||||
extra_attn_metadata_args["prev_last_scheduled_idx"] = (
|
||||
self.mamba_prev_last_scheduled_idx.gpu[:num_reqs_padded]
|
||||
)
|
||||
|
||||
if for_cudagraph_capture:
|
||||
attn_metadata_i = builder.build_for_cudagraph_capture(
|
||||
|
||||
@@ -1535,53 +1535,6 @@ def preprocess_mamba(
|
||||
do_mamba_copy_block(copy_bufs)
|
||||
|
||||
|
||||
def postprocess_mamba_all(
|
||||
scheduler_output: SchedulerOutput,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
input_batch: InputBatch,
|
||||
requests: dict[str, CachedRequestState],
|
||||
mamba_state_idx: dict[str, int],
|
||||
num_spec_tokens: int,
|
||||
num_reqs: int,
|
||||
):
|
||||
"""All-mode postprocess (only meaningful with num_spec_tokens > 0):
|
||||
record per-request the block index of the last token scheduled this
|
||||
step, so the next step can anchor its in-place writes when accepted
|
||||
drafts leave the sequence at a non-block-aligned position.
|
||||
"""
|
||||
if num_spec_tokens <= 0:
|
||||
return
|
||||
mamba_groups = get_mamba_groups(kv_cache_config)
|
||||
block_sizes = {mamba_spec.block_size for mamba_spec in mamba_groups}
|
||||
assert len(block_sizes) == 1, "all mamba groups must share block_size"
|
||||
block_size = next(iter(block_sizes))
|
||||
full_decode_len = 1 + num_spec_tokens
|
||||
scheduled = scheduler_output.num_scheduled_tokens
|
||||
for req_id in input_batch.req_ids[:num_reqs]:
|
||||
num_query = scheduled.get(req_id, 0)
|
||||
if num_query == full_decode_len:
|
||||
req = requests[req_id]
|
||||
seq_len = req.num_computed_tokens + num_query
|
||||
mamba_state_idx[req_id] = max(0, (seq_len - 1) // block_size)
|
||||
else:
|
||||
mamba_state_idx.pop(req_id, None)
|
||||
|
||||
|
||||
def preprocess_mamba_all_specdec(
|
||||
scheduler_output: SchedulerOutput,
|
||||
input_batch: InputBatch,
|
||||
mamba_state_idx: dict[str, int],
|
||||
num_reqs: int,
|
||||
prev_last_scheduled_idx_buf: CpuGpuBuffer,
|
||||
) -> None:
|
||||
cleanup_mamba_state_idx(scheduler_output, mamba_state_idx)
|
||||
np_view = prev_last_scheduled_idx_buf.np
|
||||
for i, req_id in enumerate(input_batch.req_ids[:num_reqs]):
|
||||
np_view[i] = mamba_state_idx.get(req_id, -1)
|
||||
np_view[num_reqs:].fill(-1)
|
||||
prev_last_scheduled_idx_buf.copy_to_gpu()
|
||||
|
||||
|
||||
def postprocess_mamba_align_gpu(
|
||||
*,
|
||||
bufs: "MambaBuffers",
|
||||
|
||||
Reference in New Issue
Block a user