[Core] Remove deprecated mamba_cache_mode "all" (#58997)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Thomas Parnell
parent
08d77cad7b
commit
cb6458c5ce
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
|
||||
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=None,
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([5]),
|
||||
)
|
||||
|
||||
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([1, 1]),
|
||||
)
|
||||
|
||||
|
||||
@@ -568,15 +568,10 @@ def test_replayssm_mixer_selects_mtp_layout(
|
||||
num_reqs=2,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
|
||||
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
|
||||
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
|
||||
|
||||
@@ -270,9 +270,6 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.use_v2_model_runner = True
|
||||
config.cache_config.mamba_cache_mode = "all"
|
||||
with pytest.raises(ValueError, match="only none and align"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
|
||||
config.model_config.architecture = "NemotronHForCausalLM"
|
||||
@@ -286,6 +283,12 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
|
||||
|
||||
def test_mamba_cache_mode_all_is_rejected():
|
||||
"""The removed 'all' mode must fail validation instead of being ignored."""
|
||||
with pytest.raises(ValidationError, match="mamba_cache_mode"):
|
||||
CacheConfig(mamba_cache_mode="all")
|
||||
|
||||
|
||||
def test_per_request_spec_decode_metrics_requires_spec_decode():
|
||||
# The flag only makes sense with speculative decoding configured; enabling
|
||||
# it without --speculative-config should fail fast rather than silently
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for BaseMambaAttentionMetadataBuilder batch classification."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.v1.attention.utils import MockMambaBuilder
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
num_speculative_tokens: int = 0,
|
||||
block_size: int | None = None,
|
||||
):
|
||||
"""Create a minimal mock VllmConfig with only the fields the builder
|
||||
accesses, avoiding any model download / HF config inspection."""
|
||||
speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
if num_speculative_tokens > 0
|
||||
else None
|
||||
)
|
||||
return SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="none",
|
||||
use_replayssm=False,
|
||||
replayssm_buffer_len=16,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
max_cudagraph_capture_size=None,
|
||||
),
|
||||
speculative_config=speculative_config,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
model_config=SimpleNamespace(max_model_len=max_model_len),
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_single_token_prompt_runs_as_prefill():
|
||||
seq_lens = [8, 9, 1]
|
||||
config = _make_vllm_config(256, len(seq_lens), block_size=16)
|
||||
metadata = MockMambaBuilder.build_mamba_metadata(
|
||||
config,
|
||||
seq_lens=seq_lens,
|
||||
query_lens=[1] * len(seq_lens),
|
||||
is_prefilling=[False, False, True],
|
||||
)
|
||||
|
||||
assert metadata.num_decodes == 2
|
||||
assert metadata.num_prefills == 1
|
||||
assert metadata.has_initial_states_p.tolist() == [False]
|
||||
@@ -1,466 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/34865.
|
||||
|
||||
When multiple KV cache groups share the same MambaSpec (as in Nemotron
|
||||
hybrid models), the metadata caching optimization reuses metadata from
|
||||
an earlier group via update_block_table(). In 'all' mode with CUDA graphs,
|
||||
update_block_table() must copy block_idx_last_scheduled_token and
|
||||
block_idx_last_computed_token to the *current* builder's persistent
|
||||
buffers, otherwise CUDA graph replay reads stale values from uninitialized
|
||||
buffers.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import MockMambaBuilder
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
|
||||
def _make_vllm_config(
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
num_speculative_tokens: int = 0,
|
||||
block_size: int | None = None,
|
||||
):
|
||||
"""Create a minimal mock VllmConfig with only the fields the builder
|
||||
accesses, avoiding any model download / HF config inspection."""
|
||||
speculative_config = (
|
||||
SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
if num_speculative_tokens > 0
|
||||
else None
|
||||
)
|
||||
return SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="all",
|
||||
use_replayssm=False,
|
||||
replayssm_buffer_len=16,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.FULL,
|
||||
max_cudagraph_capture_size=None,
|
||||
),
|
||||
speculative_config=speculative_config,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
model_config=SimpleNamespace(max_model_len=max_model_len),
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_single_token_prompt_runs_as_prefill():
|
||||
seq_lens = [8, 9, 1]
|
||||
config = _make_vllm_config(256, len(seq_lens), block_size=16)
|
||||
metadata = MockMambaBuilder.build_mamba_metadata(
|
||||
config,
|
||||
seq_lens=seq_lens,
|
||||
query_lens=[1] * len(seq_lens),
|
||||
is_prefilling=[False, False, True],
|
||||
)
|
||||
|
||||
assert metadata.num_decodes == 2
|
||||
assert metadata.num_prefills == 1
|
||||
assert metadata.has_initial_states_p.tolist() == [False]
|
||||
|
||||
|
||||
def test_update_block_table_copies_block_idx_to_persistent_buffers():
|
||||
"""update_block_table() must write block_idx tensors to the current
|
||||
builder's persistent buffers, not leave them pointing to a different
|
||||
builder's buffers."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
num_reqs = 4
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(max_model_len, num_reqs)
|
||||
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
|
||||
# Two builders simulating two KV cache groups with the same MambaSpec.
|
||||
builder_a = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
builder_b = MockMambaBuilder(spec, ["layer1"], vllm_config, device)
|
||||
|
||||
# Sanity: each builder has its own persistent buffer.
|
||||
assert (
|
||||
builder_a.block_idx_last_scheduled_token.data_ptr()
|
||||
!= builder_b.block_idx_last_scheduled_token.data_ptr()
|
||||
)
|
||||
|
||||
# Construct decode-only metadata as if builder_a.build() produced it.
|
||||
max_blocks = max_model_len // block_size
|
||||
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
|
||||
block_idx_vals = (seq_lens - 1) // block_size # [3, 3, 3, 3]
|
||||
|
||||
builder_a.block_idx_last_scheduled_token[:num_reqs].copy_(block_idx_vals)
|
||||
builder_a.block_idx_last_computed_token[:num_reqs].copy_(block_idx_vals)
|
||||
|
||||
metadata_a = BaseMambaAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_reqs,
|
||||
num_decode_tokens=num_reqs,
|
||||
num_reqs=num_reqs,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
query_start_loc_d=None,
|
||||
num_accepted_tokens=None,
|
||||
state_indices_tensor_d=builder_a.state_indices_tensor_d[:num_reqs],
|
||||
block_idx_last_scheduled_token=(
|
||||
builder_a.block_idx_last_scheduled_token[:num_reqs]
|
||||
),
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=(
|
||||
builder_a.block_idx_last_computed_token[:num_reqs]
|
||||
),
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
|
||||
# Call update_block_table on builder_b (simulates the metadata caching
|
||||
# optimization reusing metadata from builder_a's group).
|
||||
blk_table = torch.randint(
|
||||
0, 100, (num_reqs, max_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
slot_mapping = torch.zeros(num_reqs, dtype=torch.int64, device=device)
|
||||
|
||||
metadata_b = builder_b.update_block_table(metadata_a, blk_table, slot_mapping)
|
||||
|
||||
# block_idx tensors must live in builder_b's persistent buffers.
|
||||
def shares_storage(tensor, buffer):
|
||||
return (
|
||||
tensor.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
assert shares_storage(
|
||||
metadata_b.block_idx_last_scheduled_token,
|
||||
builder_b.block_idx_last_scheduled_token,
|
||||
), "block_idx_last_scheduled_token not in builder_b's persistent buffer"
|
||||
|
||||
assert shares_storage(
|
||||
metadata_b.block_idx_last_computed_token,
|
||||
builder_b.block_idx_last_computed_token,
|
||||
), "block_idx_last_computed_token not in builder_b's persistent buffer"
|
||||
|
||||
# Must NOT point to builder_a's buffers.
|
||||
assert not shares_storage(
|
||||
metadata_b.block_idx_last_scheduled_token,
|
||||
builder_a.block_idx_last_scheduled_token,
|
||||
), "block_idx_last_scheduled_token still points to builder_a's buffer"
|
||||
|
||||
# Values must be correct (copied from metadata_a).
|
||||
torch.testing.assert_close(
|
||||
metadata_b.block_idx_last_scheduled_token,
|
||||
block_idx_vals,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
metadata_b.block_idx_last_computed_token,
|
||||
block_idx_vals,
|
||||
)
|
||||
|
||||
|
||||
def test_state_indices_tensor_d_includes_num_speculative_blocks():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
|
||||
bug 1: with mamba_cache_mode='all' and speculative decoding enabled,
|
||||
the cudagraph buffer for state_indices_tensor_d must allocate the same
|
||||
per-request column count as the runtime block table, which includes
|
||||
num_speculative_blocks trailing scratch columns."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 4
|
||||
num_speculative_tokens = 1
|
||||
num_speculative_blocks = 2
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
expected_cols = (max_model_len // block_size) + num_speculative_blocks
|
||||
assert builder.state_indices_tensor_d.shape == (max_num_seqs, expected_cols)
|
||||
|
||||
|
||||
def test_block_idx_cudagraph_capture_padded_by_num_reqs():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/39809
|
||||
bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for
|
||||
_cudagraph_capture must slice block_idx_last_{scheduled,computed}_token
|
||||
by the request count (padded_bs == num_reqs), not by num_decode_tokens.
|
||||
Past num_decodes, the slice must be zero-filled."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
num_speculative_tokens = 1
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
builder.block_idx_last_scheduled_token.fill_(-1)
|
||||
builder.block_idx_last_computed_token.fill_(-1)
|
||||
|
||||
num_decodes = 2
|
||||
num_reqs = 3
|
||||
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
|
||||
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
|
||||
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
|
||||
state_indices_d = torch.zeros(
|
||||
(num_decodes, builder.state_indices_tensor_d.shape[1]),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
query_start_loc_d = torch.arange(
|
||||
num_decodes + 1, dtype=torch.int32, device=device
|
||||
) * (1 + num_speculative_tokens)
|
||||
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
|
||||
|
||||
metadata = BaseMambaAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=num_reqs,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=state_indices_d,
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
block_idx_last_scheduled_token=block_idx_vals,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=block_idx_vals,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
|
||||
out = builder._update_metadata_for_cudagraph_capture(metadata)
|
||||
|
||||
assert out.block_idx_last_scheduled_token.shape == (num_reqs,)
|
||||
assert out.block_idx_last_computed_token.shape == (num_reqs,)
|
||||
torch.testing.assert_close(
|
||||
out.block_idx_last_scheduled_token[:num_decodes], block_idx_vals
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
out.block_idx_last_computed_token[:num_decodes], block_idx_vals
|
||||
)
|
||||
assert torch.all(out.block_idx_last_scheduled_token[num_decodes:] == 0)
|
||||
assert torch.all(out.block_idx_last_computed_token[num_decodes:] == 0)
|
||||
|
||||
|
||||
def test_block_idx_prev_step_persistent_buffer_allocated():
|
||||
"""With mamba_cache_mode='all' + spec decode, the builder must allocate
|
||||
block_idx_last_scheduled_token_prev_step as a persistent buffer with the
|
||||
same shape as the existing block_idx_last_{scheduled,computed}_token
|
||||
buffers, so cudagraph capture records a stable pointer for the prev-step
|
||||
input anchor consumed by mamba_mixer2's input gather."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
num_speculative_tokens = 1
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
assert hasattr(builder, "block_idx_last_scheduled_token_prev_step")
|
||||
assert builder.block_idx_last_scheduled_token_prev_step.shape == (max_num_seqs,)
|
||||
assert builder.block_idx_last_scheduled_token_prev_step.dtype == torch.int32
|
||||
|
||||
|
||||
def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode():
|
||||
"""Without spec decode, the prev-step buffer is unused and must not be
|
||||
allocated — the input anchor reduces to last_computed_token."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len, max_num_seqs, num_speculative_tokens=0
|
||||
)
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
|
||||
assert not hasattr(builder, "block_idx_last_scheduled_token_prev_step")
|
||||
|
||||
|
||||
def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer():
|
||||
"""_update_metadata_for_cudagraph_capture must copy the prev-step anchor
|
||||
into the builder's persistent buffer (so cudagraph replay reads from the
|
||||
same underlying memory), pad past num_decodes with zero, and return a
|
||||
slice of the persistent buffer in the metadata."""
|
||||
block_size = 16
|
||||
max_model_len = 256
|
||||
max_num_seqs = 8
|
||||
num_speculative_tokens = 1
|
||||
device = torch.device("cpu")
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
max_model_len,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
spec = MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,), (1,)),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=2,
|
||||
)
|
||||
builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device)
|
||||
builder.block_idx_last_scheduled_token.fill_(-1)
|
||||
builder.block_idx_last_computed_token.fill_(-1)
|
||||
builder.block_idx_last_scheduled_token_prev_step.fill_(-1)
|
||||
|
||||
num_decodes = 2
|
||||
num_reqs = 3
|
||||
num_decode_tokens = num_decodes * (1 + num_speculative_tokens)
|
||||
seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device)
|
||||
block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device)
|
||||
prev_step_vals = torch.tensor([2, 4], dtype=torch.int32, device=device)
|
||||
state_indices_d = torch.zeros(
|
||||
(num_decodes, builder.state_indices_tensor_d.shape[1]),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
query_start_loc_d = torch.arange(
|
||||
num_decodes + 1, dtype=torch.int32, device=device
|
||||
) * (1 + num_speculative_tokens)
|
||||
num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device)
|
||||
|
||||
metadata = BaseMambaAttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=num_reqs,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=state_indices_d,
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
block_idx_last_scheduled_token=block_idx_vals,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=block_idx_vals,
|
||||
block_idx_last_scheduled_token_prev_step=prev_step_vals,
|
||||
seq_lens=seq_lens,
|
||||
)
|
||||
|
||||
out = builder._update_metadata_for_cudagraph_capture(metadata)
|
||||
|
||||
# Output field exists and is identity-shared with the persistent buffer.
|
||||
assert out.block_idx_last_scheduled_token_prev_step is not None
|
||||
assert (
|
||||
out.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
|
||||
== builder.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr()
|
||||
), (
|
||||
"prev-step buffer must live in the builder's persistent buffer, not "
|
||||
"in the caller-provided tensor"
|
||||
)
|
||||
|
||||
# Padded by num_reqs (not num_decode_tokens) — same fix as bug 2 for the
|
||||
# other block_idx_* fields.
|
||||
assert out.block_idx_last_scheduled_token_prev_step.shape == (num_reqs,)
|
||||
|
||||
# First num_decodes values: input values copied through.
|
||||
torch.testing.assert_close(
|
||||
out.block_idx_last_scheduled_token_prev_step[:num_decodes],
|
||||
prev_step_vals,
|
||||
)
|
||||
|
||||
# Tail values past num_decodes: zero-filled padding for cudagraph capture.
|
||||
assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0)
|
||||
|
||||
|
||||
def test_num_state_writes_p_matches_block_idx_tensors():
|
||||
"""The prefill state save sizes its batched copy from
|
||||
num_state_writes_p instead of reading the block index tensors back from
|
||||
the GPU, so the host count must equal what those tensors say."""
|
||||
# (block_size, seq_lens, query_lens, is_prefilling)
|
||||
cases = [
|
||||
# block-aligned and unaligned computed tokens, and a chunked prefill
|
||||
(16, [64, 33, 17], [48, 17, 1], [True, True, True]),
|
||||
(32, [200, 100, 64], [200, 36, 64], [True, True, True]),
|
||||
# decode rows ahead of the prefills
|
||||
(16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]),
|
||||
# prefills that stay inside their first block: nothing to write
|
||||
(16, [65, 66, 67], [1, 2, 3], [True, True, True]),
|
||||
]
|
||||
|
||||
for block_size, seq_lens, query_lens, is_prefilling in cases:
|
||||
metadata = MockMambaBuilder.build_mamba_metadata(
|
||||
_make_vllm_config(4096, len(seq_lens), block_size=block_size),
|
||||
seq_lens=seq_lens,
|
||||
query_lens=query_lens,
|
||||
is_prefilling=is_prefilling,
|
||||
)
|
||||
num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills
|
||||
block_idx_last_p = metadata.block_idx_last_scheduled_token[
|
||||
num_reqs - num_prefills : num_reqs
|
||||
]
|
||||
expected = int(
|
||||
(block_idx_last_p - metadata.block_idx_first_scheduled_token_p)
|
||||
.clamp(min=0)
|
||||
.sum()
|
||||
)
|
||||
|
||||
assert metadata.num_state_writes_p == expected, (
|
||||
f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}"
|
||||
)
|
||||
@@ -39,6 +39,7 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.encoder_cache_manager import EncoderCacheManager
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash
|
||||
from vllm.v1.core.sched.diffusion_scheduler import (
|
||||
DiffusionAsyncScheduler,
|
||||
@@ -6751,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
num_blocks: int = 100,
|
||||
supports_divergent_hits: bool = True,
|
||||
) -> Scheduler:
|
||||
"""FA + Mamba ("all" cache mode) scheduler with a MockKVConnector."""
|
||||
"""FA + Mamba ("align" cache mode) scheduler with a MockKVConnector."""
|
||||
model_config = ModelConfig(
|
||||
model="facebook/opt-125m",
|
||||
trust_remote_code=True,
|
||||
@@ -6772,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="MockKVConnector",
|
||||
@@ -6804,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -6820,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler(
|
||||
)
|
||||
|
||||
|
||||
def _seed_hybrid_prefix(
|
||||
manager: KVCacheManager, num_blocks: int, block_size: int
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""Prefill a prefix one block per step so that align-mode Mamba caches
|
||||
the state at every block boundary. Returns the FA and Mamba block ids."""
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=num_blocks * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
assert num_computed == 0
|
||||
mamba_ids = []
|
||||
for i in range(num_blocks):
|
||||
manager.allocate_slots(
|
||||
fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None
|
||||
)
|
||||
fill.num_computed_tokens += block_size
|
||||
fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id)
|
||||
mamba_ids.append(group_mamba_ids[i])
|
||||
manager.free(fill)
|
||||
return fa_ids, mamba_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"matched_tokens,expected_num_computed",
|
||||
[
|
||||
@@ -6845,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
manager = scheduler.kv_cache_manager
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries
|
||||
# (mamba cache mode "all" caches every block's state densely).
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
fa_ids = [b.block_id for b in blocks.blocks[0]]
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
# Seed a 4-block prefix so both groups cache all four boundaries.
|
||||
fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
|
||||
|
||||
# Evict the FA tail and the middle mamba states; block 0 (both groups)
|
||||
# and the deep mamba state at block 3 survive.
|
||||
@@ -6885,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector(
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
[new_req] = output.scheduled_new_reqs
|
||||
assert new_req.num_computed_tokens == expected_num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -6921,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
assert isinstance(manager.coordinator, HybridKVCacheCoordinator)
|
||||
|
||||
# Seed a 4-block prefix in both groups.
|
||||
[fill] = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=4 * block_size,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["fill"],
|
||||
)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(fill)
|
||||
blocks = manager.allocate_slots(
|
||||
fill, fill.num_tokens, num_computed, computed_blocks
|
||||
)
|
||||
mamba_ids = [b.block_id for b in blocks.blocks[1]]
|
||||
manager.free(fill)
|
||||
_, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size)
|
||||
|
||||
# Keep all FA blocks; evict every mamba state but block 0. FA reaches 4
|
||||
# blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba).
|
||||
@@ -6955,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy(
|
||||
|
||||
scheduler.add_request(replay)
|
||||
output = scheduler.schedule()
|
||||
num_scheduled = output.num_scheduled_tokens[replay.request_id]
|
||||
assert replay.num_tokens - num_scheduled == expected_num_computed
|
||||
[new_req] = output.scheduled_new_reqs
|
||||
assert new_req.num_computed_tokens == expected_num_computed
|
||||
|
||||
|
||||
def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16):
|
||||
|
||||
@@ -60,7 +60,7 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode all \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
@@ -78,7 +78,7 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode all \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
|
||||
@@ -67,7 +67,6 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._attn_group_ids: list[int] = [0]
|
||||
self._num_ssm_scratch_blocks = 0
|
||||
self._ssm_state_slots_are_positional = False
|
||||
self._is_hma_required = False
|
||||
self.kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[None, None],
|
||||
@@ -130,7 +129,7 @@ def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0):
|
||||
block_size=16,
|
||||
shapes=shapes,
|
||||
dtypes=(torch.float32,) * num_states,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
@@ -146,7 +145,7 @@ def _gate_vllm_config(
|
||||
block_size: int = 16,
|
||||
num_lookahead_tokens: int = 0,
|
||||
num_speculative_tokens: int = 0,
|
||||
mamba_cache_mode: str = "all",
|
||||
mamba_cache_mode: str = "align",
|
||||
dcp_size: int = 1,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
@@ -193,7 +192,7 @@ def test_split_block_groups_ignores_transfer_disabled_group():
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
@@ -361,7 +360,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
block_size=4,
|
||||
num_lookahead_tokens=7,
|
||||
num_speculative_tokens=7,
|
||||
mamba_cache_mode="align",
|
||||
dcp_size=dcp_size,
|
||||
)
|
||||
scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config)
|
||||
@@ -398,21 +396,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_dspark_with_positional_mamba_cache():
|
||||
"""Positional mode counts lookahead in the Mamba allocation, so the
|
||||
one-block recurrent-state tail bound no longer holds."""
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(
|
||||
speculative_config=_spec_config("dspark"),
|
||||
block_size=4,
|
||||
mamba_cache_mode="all",
|
||||
),
|
||||
"engine",
|
||||
_dspark_wrapped_attention_config(),
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_hybrid_write():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
@@ -441,20 +424,7 @@ def test_split_block_groups_accepts_empty_abort_payload():
|
||||
assert sched.split_block_groups(()) == ([], [])
|
||||
|
||||
|
||||
def test_split_block_groups_keeps_positional_slots_in_all_mode():
|
||||
# mamba_cache_mode="all" keeps a state per block position.
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_ssm_state_slots_are_positional=True,
|
||||
)
|
||||
attn, mamba = sched.split_block_groups(([1], [40, 41, 42]))
|
||||
assert attn == [1]
|
||||
assert mamba == [[40, 41, 42]]
|
||||
|
||||
|
||||
def test_split_block_groups_keeps_only_running_state_outside_all_mode():
|
||||
def test_split_block_groups_keeps_only_running_state():
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
@@ -464,24 +434,21 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"positional,blocks,expected",
|
||||
"blocks,expected",
|
||||
[
|
||||
# Trailing scratch slots go in either cache mode.
|
||||
(True, [40, 41, 42], [40]),
|
||||
# Outside "all" the running state is picked only after they are gone.
|
||||
(False, [40, 41, 42, 43], [41]),
|
||||
# The running state is picked only after the scratch slots are gone.
|
||||
([40, 41, 42, 43], [41]),
|
||||
# Never empty: a list shorter than the scratch count keeps one state.
|
||||
(True, [40], [40]),
|
||||
(False, [], []),
|
||||
([40], [40]),
|
||||
([], []),
|
||||
],
|
||||
)
|
||||
def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected):
|
||||
def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected):
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_num_ssm_scratch_blocks=2,
|
||||
_ssm_state_slots_are_positional=positional,
|
||||
)
|
||||
|
||||
assert sched.split_block_groups(([1], blocks)) == ([1], [expected])
|
||||
@@ -555,7 +522,6 @@ def _make_read_scheduler():
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_ssm_state_slots_are_positional=True,
|
||||
request_id_to_transfer_id={},
|
||||
transfer_id_to_request_id={},
|
||||
_reqs_need_recv={},
|
||||
@@ -578,17 +544,17 @@ def _make_read_request(remote_block_ids):
|
||||
|
||||
def test_update_state_drops_decode_recompute_tail_block():
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10, 11], [90, 91]])
|
||||
request = _make_read_request([[10, 11], [91]])
|
||||
blocks = _FakeBlocks(
|
||||
all_groups=([100, 101, 102], [200, 201, 202]),
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]]
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [202]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [
|
||||
[10, 11],
|
||||
[90, 91],
|
||||
[91],
|
||||
]
|
||||
|
||||
|
||||
@@ -609,7 +575,6 @@ def test_update_state_pairs_trimmed_mamba_state_with_remote_state():
|
||||
"""The DSpark shape end to end: scratch slots are stripped, the running
|
||||
state is then selected, and the attention list loses its lookahead tail."""
|
||||
sched = _make_read_scheduler()
|
||||
sched._ssm_state_slots_are_positional = False
|
||||
sched._num_ssm_scratch_blocks = 2
|
||||
sched._max_decode_tail_blocks = 2
|
||||
request = _make_read_request([[10, 11], [90]])
|
||||
|
||||
@@ -856,28 +856,6 @@ def test_apply_prefix_caching_mamba_hybrid(
|
||||
[[6, 7, 8, 9], [99]],
|
||||
id="fa_prefix_hit_and_ssm_trim",
|
||||
),
|
||||
# Multi-slot SSM ("all" mode): a local prefix hit leaves fewer local
|
||||
# slots; the earlier remote slots are covered locally → remote tail.
|
||||
pytest.param(
|
||||
10,
|
||||
10,
|
||||
[list(range(10)), [5, 6]],
|
||||
[list(range(10)), [1, 2, 3]],
|
||||
[list(range(10)), [5, 6]],
|
||||
[list(range(10)), [2, 3]],
|
||||
id="ssm_multi_block_local_hit_tail",
|
||||
),
|
||||
# Multi-slot SSM ("all" mode): the one trailing local position holds
|
||||
# the token D recomputes itself → local head-clip.
|
||||
pytest.param(
|
||||
10,
|
||||
10,
|
||||
[list(range(10)), [4, 5, 6]],
|
||||
[list(range(10)), [8, 9]],
|
||||
[list(range(10)), [4, 5]],
|
||||
[list(range(10)), [8, 9]],
|
||||
id="ssm_multi_block_local_extra_head_clip",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_apply_prefix_caching_ssm_prefix_cache_hit(
|
||||
@@ -2034,13 +2012,12 @@ def test_logical_to_kernel_block_ids_with_remote_ratio(
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_ssm_single_state():
|
||||
"""In single-state cache modes, SSM lists are reduced to the running
|
||||
state slot: speculative scratch slots, null placeholders and the previous
|
||||
step's state carry nothing. Attention groups pass through untouched."""
|
||||
"""SSM lists are reduced to the running state slot: speculative scratch
|
||||
slots, null placeholders and the previous step's state carry nothing.
|
||||
Attention groups pass through untouched."""
|
||||
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
|
||||
sched.blocks_per_sw = [0, 0]
|
||||
sched._ssm_spec_blocks = [None, 2]
|
||||
sched._ssm_state_slots_are_positional = False
|
||||
|
||||
# Align-mode list: null placeholders, state block, 2 speculative slots.
|
||||
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 0, 7, 8, 9]))
|
||||
@@ -2069,19 +2046,6 @@ def test_exchange_clipped_blocks_ssm_single_state():
|
||||
assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_exchange_clipped_blocks_ssm_positional_states():
|
||||
"""In "all" mode every position holds a state, so only the speculative
|
||||
slots go; placeholders stay to keep the list position-indexed."""
|
||||
sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True)
|
||||
sched.blocks_per_sw = [0, 0]
|
||||
sched._ssm_spec_blocks = [None, 2]
|
||||
sched._ssm_state_slots_are_positional = True
|
||||
|
||||
clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 5, 6, 7, 8, 9]))
|
||||
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
|
||||
|
||||
|
||||
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -566,10 +566,7 @@ def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
("state-spaces/mamba-1.4b-hf", 16, 1),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"])
|
||||
def test_mamba_cpu_offload_boundary(
|
||||
model: str, block_size: int, tp_size: int, mamba_cache_mode: str
|
||||
):
|
||||
def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
@@ -586,7 +583,7 @@ def test_mamba_cpu_offload_boundary(
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
language_model_only=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
mamba_cache_mode="align",
|
||||
# Use lossless state storage so this exact-equality regression isolates
|
||||
# offload boundary selection from low-precision Mamba checkpointing.
|
||||
mamba_cache_dtype="float32",
|
||||
|
||||
@@ -474,7 +474,7 @@ def make_kv_cache_config(
|
||||
mamba_enabled: bool = False,
|
||||
sw_size: int = 128,
|
||||
num_blocks: int = 100,
|
||||
mamba_cache_mode: Literal["all", "align", "none"] = "none",
|
||||
mamba_cache_mode: Literal["align", "none"] = "none",
|
||||
) -> KVCacheConfig:
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
|
||||
@@ -2335,7 +2335,7 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None:
|
||||
# of the hash block, so it cannot exceed the smallest of them.
|
||||
hash_block_size=BLOCK_SIZE,
|
||||
dcp_world_size=dcp_world_size,
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
@@ -2747,55 +2747,6 @@ def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None:
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0]
|
||||
|
||||
|
||||
def test_eager_store_scans_mamba_all_blocks_positionally() -> None:
|
||||
"""Mamba all-mode retains positional identity and needs no handoff."""
|
||||
block_size = 4 * BLOCK_SIZE
|
||||
fix = _make_hybrid_attention_mamba_scheduler(
|
||||
num_cpu_blocks=16,
|
||||
num_gpu_blocks=24,
|
||||
block_size=block_size,
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
sched = fix.scheduler
|
||||
gpu_pool = fix.gpu_block_pool
|
||||
attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers
|
||||
assert attention_manager.has_positionally_stable_blocks
|
||||
assert mamba_manager.has_positionally_stable_blocks
|
||||
|
||||
req = _make_cp_request(num_blocks=2, virtual_block_size=block_size)
|
||||
attention_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=0
|
||||
)
|
||||
mamba_blocks = _allocate_cp_gpu_blocks(
|
||||
gpu_pool, req, 2, virtual_block_size=block_size, group_id=1
|
||||
)
|
||||
kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks))
|
||||
sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0)
|
||||
|
||||
# build_connector_meta runs before the scheduler commits this step's
|
||||
# tokens, so the positional scan picks the blocks up on the next step.
|
||||
first = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 2 * block_size},
|
||||
new_reqs={req.request_id: kv_blocks.get_block_ids()},
|
||||
)
|
||||
)
|
||||
assert first.store_gpu_blocks == []
|
||||
req.num_computed_tokens = 2 * block_size
|
||||
meta = sched.build_connector_meta(
|
||||
make_scheduler_output(
|
||||
{req.request_id: 1},
|
||||
cached_req_new_blocks={req.request_id: None},
|
||||
)
|
||||
)
|
||||
|
||||
assert set(meta.store_gpu_blocks) == {
|
||||
*(block.block_id for block in attention_blocks),
|
||||
*(block.block_id for block in mamba_blocks),
|
||||
}
|
||||
assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"])
|
||||
def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None:
|
||||
"""Handoffs for a request leaving this step must not be read.
|
||||
|
||||
@@ -189,7 +189,7 @@ def test_mixed_warmup_reserves_lookahead_blocks():
|
||||
_assert_covers_lookahead(recorder.steps, num_lookahead_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "all", "align"])
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
|
||||
"""Mamba groups hold the running-state block plus the speculative tail.
|
||||
|
||||
@@ -227,14 +227,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode):
|
||||
|
||||
|
||||
def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
"""Attention, circular, and Mamba groups exercise every reservation branch.
|
||||
|
||||
"none" and "all" reach the same branch of both `_reserved_block_count` and
|
||||
`MambaManager`, which tests only for "align". They are still both listed:
|
||||
the mode is a spec-level input, and having the real manager confirm the
|
||||
prediction for each is what keeps a future divergence between them from
|
||||
landing unnoticed.
|
||||
"""
|
||||
"""Attention, circular, and Mamba groups exercise every reservation branch."""
|
||||
return KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[],
|
||||
@@ -242,7 +235,6 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig:
|
||||
_attention_group(),
|
||||
_circular_group(),
|
||||
_mamba_group("none"),
|
||||
_mamba_group("all"),
|
||||
_mamba_group("align"),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -398,7 +398,7 @@ def _make_kv_cache_config(cfg: _TestConfig, layer_names: list[str]) -> KVCacheCo
|
||||
(cfg.temporal_state_dim,),
|
||||
),
|
||||
dtypes=(cfg.dtype, cfg.dtype),
|
||||
mamba_cache_mode="all",
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
group = KVCacheGroupSpec(
|
||||
layer_names=layer_names,
|
||||
|
||||
Reference in New Issue
Block a user