[Model Runner V2] support mamba hybrid models align prefix cache (#42406)

Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
zhrrr
2026-06-29 14:09:16 -07:00
committed by GitHub
parent a309d4fe60
commit 61ab70ec3b
9 changed files with 1046 additions and 133 deletions
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Equivalence test for ``precopy_mamba_align_fused_kernel``.
The V2 "align" pre-copy must migrate mamba state across block boundaries with
byte-identical semantics to the V1 copy specs (``get_conv_copy_spec`` /
``get_temporal_copy_spec``):
* conv state (SD layout, conv_width > 0): shift the sliding window by
``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` ->
``state[bt[dst_col], :conv_width - token_bias]``.
* temporal state (conv_width == 0): ``token_bias`` selects the accepted
speculative column -- ``state[bt[src_col + token_bias]]`` ->
``state[bt[dst_col]]``.
The kernel must also no-op when ``src_col < 0`` (fresh request) or
``src_col == dst_col`` (no boundary crossed).
"""
from __future__ import annotations
import torch
from vllm.platforms import current_platform
from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel
try:
import pytest
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="precopy_mamba_align_fused_kernel needs CUDA/Triton",
)
_parametrize = pytest.mark.parametrize
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
pytest = None
def _parametrize(_name, _values):
def _deco(fn):
return fn
return _deco
NUM_LAYERS = 3
CONV_WIDTH = 4 # conv_kernel - 1 + num_spec
CONV_DIM = 96
SSM_SHAPE = (4, 16, 16)
MAX_COLS = 8
def _build_state(num_blocks, device):
"""Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools."""
convs, ssms = [], []
for _ in range(NUM_LAYERS):
convs.append(
torch.randn(
num_blocks, CONV_WIDTH, CONV_DIM, dtype=torch.bfloat16, device=device
)
)
ssms.append(
torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device)
)
return convs, ssms
def _build_meta(convs, ssms, device):
"""Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer."""
n = NUM_LAYERS * 2
base = torch.zeros(n, dtype=torch.int64, device=device)
blk_stride = torch.zeros(n, dtype=torch.int64, device=device)
elem = torch.zeros(n, dtype=torch.int32, device=device)
inner = torch.zeros(n, dtype=torch.int64, device=device)
width = torch.zeros(n, dtype=torch.int32, device=device)
group = torch.zeros(n, dtype=torch.int32, device=device)
drc = torch.zeros(n, dtype=torch.int32, device=device) # DS rows (unused, SD)
drs = torch.zeros(n, dtype=torch.int64, device=device)
i = 0
for layer in range(NUM_LAYERS):
conv, ssm = convs[layer], ssms[layer]
# conv (SD): width = size(1), inner = stride(1)
base[i] = conv.data_ptr()
blk_stride[i] = conv.stride(0) * conv.element_size()
elem[i] = conv.element_size()
width[i] = conv.size(1)
inner[i] = conv.stride(1)
i += 1
# ssm (temporal): width = 0, inner = elems per block
base[i] = ssm.data_ptr()
blk_stride[i] = ssm.stride(0) * ssm.element_size()
elem[i] = ssm.element_size()
width[i] = 0
inner[i] = ssm[0].numel()
i += 1
return base, blk_stride, elem, inner, width, group, drc, drs
def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs):
"""Apply the V1 copy semantics on clones, reading from the pre-copy state."""
conv_pre = [c.clone() for c in convs]
ssm_pre = [s.clone() for s in ssms]
conv_ref = [c.clone() for c in convs]
ssm_ref = [s.clone() for s in ssms]
for r in range(num_reqs):
sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r])
if sc < 0 or sc == dc:
continue
sblk, dblk = int(bt[r, sc]), int(bt[r, dc])
tblk = int(bt[r, sc + tb]) # temporal src column shifted by bias
for layer in range(NUM_LAYERS):
conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:]
ssm_ref[layer][dblk] = ssm_pre[layer][tblk]
return conv_ref, ssm_ref
@_parametrize("num_reqs", [1, 4, 16])
@_parametrize("token_bias", [0, 1, 2])
def test_precopy_matches_v1_copy_specs(num_reqs, token_bias):
device = torch.device("cuda")
torch.manual_seed(0)
# Distinct physical block per (req, col) so copies never alias.
num_blocks = num_reqs * MAX_COLS + 1
bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device)
for r in range(num_reqs):
bt[r] = torch.arange(
1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device
)
# Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip),
# the rest cross from col 1 -> col 0 with the given spec token bias.
src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device)
dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device)
bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device)
if num_reqs >= 1:
src_col[0] = -1 # fresh -> no copy
if num_reqs >= 2:
dst_col[1] = 1 # src_col == dst_col -> no copy
convs, ssms = _build_state(num_blocks, device)
conv_ref, ssm_ref = _reference(
convs, ssms, bt.cpu(), src_col.cpu(), dst_col.cpu(), bias.cpu(), num_reqs
)
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
convs, ssms, device
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
grid = (num_reqs, NUM_LAYERS * 2)
precopy_mamba_align_fused_kernel[grid](
dst_col,
src_col,
bias,
bt_ptrs,
bt.stride(0),
base,
blk_stride,
elem,
inner,
width,
group,
drc,
drs,
idx_mapping,
num_reqs,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=False,
)
torch.accelerator.synchronize()
for layer in range(NUM_LAYERS):
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
if __name__ == "__main__":
for nr in (1, 4, 16):
for tb in (0, 1, 2):
test_precopy_matches_v1_copy_specs(nr, tb)
print(f"OK num_reqs={nr} token_bias={tb}")
+280 -16
View File
@@ -11,6 +11,7 @@ import datasets
import pytest
import torch
import vllm.envs as envs
from tests.utils import create_new_process_for_each_test
from vllm import LLM, SamplingParams, TokensPrompt
from vllm.config import CacheConfig
@@ -494,12 +495,7 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn)
@create_new_process_for_each_test()
def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
run_ref_mamba_state_in_subprocess()
apply_patch(monkeypatch)
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
full_prompt = prompt_dataset["train"][0]["text"]
def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
tests = {
"accept_1": TestConfig(
num_prompt_tokens=554,
@@ -731,6 +727,27 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
),
}
return tests
def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None:
for step_action_prev, step_action_next in zip(
test_config.step_actions[:-1], test_config.step_actions[1:]
):
if len(step_action_next.kv_cache_block_ids) == 0:
step_action_next.kv_cache_block_ids = (
step_action_prev.kv_cache_block_ids.copy()
)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
run_ref_mamba_state_in_subprocess()
apply_patch(monkeypatch)
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
full_prompt = prompt_dataset["train"][0]["text"]
tests = get_mamba_prefix_cache_step_configs()
engine = LLM(
model=MODEL,
enable_prefix_caching=True,
@@ -758,16 +775,7 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
)
global cur_step_action_idx
cur_step_action_idx = 0
for step_action_prev, step_action_next in zip(
test_config.step_actions[:-1], test_config.step_actions[1:]
):
if (
step_action_next.kv_cache_block_ids is not None
and len(step_action_next.kv_cache_block_ids) == 0
):
prev_block_ids = step_action_prev.kv_cache_block_ids
if prev_block_ids is not None:
step_action_next.kv_cache_block_ids = prev_block_ids.copy()
fill_following_kv_cache_block_ids(test_config)
global step_actions
step_actions = test_config.step_actions
_ = engine.generate(
@@ -787,3 +795,259 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
del engine
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
envs.disable_envs_cache()
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as MRV2GPUModelRunner
from vllm.v1.worker.gpu.model_states.mamba_hybrid import (
MambaHybridModelState,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput as MRV2SamplerOutput
events: list[int] = []
original_execute_model = MRV2GPUModelRunner.execute_model
original_sample = MRV2GPUModelRunner.sample
original_preprocess_state = MambaHybridModelState.preprocess_state
original_postprocess_state = MambaHybridModelState.postprocess_state
original_step_action_fn = InprocClient.get_output
original_allocate_slots = KVCacheManager.allocate_slots
captured: dict[str, Any] = {}
def temporal_states(model_state, block_tables, kv_cache_config):
# Qwen3-Next keeps the temporal (ssm) state as the last Mamba cache.
forward_context = (
model_state.vllm_config.compilation_config.static_forward_context
)
group_ids, _ = get_mamba_groups(kv_cache_config)
for group_id in group_ids:
block_table = block_tables[group_id]
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
yield forward_context[layer_name].kv_cache[-1], block_table
def temporal_block(temporal_state, block_table, col):
return temporal_state[int(block_table[0, col].item())]
def wrapped_preprocess_state(
self: MambaHybridModelState,
input_batch: Any,
block_tables: tuple[torch.Tensor, ...],
kv_cache_config: KVCacheConfig,
num_computed_tokens: torch.Tensor,
) -> None:
captured["block_tables"] = block_tables
captured["kv_cache_config"] = kv_cache_config
expected = (
None if cur_step_action is None else cur_step_action.preprocess_copy_idx
)
snapshots = []
if expected is not None and expected != (-1, -1):
for temporal, bt in temporal_states(self, block_tables, kv_cache_config):
snapshots.append(
(temporal, bt, temporal_block(temporal, bt, expected[0]).clone())
)
ret = original_preprocess_state(
self, input_batch, block_tables, kv_cache_config, num_computed_tokens
)
if cur_step_action is not None:
req_idx = int(input_batch.idx_mapping[0].item())
src_col = int(self._mamba_src_col_gpu[req_idx].item())
off = int(self._mamba_src_off_gpu[req_idx].item())
dst = int(self._mamba_state_idx_gpu[req_idx].item())
actual = (-1, -1) if src_col < 0 or src_col == dst else (src_col + off, dst)
assert actual == expected, (
f"V2 align preprocess copy: expected={expected}, "
f"actual={actual}, {cur_step_action=}"
)
for temporal, bt, src_state in snapshots:
torch.testing.assert_close(
temporal_block(temporal, bt, expected[1]), src_state
)
return ret
def wrapped_postprocess_state(
self: MambaHybridModelState,
idx_mapping: torch.Tensor,
num_sampled: torch.Tensor | int,
num_computed_tokens: torch.Tensor | None = None,
) -> None:
action = cur_step_action
block_tables = captured.get("block_tables")
kv_cache_config = captured.get("kv_cache_config")
# The postprocess kernel does not expose its indices, so only the copy
# case is checked, by effect: snapshot the src block, expect dst == src.
if (
action is None
or num_computed_tokens is None
or block_tables is None
or action.postprocess_copy_idx == (-1, -1)
):
return original_postprocess_state(
self, idx_mapping, num_sampled, num_computed_tokens
)
expected = action.postprocess_copy_idx
snapshots = [
(temporal, bt, temporal_block(temporal, bt, expected[0]).clone())
for temporal, bt in temporal_states(self, block_tables, kv_cache_config)
]
ret = original_postprocess_state(
self, idx_mapping, num_sampled, num_computed_tokens
)
for temporal, bt, src_state in snapshots:
torch.testing.assert_close(
temporal_block(temporal, bt, expected[1]), src_state
)
return ret
def wrapped_execute_model(
self: MRV2GPUModelRunner,
scheduler_output: SchedulerOutput,
*args: Any,
**kwargs: Any,
):
events.extend(
req.num_computed_tokens for req in scheduler_output.scheduled_new_reqs
)
events.extend(scheduler_output.scheduled_cached_reqs.num_computed_tokens)
if cur_step_action is not None:
num_scheduled_tokens = next(
iter(scheduler_output.num_scheduled_tokens.values())
)
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
ret = original_execute_model(self, scheduler_output, *args, **kwargs)
if cur_step_action is not None and self.execute_model_state is not None:
input_batch = self.execute_model_state.input_batch
assert (
cur_step_action.num_computed_tokens_start
== input_batch.positions[input_batch.query_start_loc[0]].item()
)
return ret
def fake_sample(
self: MRV2GPUModelRunner,
hidden_states: torch.Tensor,
input_batch: Any,
grammar_output: Any,
):
if cur_step_action is None:
return original_sample(self, hidden_states, input_batch, grammar_output)
num_reqs = input_batch.num_reqs
sampled_token_ids = torch.ones(
(num_reqs, self.num_speculative_steps + 1),
device=hidden_states.device,
dtype=torch.int64,
)
num_logits = torch.tensor(
input_batch.cu_num_logits_np[1 : num_reqs + 1]
- input_batch.cu_num_logits_np[:num_reqs],
device=hidden_states.device,
dtype=torch.int32,
)
accepted = torch.full_like(num_logits, num_accepted_tokens)
num_sampled = torch.minimum(accepted, num_logits)
prefill_lens = self.req_states.prefill_len.gpu[input_batch.idx_mapping]
is_chunked_prefill = input_batch.seq_lens[:num_reqs] < prefill_lens
num_sampled = torch.where(is_chunked_prefill, 0, num_sampled)
num_rejected = torch.where(is_chunked_prefill, 0, num_logits - num_sampled)
sampler_output = MRV2SamplerOutput(
sampled_token_ids=sampled_token_ids,
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled,
)
return sampler_output, num_sampled, num_rejected
monkeypatch.setattr(
InprocClient,
"get_output",
get_fake_step_action_fn(original_step_action_fn),
)
monkeypatch.setattr(
KVCacheManager,
"allocate_slots",
get_fake_allocate_slots_fn(original_allocate_slots),
)
monkeypatch.setattr(MRV2GPUModelRunner, "execute_model", wrapped_execute_model)
monkeypatch.setattr(MRV2GPUModelRunner, "sample", fake_sample)
monkeypatch.setattr(
MambaHybridModelState, "preprocess_state", wrapped_preprocess_state
)
monkeypatch.setattr(
MambaHybridModelState, "postprocess_state", wrapped_postprocess_state
)
engine = LLM(
model=MODEL,
load_format="dummy",
enforce_eager=True,
skip_tokenizer_init=True,
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
},
max_num_batched_tokens=3072,
max_model_len=BLOCK_SIZE * 12,
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
seed=42,
)
try:
tests = get_mamba_prefix_cache_step_configs()
global step_actions
global cur_step_action_idx
global num_accepted_tokens
for test_name, test_config in tests.items():
num_accepted_tokens = test_config.num_accepted_tokens
cur_step_action_idx = 0
fill_following_kv_cache_block_ids(test_config)
step_actions = test_config.step_actions
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=test_config.num_generated_tokens,
ignore_eos=True,
)
_ = engine.generate(
[TokensPrompt(prompt_token_ids=[1] * test_config.num_prompt_tokens)],
sampling_params=sampling_params,
)
assert cur_step_action_idx == len(test_config.step_actions), test_name
assert (
engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
)
step_actions = []
cur_step_action_idx = 0
num_accepted_tokens = 1
prompt = TokensPrompt(prompt_token_ids=[1] * (BLOCK_SIZE * 2))
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
ignore_eos=True,
)
_ = engine.generate([prompt], sampling_params=sampling_params)
first_event_count = len(events)
_ = engine.generate([prompt], sampling_params=sampling_params)
second_events = events[first_event_count:]
prefix_hits = [
num_computed_tokens
for num_computed_tokens in second_events
if num_computed_tokens >= BLOCK_SIZE
]
assert prefix_hits, (
"Expected the second identical prompt to hit prefix cache, "
f"got events={second_events!r}"
)
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
finally:
del engine
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()