[Model Runner V2] support mamba hybrid models align prefix cache (#42406)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Equivalence test for ``precopy_mamba_align_fused_kernel``.
|
||||
|
||||
The V2 "align" pre-copy must migrate mamba state across block boundaries with
|
||||
byte-identical semantics to the V1 copy specs (``get_conv_copy_spec`` /
|
||||
``get_temporal_copy_spec``):
|
||||
|
||||
* conv state (SD layout, conv_width > 0): shift the sliding window by
|
||||
``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` ->
|
||||
``state[bt[dst_col], :conv_width - token_bias]``.
|
||||
* temporal state (conv_width == 0): ``token_bias`` selects the accepted
|
||||
speculative column -- ``state[bt[src_col + token_bias]]`` ->
|
||||
``state[bt[dst_col]]``.
|
||||
|
||||
The kernel must also no-op when ``src_col < 0`` (fresh request) or
|
||||
``src_col == dst_col`` (no boundary crossed).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel
|
||||
|
||||
try:
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="precopy_mamba_align_fused_kernel needs CUDA/Triton",
|
||||
)
|
||||
_parametrize = pytest.mark.parametrize
|
||||
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
|
||||
pytest = None
|
||||
|
||||
def _parametrize(_name, _values):
|
||||
def _deco(fn):
|
||||
return fn
|
||||
|
||||
return _deco
|
||||
|
||||
|
||||
NUM_LAYERS = 3
|
||||
CONV_WIDTH = 4 # conv_kernel - 1 + num_spec
|
||||
CONV_DIM = 96
|
||||
SSM_SHAPE = (4, 16, 16)
|
||||
MAX_COLS = 8
|
||||
|
||||
|
||||
def _build_state(num_blocks, device):
|
||||
"""Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools."""
|
||||
convs, ssms = [], []
|
||||
for _ in range(NUM_LAYERS):
|
||||
convs.append(
|
||||
torch.randn(
|
||||
num_blocks, CONV_WIDTH, CONV_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
)
|
||||
ssms.append(
|
||||
torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device)
|
||||
)
|
||||
return convs, ssms
|
||||
|
||||
|
||||
def _build_meta(convs, ssms, device):
|
||||
"""Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer."""
|
||||
n = NUM_LAYERS * 2
|
||||
base = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
blk_stride = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
elem = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
inner = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
width = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
group = torch.zeros(n, dtype=torch.int32, device=device)
|
||||
drc = torch.zeros(n, dtype=torch.int32, device=device) # DS rows (unused, SD)
|
||||
drs = torch.zeros(n, dtype=torch.int64, device=device)
|
||||
i = 0
|
||||
for layer in range(NUM_LAYERS):
|
||||
conv, ssm = convs[layer], ssms[layer]
|
||||
# conv (SD): width = size(1), inner = stride(1)
|
||||
base[i] = conv.data_ptr()
|
||||
blk_stride[i] = conv.stride(0) * conv.element_size()
|
||||
elem[i] = conv.element_size()
|
||||
width[i] = conv.size(1)
|
||||
inner[i] = conv.stride(1)
|
||||
i += 1
|
||||
# ssm (temporal): width = 0, inner = elems per block
|
||||
base[i] = ssm.data_ptr()
|
||||
blk_stride[i] = ssm.stride(0) * ssm.element_size()
|
||||
elem[i] = ssm.element_size()
|
||||
width[i] = 0
|
||||
inner[i] = ssm[0].numel()
|
||||
i += 1
|
||||
return base, blk_stride, elem, inner, width, group, drc, drs
|
||||
|
||||
|
||||
def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs):
|
||||
"""Apply the V1 copy semantics on clones, reading from the pre-copy state."""
|
||||
conv_pre = [c.clone() for c in convs]
|
||||
ssm_pre = [s.clone() for s in ssms]
|
||||
conv_ref = [c.clone() for c in convs]
|
||||
ssm_ref = [s.clone() for s in ssms]
|
||||
for r in range(num_reqs):
|
||||
sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r])
|
||||
if sc < 0 or sc == dc:
|
||||
continue
|
||||
sblk, dblk = int(bt[r, sc]), int(bt[r, dc])
|
||||
tblk = int(bt[r, sc + tb]) # temporal src column shifted by bias
|
||||
for layer in range(NUM_LAYERS):
|
||||
conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:]
|
||||
ssm_ref[layer][dblk] = ssm_pre[layer][tblk]
|
||||
return conv_ref, ssm_ref
|
||||
|
||||
|
||||
@_parametrize("num_reqs", [1, 4, 16])
|
||||
@_parametrize("token_bias", [0, 1, 2])
|
||||
def test_precopy_matches_v1_copy_specs(num_reqs, token_bias):
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
# Distinct physical block per (req, col) so copies never alias.
|
||||
num_blocks = num_reqs * MAX_COLS + 1
|
||||
bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device)
|
||||
for r in range(num_reqs):
|
||||
bt[r] = torch.arange(
|
||||
1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip),
|
||||
# the rest cross from col 1 -> col 0 with the given spec token bias.
|
||||
src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device)
|
||||
dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device)
|
||||
if num_reqs >= 1:
|
||||
src_col[0] = -1 # fresh -> no copy
|
||||
if num_reqs >= 2:
|
||||
dst_col[1] = 1 # src_col == dst_col -> no copy
|
||||
|
||||
convs, ssms = _build_state(num_blocks, device)
|
||||
conv_ref, ssm_ref = _reference(
|
||||
convs, ssms, bt.cpu(), src_col.cpu(), dst_col.cpu(), bias.cpu(), num_reqs
|
||||
)
|
||||
|
||||
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
|
||||
convs, ssms, device
|
||||
)
|
||||
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
grid = (num_reqs, NUM_LAYERS * 2)
|
||||
precopy_mamba_align_fused_kernel[grid](
|
||||
dst_col,
|
||||
src_col,
|
||||
bias,
|
||||
bt_ptrs,
|
||||
bt.stride(0),
|
||||
base,
|
||||
blk_stride,
|
||||
elem,
|
||||
inner,
|
||||
width,
|
||||
group,
|
||||
drc,
|
||||
drs,
|
||||
idx_mapping,
|
||||
num_reqs,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=False,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for layer in range(NUM_LAYERS):
|
||||
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
|
||||
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for nr in (1, 4, 16):
|
||||
for tb in (0, 1, 2):
|
||||
test_precopy_matches_v1_copy_specs(nr, tb)
|
||||
print(f"OK num_reqs={nr} token_bias={tb}")
|
||||
@@ -11,6 +11,7 @@ import datasets
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import LLM, SamplingParams, TokensPrompt
|
||||
from vllm.config import CacheConfig
|
||||
@@ -494,12 +495,7 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
|
||||
run_ref_mamba_state_in_subprocess()
|
||||
apply_patch(monkeypatch)
|
||||
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
|
||||
full_prompt = prompt_dataset["train"][0]["text"]
|
||||
def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
|
||||
tests = {
|
||||
"accept_1": TestConfig(
|
||||
num_prompt_tokens=554,
|
||||
@@ -731,6 +727,27 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
|
||||
),
|
||||
}
|
||||
|
||||
return tests
|
||||
|
||||
|
||||
def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None:
|
||||
for step_action_prev, step_action_next in zip(
|
||||
test_config.step_actions[:-1], test_config.step_actions[1:]
|
||||
):
|
||||
if len(step_action_next.kv_cache_block_ids) == 0:
|
||||
step_action_next.kv_cache_block_ids = (
|
||||
step_action_prev.kv_cache_block_ids.copy()
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
||||
run_ref_mamba_state_in_subprocess()
|
||||
apply_patch(monkeypatch)
|
||||
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
|
||||
full_prompt = prompt_dataset["train"][0]["text"]
|
||||
tests = get_mamba_prefix_cache_step_configs()
|
||||
|
||||
engine = LLM(
|
||||
model=MODEL,
|
||||
enable_prefix_caching=True,
|
||||
@@ -758,16 +775,7 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
global cur_step_action_idx
|
||||
cur_step_action_idx = 0
|
||||
for step_action_prev, step_action_next in zip(
|
||||
test_config.step_actions[:-1], test_config.step_actions[1:]
|
||||
):
|
||||
if (
|
||||
step_action_next.kv_cache_block_ids is not None
|
||||
and len(step_action_next.kv_cache_block_ids) == 0
|
||||
):
|
||||
prev_block_ids = step_action_prev.kv_cache_block_ids
|
||||
if prev_block_ids is not None:
|
||||
step_action_next.kv_cache_block_ids = prev_block_ids.copy()
|
||||
fill_following_kv_cache_block_ids(test_config)
|
||||
global step_actions
|
||||
step_actions = test_config.step_actions
|
||||
_ = engine.generate(
|
||||
@@ -787,3 +795,259 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
envs.disable_envs_cache()
|
||||
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as MRV2GPUModelRunner
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import (
|
||||
MambaHybridModelState,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput as MRV2SamplerOutput
|
||||
|
||||
events: list[int] = []
|
||||
original_execute_model = MRV2GPUModelRunner.execute_model
|
||||
original_sample = MRV2GPUModelRunner.sample
|
||||
original_preprocess_state = MambaHybridModelState.preprocess_state
|
||||
original_postprocess_state = MambaHybridModelState.postprocess_state
|
||||
original_step_action_fn = InprocClient.get_output
|
||||
original_allocate_slots = KVCacheManager.allocate_slots
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def temporal_states(model_state, block_tables, kv_cache_config):
|
||||
# Qwen3-Next keeps the temporal (ssm) state as the last Mamba cache.
|
||||
forward_context = (
|
||||
model_state.vllm_config.compilation_config.static_forward_context
|
||||
)
|
||||
group_ids, _ = get_mamba_groups(kv_cache_config)
|
||||
for group_id in group_ids:
|
||||
block_table = block_tables[group_id]
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
|
||||
yield forward_context[layer_name].kv_cache[-1], block_table
|
||||
|
||||
def temporal_block(temporal_state, block_table, col):
|
||||
return temporal_state[int(block_table[0, col].item())]
|
||||
|
||||
def wrapped_preprocess_state(
|
||||
self: MambaHybridModelState,
|
||||
input_batch: Any,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
num_computed_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
captured["block_tables"] = block_tables
|
||||
captured["kv_cache_config"] = kv_cache_config
|
||||
expected = (
|
||||
None if cur_step_action is None else cur_step_action.preprocess_copy_idx
|
||||
)
|
||||
snapshots = []
|
||||
if expected is not None and expected != (-1, -1):
|
||||
for temporal, bt in temporal_states(self, block_tables, kv_cache_config):
|
||||
snapshots.append(
|
||||
(temporal, bt, temporal_block(temporal, bt, expected[0]).clone())
|
||||
)
|
||||
ret = original_preprocess_state(
|
||||
self, input_batch, block_tables, kv_cache_config, num_computed_tokens
|
||||
)
|
||||
if cur_step_action is not None:
|
||||
req_idx = int(input_batch.idx_mapping[0].item())
|
||||
src_col = int(self._mamba_src_col_gpu[req_idx].item())
|
||||
off = int(self._mamba_src_off_gpu[req_idx].item())
|
||||
dst = int(self._mamba_state_idx_gpu[req_idx].item())
|
||||
actual = (-1, -1) if src_col < 0 or src_col == dst else (src_col + off, dst)
|
||||
assert actual == expected, (
|
||||
f"V2 align preprocess copy: expected={expected}, "
|
||||
f"actual={actual}, {cur_step_action=}"
|
||||
)
|
||||
for temporal, bt, src_state in snapshots:
|
||||
torch.testing.assert_close(
|
||||
temporal_block(temporal, bt, expected[1]), src_state
|
||||
)
|
||||
return ret
|
||||
|
||||
def wrapped_postprocess_state(
|
||||
self: MambaHybridModelState,
|
||||
idx_mapping: torch.Tensor,
|
||||
num_sampled: torch.Tensor | int,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
action = cur_step_action
|
||||
block_tables = captured.get("block_tables")
|
||||
kv_cache_config = captured.get("kv_cache_config")
|
||||
# The postprocess kernel does not expose its indices, so only the copy
|
||||
# case is checked, by effect: snapshot the src block, expect dst == src.
|
||||
if (
|
||||
action is None
|
||||
or num_computed_tokens is None
|
||||
or block_tables is None
|
||||
or action.postprocess_copy_idx == (-1, -1)
|
||||
):
|
||||
return original_postprocess_state(
|
||||
self, idx_mapping, num_sampled, num_computed_tokens
|
||||
)
|
||||
expected = action.postprocess_copy_idx
|
||||
snapshots = [
|
||||
(temporal, bt, temporal_block(temporal, bt, expected[0]).clone())
|
||||
for temporal, bt in temporal_states(self, block_tables, kv_cache_config)
|
||||
]
|
||||
ret = original_postprocess_state(
|
||||
self, idx_mapping, num_sampled, num_computed_tokens
|
||||
)
|
||||
for temporal, bt, src_state in snapshots:
|
||||
torch.testing.assert_close(
|
||||
temporal_block(temporal, bt, expected[1]), src_state
|
||||
)
|
||||
return ret
|
||||
|
||||
def wrapped_execute_model(
|
||||
self: MRV2GPUModelRunner,
|
||||
scheduler_output: SchedulerOutput,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
):
|
||||
events.extend(
|
||||
req.num_computed_tokens for req in scheduler_output.scheduled_new_reqs
|
||||
)
|
||||
events.extend(scheduler_output.scheduled_cached_reqs.num_computed_tokens)
|
||||
if cur_step_action is not None:
|
||||
num_scheduled_tokens = next(
|
||||
iter(scheduler_output.num_scheduled_tokens.values())
|
||||
)
|
||||
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
|
||||
ret = original_execute_model(self, scheduler_output, *args, **kwargs)
|
||||
if cur_step_action is not None and self.execute_model_state is not None:
|
||||
input_batch = self.execute_model_state.input_batch
|
||||
assert (
|
||||
cur_step_action.num_computed_tokens_start
|
||||
== input_batch.positions[input_batch.query_start_loc[0]].item()
|
||||
)
|
||||
return ret
|
||||
|
||||
def fake_sample(
|
||||
self: MRV2GPUModelRunner,
|
||||
hidden_states: torch.Tensor,
|
||||
input_batch: Any,
|
||||
grammar_output: Any,
|
||||
):
|
||||
if cur_step_action is None:
|
||||
return original_sample(self, hidden_states, input_batch, grammar_output)
|
||||
|
||||
num_reqs = input_batch.num_reqs
|
||||
sampled_token_ids = torch.ones(
|
||||
(num_reqs, self.num_speculative_steps + 1),
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
num_logits = torch.tensor(
|
||||
input_batch.cu_num_logits_np[1 : num_reqs + 1]
|
||||
- input_batch.cu_num_logits_np[:num_reqs],
|
||||
device=hidden_states.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
accepted = torch.full_like(num_logits, num_accepted_tokens)
|
||||
num_sampled = torch.minimum(accepted, num_logits)
|
||||
prefill_lens = self.req_states.prefill_len.gpu[input_batch.idx_mapping]
|
||||
is_chunked_prefill = input_batch.seq_lens[:num_reqs] < prefill_lens
|
||||
num_sampled = torch.where(is_chunked_prefill, 0, num_sampled)
|
||||
num_rejected = torch.where(is_chunked_prefill, 0, num_logits - num_sampled)
|
||||
sampler_output = MRV2SamplerOutput(
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
logprobs_tensors=None,
|
||||
num_nans=None,
|
||||
num_sampled=num_sampled,
|
||||
)
|
||||
return sampler_output, num_sampled, num_rejected
|
||||
|
||||
monkeypatch.setattr(
|
||||
InprocClient,
|
||||
"get_output",
|
||||
get_fake_step_action_fn(original_step_action_fn),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
KVCacheManager,
|
||||
"allocate_slots",
|
||||
get_fake_allocate_slots_fn(original_allocate_slots),
|
||||
)
|
||||
monkeypatch.setattr(MRV2GPUModelRunner, "execute_model", wrapped_execute_model)
|
||||
monkeypatch.setattr(MRV2GPUModelRunner, "sample", fake_sample)
|
||||
monkeypatch.setattr(
|
||||
MambaHybridModelState, "preprocess_state", wrapped_preprocess_state
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MambaHybridModelState, "postprocess_state", wrapped_postprocess_state
|
||||
)
|
||||
|
||||
engine = LLM(
|
||||
model=MODEL,
|
||||
load_format="dummy",
|
||||
enforce_eager=True,
|
||||
skip_tokenizer_init=True,
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
},
|
||||
max_num_batched_tokens=3072,
|
||||
max_model_len=BLOCK_SIZE * 12,
|
||||
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||
seed=42,
|
||||
)
|
||||
|
||||
try:
|
||||
tests = get_mamba_prefix_cache_step_configs()
|
||||
|
||||
global step_actions
|
||||
global cur_step_action_idx
|
||||
global num_accepted_tokens
|
||||
for test_name, test_config in tests.items():
|
||||
num_accepted_tokens = test_config.num_accepted_tokens
|
||||
cur_step_action_idx = 0
|
||||
fill_following_kv_cache_block_ids(test_config)
|
||||
step_actions = test_config.step_actions
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=test_config.num_generated_tokens,
|
||||
ignore_eos=True,
|
||||
)
|
||||
_ = engine.generate(
|
||||
[TokensPrompt(prompt_token_ids=[1] * test_config.num_prompt_tokens)],
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
assert cur_step_action_idx == len(test_config.step_actions), test_name
|
||||
assert (
|
||||
engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
|
||||
)
|
||||
|
||||
step_actions = []
|
||||
cur_step_action_idx = 0
|
||||
num_accepted_tokens = 1
|
||||
prompt = TokensPrompt(prompt_token_ids=[1] * (BLOCK_SIZE * 2))
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=1,
|
||||
ignore_eos=True,
|
||||
)
|
||||
_ = engine.generate([prompt], sampling_params=sampling_params)
|
||||
first_event_count = len(events)
|
||||
_ = engine.generate([prompt], sampling_params=sampling_params)
|
||||
second_events = events[first_event_count:]
|
||||
prefix_hits = [
|
||||
num_computed_tokens
|
||||
for num_computed_tokens in second_events
|
||||
if num_computed_tokens >= BLOCK_SIZE
|
||||
]
|
||||
assert prefix_hits, (
|
||||
"Expected the second identical prompt to hit prefix cache, "
|
||||
f"got events={second_events!r}"
|
||||
)
|
||||
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
|
||||
finally:
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
@@ -1997,13 +1997,6 @@ class VllmConfig:
|
||||
model_config = self.model_config
|
||||
speculative_config = self.speculative_config
|
||||
|
||||
if (
|
||||
model_config is not None
|
||||
and model_config.has_inner_state
|
||||
and self.cache_config.mamba_cache_mode == "align"
|
||||
):
|
||||
unsupported.append("hybrid/mamba models with align cache mode")
|
||||
|
||||
if self.parallel_config.prefill_context_parallel_size > 1:
|
||||
unsupported.append("prefill context parallelism")
|
||||
|
||||
@@ -2152,10 +2145,6 @@ class VllmConfig:
|
||||
"to schedule a multiple of block_size tokens even if they are "
|
||||
"in the middle of a mm input"
|
||||
)
|
||||
# TODO: support align mamba cache mode for model runner v2
|
||||
assert not envs.VLLM_USE_V2_MODEL_RUNNER, (
|
||||
"Model Runner V2 has not yet supported mamba_cache_mode='align'. "
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
|
||||
|
||||
@@ -973,7 +973,9 @@ class DiffusionGemmaModelState(ModelState):
|
||||
# so the captured graph and runtime point to identical addresses.
|
||||
return {"inputs_embeds": self._inputs_embeds_buf[:num_tokens]}
|
||||
|
||||
def postprocess_state(self, idx_mapping, num_sampled) -> None:
|
||||
def postprocess_state(
|
||||
self, idx_mapping, num_sampled, num_computed_tokens=None
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def prepare_attn(
|
||||
|
||||
@@ -1111,7 +1111,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.total_len.gpu,
|
||||
)
|
||||
|
||||
self.model_state.postprocess_state(idx_mapping, num_sampled)
|
||||
self.model_state.postprocess_state(
|
||||
idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -1176,6 +1178,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Prepare all the inputs and copy to the input buffers.
|
||||
input_batch = self.prepare_inputs(scheduler_output, batch_desc)
|
||||
block_tables, slot_mappings = self.prepare_attn(input_batch)
|
||||
# Mamba "align" pre-copy: migrate recurrent state across block
|
||||
# boundaries before the forward. Runs only on real batches, and
|
||||
# before model_state.prepare_attn gathers num_accepted_tokens so the
|
||||
# boundary reset is visible to the attention metadata.
|
||||
self.model_state.preprocess_state(
|
||||
input_batch,
|
||||
block_tables,
|
||||
self.kv_cache_config,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
)
|
||||
|
||||
if self.lora_config:
|
||||
# Activate LoRA adapters.
|
||||
|
||||
@@ -95,8 +95,23 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def preprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
num_computed_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
"""Hook run on real batches before the forward pass (after block tables
|
||||
are gathered). Used by mamba "align" prefix caching to pre-copy state
|
||||
across block boundaries. No-op by default."""
|
||||
return None
|
||||
|
||||
def postprocess_state(
|
||||
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@@ -9,15 +9,25 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
get_conv_copy_spec,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
|
||||
from vllm.v1.worker.mamba_utils import (
|
||||
MambaSpecDecodeGPUContext,
|
||||
preprocess_mamba_align_fused_kernel,
|
||||
)
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
@@ -65,9 +75,142 @@ class MambaHybridModelState(DefaultModelState):
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.num_accepted_tokens_gpu = torch.ones(
|
||||
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
# Pre-copy "align" prefix-cache state (V2). The migration of each
|
||||
# request's mamba state across block boundaries runs as a fused GPU
|
||||
# kernel reusing the postprocess copy machinery, so the per-step src
|
||||
# columns and the running state_idx are kept GPU-resident.
|
||||
self._align_mode = self.cache_config.mamba_cache_mode == "align"
|
||||
if self._align_mode:
|
||||
self._mamba_state_idx_gpu = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self._mamba_src_col_gpu = torch.full(
|
||||
(self.max_num_reqs,), -1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self._mamba_src_off_gpu = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self._mamba_ctx: MambaSpecDecodeGPUContext | None = None
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._mamba_spec: MambaSpec | None = None
|
||||
|
||||
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
|
||||
super().add_request(req_index, new_req_data)
|
||||
if self._align_mode:
|
||||
# Seed the running state block from the resumed/prefilled position.
|
||||
self._mamba_state_idx_gpu[req_index] = (
|
||||
new_req_data.num_computed_tokens - 1
|
||||
) // self.cache_config.block_size
|
||||
self.num_accepted_tokens_gpu[req_index] = 1
|
||||
|
||||
def _get_mamba_group_info(
|
||||
self, kv_cache_config: KVCacheConfig
|
||||
) -> tuple[list[int], MambaSpec]:
|
||||
if self._mamba_spec is None:
|
||||
group_ids: list[int] = []
|
||||
specs: list[MambaSpec] = []
|
||||
for i, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, MambaSpec):
|
||||
group_ids.append(i)
|
||||
specs.append(spec)
|
||||
assert specs, "no mamba layers in the model"
|
||||
assert all(specs[0] == s for s in specs)
|
||||
self._mamba_group_ids = group_ids
|
||||
self._mamba_spec = specs[0]
|
||||
return self._mamba_group_ids, self._mamba_spec
|
||||
|
||||
def _ensure_align_ctx(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
mamba_group_ids: list[int],
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
) -> MambaSpecDecodeGPUContext:
|
||||
if self._mamba_ctx is None:
|
||||
copy_funcs = self.model.get_mamba_state_copy_func()
|
||||
# The fused copy kernels shift conv windows assuming the SD layout;
|
||||
# the DS layout cannot express a >0 spec-decode shift as a single
|
||||
# contiguous copy (mirrors get_conv_copy_spec's NotImplementedError).
|
||||
if get_conv_copy_spec in copy_funcs and is_conv_state_dim_first():
|
||||
assert self.vllm_config.speculative_config is None, (
|
||||
"DS conv state layout does not support mamba align state "
|
||||
"copies with speculative decoding"
|
||||
)
|
||||
self._mamba_ctx = MambaSpecDecodeGPUContext.create(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
kv_cache_config=kv_cache_config,
|
||||
num_state_types=len(copy_funcs),
|
||||
device=self.device,
|
||||
make_buffer=lambda n, dtype: CpuGpuBuffer(
|
||||
n, dtype=dtype, device=self.device
|
||||
),
|
||||
)
|
||||
ctx = self._mamba_ctx
|
||||
if not ctx.is_initialized:
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
# block_tables are batch-order slices of the persistent
|
||||
# input_block_tables (stable data_ptr), so the metadata is captured
|
||||
# once here and reused across steps.
|
||||
ctx.initialize_from_forward_context(
|
||||
kv_cache_config,
|
||||
forward_context,
|
||||
self.model.get_mamba_state_copy_func(),
|
||||
[block_tables[gid] for gid in mamba_group_ids],
|
||||
)
|
||||
return ctx
|
||||
|
||||
def preprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
num_computed_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
"""Migrate each request's mamba state across block boundaries before the
|
||||
forward (V1 align semantics, done on GPU). Runs on real batches only
|
||||
(dummy DP/profiling runs skip preprocess_state), and before
|
||||
``prepare_attn`` gathers ``num_accepted_tokens``, so the boundary reset
|
||||
is visible to the forward kernels.
|
||||
"""
|
||||
if not self._align_mode:
|
||||
return
|
||||
num_reqs = input_batch.num_reqs
|
||||
if num_reqs == 0:
|
||||
return
|
||||
mamba_group_ids, mamba_spec = self._get_mamba_group_info(kv_cache_config)
|
||||
ctx = self._ensure_align_ctx(kv_cache_config, mamba_group_ids, block_tables)
|
||||
|
||||
# The state-advance + pre-copy kernels run every step; they fast-exit per
|
||||
# request when src_col < 0 or src_col == dst_col, so no copy happens on
|
||||
# steps that don't cross a block boundary. (Skipping the launch entirely
|
||||
# would need a V1-style async-D2H of the actual num_computed, since
|
||||
# num_computed_tokens_np is an optimistic mirror under async scheduling;
|
||||
# the launch cost is ~0.3% of TPOT, so the GPU fast-exit suffices.)
|
||||
block = 256
|
||||
grid = (triton.cdiv(num_reqs, block),)
|
||||
preprocess_mamba_align_fused_kernel[grid](
|
||||
input_batch.idx_mapping,
|
||||
self._mamba_state_idx_gpu,
|
||||
num_computed_tokens,
|
||||
input_batch.query_start_loc,
|
||||
self.num_accepted_tokens_gpu,
|
||||
self._mamba_src_col_gpu,
|
||||
self._mamba_src_off_gpu,
|
||||
num_reqs,
|
||||
BLOCK_SIZE=block,
|
||||
MAMBA_BLOCK_SIZE=mamba_spec.block_size,
|
||||
)
|
||||
ctx.run_fused_precopy(
|
||||
num_reqs,
|
||||
self._mamba_state_idx_gpu,
|
||||
self._mamba_src_col_gpu,
|
||||
self._mamba_src_off_gpu,
|
||||
input_batch.idx_mapping,
|
||||
)
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
@@ -145,22 +288,45 @@ class MambaHybridModelState(DefaultModelState):
|
||||
)
|
||||
|
||||
def postprocess_state(
|
||||
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor | int
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
num_sampled: torch.Tensor | int,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# Chunked prefill does not sample a token, so num_sampled can be 0.
|
||||
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
|
||||
if not isinstance(num_sampled, int):
|
||||
# idx_mapping may contain -1 sentinels (filtered rows) under PP; the
|
||||
# kernel skips them rather than scattering with a host-side gather.
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
if num_reqs:
|
||||
_scatter_num_accepted_kernel[(num_reqs,)](
|
||||
n = idx_mapping.shape[0]
|
||||
if n:
|
||||
_scatter_num_accepted_kernel[(n,)](
|
||||
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
|
||||
)
|
||||
return
|
||||
else:
|
||||
# Fill with single value.
|
||||
self.num_accepted_tokens_gpu.index_fill_(
|
||||
0, idx_mapping, max(num_sampled, 1)
|
||||
)
|
||||
|
||||
# Fill with single value.
|
||||
self.num_accepted_tokens_gpu.index_fill_(0, idx_mapping, max(num_sampled, 1))
|
||||
# Align: save the running state to the block-aligned position when
|
||||
# spec-decode acceptance leaves the sequence non-block-aligned (mirrors
|
||||
# the V1 align postprocess). num_computed_tokens already holds the
|
||||
# post-step advanced count.
|
||||
if (
|
||||
self._align_mode
|
||||
and num_computed_tokens is not None
|
||||
and self._mamba_ctx is not None
|
||||
):
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
if num_reqs:
|
||||
self._mamba_ctx.run_fused_postprocess_align(
|
||||
num_reqs,
|
||||
self.num_accepted_tokens_gpu,
|
||||
self._mamba_state_idx_gpu,
|
||||
num_computed_tokens,
|
||||
idx_mapping,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.v1.core.sched.output import (
|
||||
NewRequestData,
|
||||
SchedulerOutput,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
|
||||
@@ -177,9 +178,17 @@ def warmup_kernels(
|
||||
num_kv_cache_groups = len(kv_cache_groups)
|
||||
|
||||
# Compute per-request block counts for each KV cache group.
|
||||
group_block_sizes = [g.kv_cache_spec.block_size for g in kv_cache_groups]
|
||||
prefill_block_counts = [cdiv(prompt_len, bs) for bs in group_block_sizes]
|
||||
decode_block_counts = [cdiv(decode_len, bs) for bs in group_block_sizes]
|
||||
def _warmup_block_count(num_tokens: int, spec: Any) -> int:
|
||||
num_blocks = cdiv(num_tokens, spec.block_size)
|
||||
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode == "align":
|
||||
# Align mode reserves extra blocks beyond the token range for the
|
||||
# speculative-decode running-state snapshots.
|
||||
num_blocks += spec.num_speculative_blocks
|
||||
return num_blocks
|
||||
|
||||
kv_cache_specs = [g.kv_cache_spec for g in kv_cache_groups]
|
||||
prefill_block_counts = [_warmup_block_count(prompt_len, s) for s in kv_cache_specs]
|
||||
decode_block_counts = [_warmup_block_count(decode_len, s) for s in kv_cache_specs]
|
||||
decode_block_deltas = [
|
||||
d - p for d, p in zip(decode_block_counts, prefill_block_counts)
|
||||
]
|
||||
|
||||
+368
-92
@@ -23,6 +23,112 @@ from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _copy_mamba_state_block(
|
||||
state_idx,
|
||||
bt_row_idx,
|
||||
src_col,
|
||||
dst_col,
|
||||
token_bias,
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
state_inner_sizes_ptr,
|
||||
state_conv_widths_ptr,
|
||||
state_group_indices_ptr,
|
||||
# DS conv row metadata. Zero keeps the single-region copy path.
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
CONV_STATE_DIM_FIRST: tl.constexpr,
|
||||
):
|
||||
"""Copy one (layer, state-type) mamba state block between block columns.
|
||||
|
||||
Shared copy body of ``postprocess_mamba_fused_kernel`` and
|
||||
``precopy_mamba_align_fused_kernel``, mirroring the V1 copy specs
|
||||
(``get_conv_copy_spec`` / ``get_temporal_copy_spec``):
|
||||
- conv state (conv_width > 0): shift the window by ``token_bias`` tokens,
|
||||
``state[bt[src_col], token_bias:] ->
|
||||
state[bt[dst_col], :conv_width - token_bias]``
|
||||
- temporal state: ``token_bias`` selects the accepted speculative column,
|
||||
``state[bt[src_col + token_bias]] -> state[bt[dst_col]]``
|
||||
|
||||
The caller owns the decision logic (which columns, whether to copy); this
|
||||
device function only performs the byte copy for the given metadata slot.
|
||||
"""
|
||||
state_base_addr = tl.load(state_base_addrs_ptr + state_idx)
|
||||
state_block_stride = tl.load(state_block_strides_ptr + state_idx)
|
||||
state_elem_size = tl.load(state_elem_sizes_ptr + state_idx)
|
||||
state_inner_size = tl.load(state_inner_sizes_ptr + state_idx)
|
||||
conv_width = tl.load(state_conv_widths_ptr + state_idx)
|
||||
|
||||
# Load the group index for this state, then index into the correct
|
||||
# group's block table. Each mamba group has independently allocated
|
||||
# physical blocks. Reinterpret as int32* since block ids are int32.
|
||||
group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64)
|
||||
group_base_addr = tl.load(block_table_ptrs_ptr + group_idx)
|
||||
block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32))
|
||||
block_table_base = block_table_typed + bt_row_idx * block_table_stride_req
|
||||
|
||||
# Widen block ids to int64 before they reach `block_id * state_block_stride`
|
||||
# below: state_block_stride can exceed 2**31 bytes for large mamba caches,
|
||||
# and Triton would otherwise do the multiply in int32 and wrap.
|
||||
dest_block_id = tl.load(block_table_base + dst_col).to(tl.int64)
|
||||
dst_addr = state_base_addr + dest_block_id * state_block_stride
|
||||
|
||||
is_conv_state = conv_width > 0
|
||||
|
||||
if CONV_STATE_DIM_FIRST and is_conv_state:
|
||||
# DS conv layout: state_len is the slide axis; copy per dim row.
|
||||
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
|
||||
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
|
||||
row_stride = tl.load(state_dim_row_stride_ptr + state_idx)
|
||||
per_row_bytes = (conv_width - token_bias).to(tl.int64) * state_elem_size
|
||||
bias_bytes = token_bias.to(tl.int64) * state_elem_size
|
||||
src_block_addr = state_base_addr + src_block_id * state_block_stride
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for d in range(0, dim_rows):
|
||||
row_src = src_block_addr + d * row_stride + bias_bytes
|
||||
row_dst = dst_addr + d * row_stride
|
||||
for i in range(0, per_row_bytes, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < per_row_bytes
|
||||
curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
data = tl.load(curr_src, mask=mask)
|
||||
tl.store(curr_dst, data, mask=mask)
|
||||
return
|
||||
|
||||
if is_conv_state:
|
||||
# SD conv: copy
|
||||
# state[bt[src_col], token_bias:] ->
|
||||
# state[bt[dst_col], :conv_width - token_bias]
|
||||
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
|
||||
src_offset = token_bias.to(tl.int64) * state_inner_size * state_elem_size
|
||||
src_addr = state_base_addr + src_block_id * state_block_stride + src_offset
|
||||
num_elems_to_copy = (conv_width - token_bias).to(tl.int64) * state_inner_size
|
||||
copy_size = num_elems_to_copy * state_elem_size
|
||||
else:
|
||||
# Temporal state: copy state[bt[src_col + token_bias]] -> state[bt[dst_col]]
|
||||
actual_src_block_id = tl.load(block_table_base + src_col + token_bias).to(
|
||||
tl.int64
|
||||
)
|
||||
src_addr = state_base_addr + actual_src_block_id * state_block_stride
|
||||
# Use natural block data size (inner_size * elem_size), NOT
|
||||
# state_block_stride which is the page stride and can exceed the
|
||||
# actual data when the state tensor uses as_strided page padding.
|
||||
copy_size = state_inner_size * state_elem_size
|
||||
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for i in range(0, copy_size, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < copy_size
|
||||
curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
data = tl.load(curr_src, mask=mask)
|
||||
tl.store(curr_dst, data, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def postprocess_mamba_fused_kernel(
|
||||
# Decision inputs (per-request)
|
||||
@@ -49,6 +155,10 @@ def postprocess_mamba_fused_kernel(
|
||||
state_dim_row_stride_ptr, # int64: bytes between rows for DS conv
|
||||
# Output: num_accepted_tokens update (for src==dst case)
|
||||
num_accepted_tokens_out_ptr,
|
||||
# Optional: batch_idx -> req_idx mapping (V2 model runner / PP). The
|
||||
# per-request decision arrays are in req-state-slot order; the block table
|
||||
# is in batch order, so HAS_IDX_MAPPING splits the two indexings.
|
||||
idx_mapping_ptr,
|
||||
# Runtime parameter (varies per batch - NOT constexpr to avoid recompilation)
|
||||
num_reqs,
|
||||
# Compile-time constants (fixed after model initialization)
|
||||
@@ -57,35 +167,53 @@ def postprocess_mamba_fused_kernel(
|
||||
# COPY_BLOCK_SIZE: fixed tuning parameter for memory copy loop
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
CONV_STATE_DIM_FIRST: tl.constexpr,
|
||||
# HAS_IDX_MAPPING: when True, program_id(0) is a batch index resolved to a
|
||||
# req-state slot via idx_mapping_ptr (V2). When False, it is the req index.
|
||||
HAS_IDX_MAPPING: tl.constexpr = False,
|
||||
# PRECOMPUTED_NEW_COMPUTED: when True, num_computed_tokens_ptr already holds
|
||||
# the post-step new_num_computed value (V2 supplies the advanced count).
|
||||
PRECOMPUTED_NEW_COMPUTED: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Fused GPU kernel for postprocess_mamba that computes decisions AND performs
|
||||
mamba state copies without any CPU-GPU synchronization.
|
||||
|
||||
Grid: (num_reqs, num_layers * num_state_types)
|
||||
- program_id(0) = request index
|
||||
- program_id(0) = request/batch index
|
||||
- program_id(1) = state_idx (flattened index into layer/state_type metadata)
|
||||
|
||||
Note: num_layers and num_state_types are not passed as kernel parameters
|
||||
because the kernel indexes directly into pre-flattened metadata arrays
|
||||
using program_id(1). The grid dimensions encode the total state count.
|
||||
"""
|
||||
req_idx = tl.program_id(0)
|
||||
batch_idx = tl.program_id(0)
|
||||
state_idx = tl.program_id(1)
|
||||
|
||||
# Bounds check
|
||||
if req_idx >= num_reqs:
|
||||
if batch_idx >= num_reqs:
|
||||
return
|
||||
|
||||
if HAS_IDX_MAPPING:
|
||||
req_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
if req_idx < 0:
|
||||
return
|
||||
else:
|
||||
req_idx = batch_idx
|
||||
|
||||
# Compute decision logic (mirrors postprocess_mamba Python reference)
|
||||
num_accepted = tl.load(num_accepted_tokens_ptr + req_idx)
|
||||
src_block_idx = tl.load(mamba_state_idx_ptr + req_idx)
|
||||
num_scheduled = tl.load(num_scheduled_tokens_ptr + req_idx)
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_idx)
|
||||
num_draft = tl.load(num_draft_tokens_ptr + req_idx)
|
||||
|
||||
num_tokens_running_state = num_computed + num_scheduled - num_draft
|
||||
new_num_computed = num_tokens_running_state + num_accepted - 1
|
||||
if PRECOMPUTED_NEW_COMPUTED:
|
||||
new_num_computed = tl.load(num_computed_tokens_ptr + req_idx)
|
||||
num_tokens_running_state = new_num_computed - num_accepted + 1
|
||||
else:
|
||||
num_scheduled = tl.load(num_scheduled_tokens_ptr + req_idx)
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_idx)
|
||||
num_draft = tl.load(num_draft_tokens_ptr + req_idx)
|
||||
num_tokens_running_state = num_computed + num_scheduled - num_draft
|
||||
new_num_computed = num_tokens_running_state + num_accepted - 1
|
||||
|
||||
aligned_new_computed = (new_num_computed // block_size) * block_size
|
||||
|
||||
needs_copy = aligned_new_computed >= num_tokens_running_state
|
||||
@@ -97,99 +225,158 @@ def postprocess_mamba_fused_kernel(
|
||||
accept_token_bias = aligned_new_computed - num_tokens_running_state
|
||||
dest_block_idx = aligned_new_computed // block_size - 1
|
||||
|
||||
# Load state metadata for this layer/state_type
|
||||
state_base_addr = tl.load(state_base_addrs_ptr + state_idx)
|
||||
state_block_stride = tl.load(state_block_strides_ptr + state_idx)
|
||||
state_elem_size = tl.load(state_elem_sizes_ptr + state_idx)
|
||||
state_inner_size = tl.load(state_inner_sizes_ptr + state_idx)
|
||||
conv_width = tl.load(state_conv_widths_ptr + state_idx)
|
||||
|
||||
# Load the group index for this state, then index into the correct
|
||||
# group's block table. Each mamba group has independently allocated
|
||||
# physical blocks.
|
||||
group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64)
|
||||
|
||||
# block_table_ptrs_ptr holds one pointer per group (each group owns its own
|
||||
# block table). Reinterpret as int32* since block ids are int32.
|
||||
group_base_addr = tl.load(block_table_ptrs_ptr + group_idx)
|
||||
block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32))
|
||||
block_table_base = block_table_typed + req_idx * block_table_stride_req
|
||||
|
||||
# Widen block ids to int64 before they reach `block_id * state_block_stride`
|
||||
# below: state_block_stride can exceed 2**31 bytes for large mamba caches,
|
||||
# and Triton would otherwise do the multiply in int32 and wrap.
|
||||
src_block_id = tl.load(block_table_base + src_block_idx).to(tl.int64)
|
||||
dest_block_id = tl.load(block_table_base + dest_block_idx).to(tl.int64)
|
||||
|
||||
# Compute source and destination addresses based on state type
|
||||
# conv_width > 0 means this is a conv state (get_conv_copy_spec logic)
|
||||
# conv_width == 0 means this is a temporal state (get_temporal_copy_spec logic)
|
||||
is_conv_state = conv_width > 0
|
||||
|
||||
# Update accepted-token count before early exits.
|
||||
# Update accepted-token count before early exits (per-request, so only
|
||||
# state_idx == 0 writes). V2 updates in place; V1 writes the _out buffer.
|
||||
if src_block_idx == dest_block_idx and state_idx == 0:
|
||||
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
|
||||
if HAS_IDX_MAPPING:
|
||||
tl.store(num_accepted_tokens_ptr + req_idx, 1)
|
||||
else:
|
||||
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
|
||||
|
||||
# Skip no-op self-copy.
|
||||
if src_block_idx == dest_block_idx and accept_token_bias == 0:
|
||||
return
|
||||
|
||||
if CONV_STATE_DIM_FIRST and is_conv_state:
|
||||
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
|
||||
row_stride = tl.load(state_dim_row_stride_ptr + state_idx)
|
||||
per_row_bytes = (conv_width - accept_token_bias).to(tl.int64) * state_elem_size
|
||||
bias_bytes = accept_token_bias.to(tl.int64) * state_elem_size
|
||||
src_block_addr = state_base_addr + src_block_id * state_block_stride
|
||||
dst_block_addr = state_base_addr + dest_block_id * state_block_stride
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for d in range(0, dim_rows):
|
||||
row_src = src_block_addr + d * row_stride + bias_bytes
|
||||
row_dst = dst_block_addr + d * row_stride
|
||||
for i in range(0, per_row_bytes, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < per_row_bytes
|
||||
curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
data = tl.load(curr_src, mask=mask)
|
||||
tl.store(curr_dst, data, mask=mask)
|
||||
bt_row_idx = batch_idx if HAS_IDX_MAPPING else req_idx
|
||||
_copy_mamba_state_block(
|
||||
state_idx,
|
||||
bt_row_idx,
|
||||
src_block_idx,
|
||||
dest_block_idx,
|
||||
accept_token_bias,
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
state_inner_sizes_ptr,
|
||||
state_conv_widths_ptr,
|
||||
state_group_indices_ptr,
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
COPY_BLOCK_SIZE,
|
||||
CONV_STATE_DIM_FIRST,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def preprocess_mamba_align_fused_kernel(
|
||||
idx_mapping_ptr,
|
||||
state_idx_ptr,
|
||||
num_computed_tokens_ptr,
|
||||
query_start_loc_ptr,
|
||||
num_accepted_tokens_ptr,
|
||||
src_col_ptr,
|
||||
src_off_ptr,
|
||||
num_reqs,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
MAMBA_BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Fused align preprocess: emit the pre-copy src column/offset AND advance
|
||||
state_idx (with accepted-token reset) in a single launch (V2 align).
|
||||
|
||||
Per batch_idx (0..num_reqs-1), resolving req slot via idx_mapping:
|
||||
1. Read pre-advance state_idx and num_accepted (last step's values).
|
||||
2. Store the pre-copy src columns for ``precopy_mamba_align_fused_kernel``:
|
||||
- src_col = state_idx (the previous running block column)
|
||||
- src_off = max(num_accepted - 1, 0) (the accepted-token bias)
|
||||
3. Advance state_idx to the new running block, and reset num_accepted to 1
|
||||
when a block boundary is crossed (so the migrated state, now at the
|
||||
start of the new block, is read with the neutral bias).
|
||||
"""
|
||||
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < num_reqs
|
||||
req_indices = tl.load(idx_mapping_ptr + offsets, mask=mask, other=0)
|
||||
|
||||
state_idx = tl.load(state_idx_ptr + req_indices, mask=mask, other=-1)
|
||||
num_accepted = tl.load(num_accepted_tokens_ptr + req_indices, mask=mask, other=1)
|
||||
|
||||
src_off = tl.maximum(num_accepted - 1, 0)
|
||||
tl.store(src_col_ptr + req_indices, state_idx, mask=mask)
|
||||
tl.store(src_off_ptr + req_indices, src_off, mask=mask)
|
||||
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_indices, mask=mask, other=0)
|
||||
query_start = tl.load(query_start_loc_ptr + offsets, mask=mask, other=0)
|
||||
query_end = tl.load(query_start_loc_ptr + offsets + 1, mask=mask, other=0)
|
||||
computed_after = num_computed + query_end - query_start
|
||||
new_state_idx = (computed_after + MAMBA_BLOCK_SIZE - 1) // MAMBA_BLOCK_SIZE - 1
|
||||
tl.store(state_idx_ptr + req_indices, new_state_idx, mask=mask)
|
||||
should_reset = (state_idx >= 0) & (state_idx != new_state_idx)
|
||||
tl.store(num_accepted_tokens_ptr + req_indices, 1, mask=mask & should_reset)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def precopy_mamba_align_fused_kernel(
|
||||
# Per-request-slot inputs (indexed by req_idx via idx_mapping), produced by
|
||||
# the V2 fused align preprocess kernel for the current step:
|
||||
mamba_state_idx_ptr, # post-advance dst block column
|
||||
src_col_ptr, # pre-advance src block column (-1 = fresh)
|
||||
token_bias_ptr, # accepted-token bias = num_accepted - 1 (pre-reset)
|
||||
# Same flattened state-layout metadata as postprocess_mamba_fused_kernel
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req: tl.int64,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
state_inner_sizes_ptr,
|
||||
state_conv_widths_ptr,
|
||||
state_group_indices_ptr,
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip)
|
||||
num_reqs,
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
CONV_STATE_DIM_FIRST: tl.constexpr,
|
||||
):
|
||||
"""Pre-copy mamba "align" state across block boundaries on the V2 runner.
|
||||
|
||||
Before the forward pass, copy each request's last SSM/conv state from its
|
||||
previous block column into the new window block column, so the kernels read
|
||||
the initial state from the write-side block as usual (V1 align semantics).
|
||||
Same per-(layer, state) copy semantics as ``postprocess_mamba_fused_kernel``
|
||||
(shared ``_copy_mamba_state_block`` body, i.e. the V1 ``preprocess_mamba``
|
||||
copy specs), but driven by the GPU-resident src columns so it needs no
|
||||
CPU-GPU sync (async-scheduling safe).
|
||||
|
||||
Grid: (num_reqs, num_layers * num_state_types); block tables are indexed by
|
||||
batch row, per-request state by req_idx via idx_mapping (V2 layout).
|
||||
"""
|
||||
batch_idx = tl.program_id(0)
|
||||
state_idx = tl.program_id(1)
|
||||
if batch_idx >= num_reqs:
|
||||
return
|
||||
req_idx = tl.load(idx_mapping_ptr + batch_idx)
|
||||
if req_idx < 0:
|
||||
return
|
||||
|
||||
if is_conv_state:
|
||||
# SD conv: copy
|
||||
# state[block_table[req_idx, src_block_idx], accept_token_bias:]
|
||||
# to
|
||||
# state[block_table[req_idx, dest_block_idx], :conv_width - accept_token_bias]
|
||||
src_offset = accept_token_bias.to(tl.int64) * state_inner_size * state_elem_size
|
||||
src_addr = state_base_addr + src_block_id * state_block_stride + src_offset
|
||||
dst_addr = state_base_addr + dest_block_id * state_block_stride
|
||||
# Number of elements to copy:
|
||||
# (conv_width - accept_token_bias) * inner_size
|
||||
num_elems_to_copy = (conv_width - accept_token_bias).to(
|
||||
tl.int64
|
||||
) * state_inner_size
|
||||
copy_size = num_elems_to_copy * state_elem_size
|
||||
else:
|
||||
# Temporal state: copy
|
||||
# state[block_table[req_idx, src_block_idx + accept_token_bias]]
|
||||
# to
|
||||
# state[block_table[req_idx, dest_block_idx]]
|
||||
actual_src_block_idx = src_block_idx + accept_token_bias
|
||||
actual_src_block_id = tl.load(block_table_base + actual_src_block_idx).to(
|
||||
tl.int64
|
||||
)
|
||||
src_addr = state_base_addr + actual_src_block_id * state_block_stride
|
||||
dst_addr = state_base_addr + dest_block_id * state_block_stride
|
||||
# Use natural block data size (inner_size * elem_size), NOT
|
||||
# state_block_stride which is the page stride and can exceed the
|
||||
# actual data when the state tensor uses as_strided page padding.
|
||||
copy_size = state_inner_size * state_elem_size
|
||||
src_col = tl.load(src_col_ptr + req_idx)
|
||||
dst_col = tl.load(mamba_state_idx_ptr + req_idx)
|
||||
# Fresh state, or still writing the same block: kernels locate the initial
|
||||
# state in-block via num_accepted (preserved when no boundary is crossed),
|
||||
# so there is nothing to copy.
|
||||
if src_col < 0 or src_col == dst_col:
|
||||
return
|
||||
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for i in range(0, copy_size, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < copy_size
|
||||
curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
data = tl.load(curr_src, mask=mask)
|
||||
tl.store(curr_dst, data, mask=mask)
|
||||
token_bias = tl.load(token_bias_ptr + req_idx)
|
||||
_copy_mamba_state_block(
|
||||
state_idx,
|
||||
batch_idx,
|
||||
src_col,
|
||||
dst_col,
|
||||
token_bias,
|
||||
block_table_ptrs_ptr,
|
||||
block_table_stride_req,
|
||||
state_base_addrs_ptr,
|
||||
state_block_strides_ptr,
|
||||
state_elem_sizes_ptr,
|
||||
state_inner_sizes_ptr,
|
||||
state_conv_widths_ptr,
|
||||
state_group_indices_ptr,
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
COPY_BLOCK_SIZE,
|
||||
CONV_STATE_DIM_FIRST,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -559,12 +746,101 @@ class MambaSpecDecodeGPUContext:
|
||||
self.state_dim_row_count,
|
||||
self.state_dim_row_stride,
|
||||
self.num_accepted_tokens_out,
|
||||
None, # idx_mapping: V1 decision arrays are already in req order
|
||||
num_reqs,
|
||||
block_size=self.block_size,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
)
|
||||
|
||||
def run_fused_precopy(
|
||||
self,
|
||||
num_reqs: int,
|
||||
state_idx_gpu: torch.Tensor,
|
||||
src_col_gpu: torch.Tensor,
|
||||
token_bias_gpu: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""Pre-copy each request's previous running block into its new window
|
||||
block before the forward pass (V2 align boundary migration).
|
||||
|
||||
Args:
|
||||
num_reqs: Number of active requests (batch order).
|
||||
state_idx_gpu: [max_reqs] post-advance dst block column per req slot.
|
||||
src_col_gpu: [max_reqs] pre-advance src block column (-1 = fresh).
|
||||
token_bias_gpu: [max_reqs] accepted-token bias (num_accepted - 1).
|
||||
idx_mapping: [num_reqs] batch_idx -> req_state_idx (-1 to skip).
|
||||
"""
|
||||
if num_reqs == 0 or not self.is_initialized:
|
||||
return
|
||||
total_states = self.num_layers * self.num_state_types
|
||||
grid = (num_reqs, total_states)
|
||||
precopy_mamba_align_fused_kernel[grid](
|
||||
state_idx_gpu,
|
||||
src_col_gpu,
|
||||
token_bias_gpu,
|
||||
self.block_table_ptrs,
|
||||
self.block_table_stride_req,
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
self.state_elem_sizes,
|
||||
self.state_inner_sizes,
|
||||
self.state_conv_widths,
|
||||
self.state_group_indices,
|
||||
self.state_dim_row_count,
|
||||
self.state_dim_row_stride,
|
||||
idx_mapping,
|
||||
num_reqs,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
)
|
||||
|
||||
def run_fused_postprocess_align(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_accepted_tokens_gpu: torch.Tensor,
|
||||
state_idx_gpu: torch.Tensor,
|
||||
new_num_computed_tokens_gpu: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
"""V2 align postprocess: save the running state to the block-aligned
|
||||
position after spec-decode acceptance leaves the sequence non-aligned.
|
||||
|
||||
``num_accepted_tokens_gpu`` is updated in place (reset to 1 when the
|
||||
accepted position stays in the running block); ``new_num_computed_tokens``
|
||||
already holds the post-step computed count (PRECOMPUTED_NEW_COMPUTED).
|
||||
``idx_mapping`` maps batch row -> req-state slot (HAS_IDX_MAPPING).
|
||||
"""
|
||||
if num_reqs == 0 or not self.is_initialized:
|
||||
return
|
||||
total_states = self.num_layers * self.num_state_types
|
||||
grid = (num_reqs, total_states)
|
||||
postprocess_mamba_fused_kernel[grid](
|
||||
num_accepted_tokens_gpu,
|
||||
state_idx_gpu,
|
||||
None, # num_scheduled: unused under PRECOMPUTED_NEW_COMPUTED
|
||||
new_num_computed_tokens_gpu,
|
||||
None, # num_draft: unused under PRECOMPUTED_NEW_COMPUTED
|
||||
self.block_table_ptrs,
|
||||
self.block_table_stride_req,
|
||||
self.state_base_addrs,
|
||||
self.state_block_strides,
|
||||
self.state_elem_sizes,
|
||||
self.state_inner_sizes,
|
||||
self.state_conv_widths,
|
||||
self.state_group_indices,
|
||||
self.state_dim_row_count,
|
||||
self.state_dim_row_stride,
|
||||
None, # num_accepted_out: V2 updates num_accepted in place
|
||||
idx_mapping,
|
||||
num_reqs,
|
||||
block_size=self.block_size,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
HAS_IDX_MAPPING=True,
|
||||
PRECOMPUTED_NEW_COMPUTED=True,
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MambaBuffers:
|
||||
|
||||
Reference in New Issue
Block a user