[Model Runner V2] support mamba hybrid models align prefix cache (#42406)

Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>
This commit is contained in:
zhrrr
2026-06-29 14:09:16 -07:00
committed by GitHub
parent a309d4fe60
commit 61ab70ec3b
9 changed files with 1046 additions and 133 deletions
@@ -0,0 +1,180 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Equivalence test for ``precopy_mamba_align_fused_kernel``.
The V2 "align" pre-copy must migrate mamba state across block boundaries with
byte-identical semantics to the V1 copy specs (``get_conv_copy_spec`` /
``get_temporal_copy_spec``):
* conv state (SD layout, conv_width > 0): shift the sliding window by
``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` ->
``state[bt[dst_col], :conv_width - token_bias]``.
* temporal state (conv_width == 0): ``token_bias`` selects the accepted
speculative column -- ``state[bt[src_col + token_bias]]`` ->
``state[bt[dst_col]]``.
The kernel must also no-op when ``src_col < 0`` (fresh request) or
``src_col == dst_col`` (no boundary crossed).
"""
from __future__ import annotations
import torch
from vllm.platforms import current_platform
from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel
try:
import pytest
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="precopy_mamba_align_fused_kernel needs CUDA/Triton",
)
_parametrize = pytest.mark.parametrize
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
pytest = None
def _parametrize(_name, _values):
def _deco(fn):
return fn
return _deco
NUM_LAYERS = 3
CONV_WIDTH = 4 # conv_kernel - 1 + num_spec
CONV_DIM = 96
SSM_SHAPE = (4, 16, 16)
MAX_COLS = 8
def _build_state(num_blocks, device):
"""Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools."""
convs, ssms = [], []
for _ in range(NUM_LAYERS):
convs.append(
torch.randn(
num_blocks, CONV_WIDTH, CONV_DIM, dtype=torch.bfloat16, device=device
)
)
ssms.append(
torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device)
)
return convs, ssms
def _build_meta(convs, ssms, device):
"""Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer."""
n = NUM_LAYERS * 2
base = torch.zeros(n, dtype=torch.int64, device=device)
blk_stride = torch.zeros(n, dtype=torch.int64, device=device)
elem = torch.zeros(n, dtype=torch.int32, device=device)
inner = torch.zeros(n, dtype=torch.int64, device=device)
width = torch.zeros(n, dtype=torch.int32, device=device)
group = torch.zeros(n, dtype=torch.int32, device=device)
drc = torch.zeros(n, dtype=torch.int32, device=device) # DS rows (unused, SD)
drs = torch.zeros(n, dtype=torch.int64, device=device)
i = 0
for layer in range(NUM_LAYERS):
conv, ssm = convs[layer], ssms[layer]
# conv (SD): width = size(1), inner = stride(1)
base[i] = conv.data_ptr()
blk_stride[i] = conv.stride(0) * conv.element_size()
elem[i] = conv.element_size()
width[i] = conv.size(1)
inner[i] = conv.stride(1)
i += 1
# ssm (temporal): width = 0, inner = elems per block
base[i] = ssm.data_ptr()
blk_stride[i] = ssm.stride(0) * ssm.element_size()
elem[i] = ssm.element_size()
width[i] = 0
inner[i] = ssm[0].numel()
i += 1
return base, blk_stride, elem, inner, width, group, drc, drs
def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs):
"""Apply the V1 copy semantics on clones, reading from the pre-copy state."""
conv_pre = [c.clone() for c in convs]
ssm_pre = [s.clone() for s in ssms]
conv_ref = [c.clone() for c in convs]
ssm_ref = [s.clone() for s in ssms]
for r in range(num_reqs):
sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r])
if sc < 0 or sc == dc:
continue
sblk, dblk = int(bt[r, sc]), int(bt[r, dc])
tblk = int(bt[r, sc + tb]) # temporal src column shifted by bias
for layer in range(NUM_LAYERS):
conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:]
ssm_ref[layer][dblk] = ssm_pre[layer][tblk]
return conv_ref, ssm_ref
@_parametrize("num_reqs", [1, 4, 16])
@_parametrize("token_bias", [0, 1, 2])
def test_precopy_matches_v1_copy_specs(num_reqs, token_bias):
device = torch.device("cuda")
torch.manual_seed(0)
# Distinct physical block per (req, col) so copies never alias.
num_blocks = num_reqs * MAX_COLS + 1
bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device)
for r in range(num_reqs):
bt[r] = torch.arange(
1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device
)
# Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip),
# the rest cross from col 1 -> col 0 with the given spec token bias.
src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device)
dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device)
bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device)
if num_reqs >= 1:
src_col[0] = -1 # fresh -> no copy
if num_reqs >= 2:
dst_col[1] = 1 # src_col == dst_col -> no copy
convs, ssms = _build_state(num_blocks, device)
conv_ref, ssm_ref = _reference(
convs, ssms, bt.cpu(), src_col.cpu(), dst_col.cpu(), bias.cpu(), num_reqs
)
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
convs, ssms, device
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
grid = (num_reqs, NUM_LAYERS * 2)
precopy_mamba_align_fused_kernel[grid](
dst_col,
src_col,
bias,
bt_ptrs,
bt.stride(0),
base,
blk_stride,
elem,
inner,
width,
group,
drc,
drs,
idx_mapping,
num_reqs,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=False,
)
torch.accelerator.synchronize()
for layer in range(NUM_LAYERS):
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
if __name__ == "__main__":
for nr in (1, 4, 16):
for tb in (0, 1, 2):
test_precopy_matches_v1_copy_specs(nr, tb)
print(f"OK num_reqs={nr} token_bias={tb}")
+280 -16
View File
@@ -11,6 +11,7 @@ import datasets
import pytest
import torch
import vllm.envs as envs
from tests.utils import create_new_process_for_each_test
from vllm import LLM, SamplingParams, TokensPrompt
from vllm.config import CacheConfig
@@ -494,12 +495,7 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn)
@create_new_process_for_each_test()
def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
run_ref_mamba_state_in_subprocess()
apply_patch(monkeypatch)
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
full_prompt = prompt_dataset["train"][0]["text"]
def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]:
tests = {
"accept_1": TestConfig(
num_prompt_tokens=554,
@@ -731,6 +727,27 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
),
}
return tests
def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None:
for step_action_prev, step_action_next in zip(
test_config.step_actions[:-1], test_config.step_actions[1:]
):
if len(step_action_next.kv_cache_block_ids) == 0:
step_action_next.kv_cache_block_ids = (
step_action_prev.kv_cache_block_ids.copy()
)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
run_ref_mamba_state_in_subprocess()
apply_patch(monkeypatch)
prompt_dataset = datasets.load_dataset("heheda/a_long_article")
full_prompt = prompt_dataset["train"][0]["text"]
tests = get_mamba_prefix_cache_step_configs()
engine = LLM(
model=MODEL,
enable_prefix_caching=True,
@@ -758,16 +775,7 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
)
global cur_step_action_idx
cur_step_action_idx = 0
for step_action_prev, step_action_next in zip(
test_config.step_actions[:-1], test_config.step_actions[1:]
):
if (
step_action_next.kv_cache_block_ids is not None
and len(step_action_next.kv_cache_block_ids) == 0
):
prev_block_ids = step_action_prev.kv_cache_block_ids
if prev_block_ids is not None:
step_action_next.kv_cache_block_ids = prev_block_ids.copy()
fill_following_kv_cache_block_ids(test_config)
global step_actions
step_actions = test_config.step_actions
_ = engine.generate(
@@ -787,3 +795,259 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch):
del engine
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
envs.disable_envs_cache()
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as MRV2GPUModelRunner
from vllm.v1.worker.gpu.model_states.mamba_hybrid import (
MambaHybridModelState,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput as MRV2SamplerOutput
events: list[int] = []
original_execute_model = MRV2GPUModelRunner.execute_model
original_sample = MRV2GPUModelRunner.sample
original_preprocess_state = MambaHybridModelState.preprocess_state
original_postprocess_state = MambaHybridModelState.postprocess_state
original_step_action_fn = InprocClient.get_output
original_allocate_slots = KVCacheManager.allocate_slots
captured: dict[str, Any] = {}
def temporal_states(model_state, block_tables, kv_cache_config):
# Qwen3-Next keeps the temporal (ssm) state as the last Mamba cache.
forward_context = (
model_state.vllm_config.compilation_config.static_forward_context
)
group_ids, _ = get_mamba_groups(kv_cache_config)
for group_id in group_ids:
block_table = block_tables[group_id]
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names:
yield forward_context[layer_name].kv_cache[-1], block_table
def temporal_block(temporal_state, block_table, col):
return temporal_state[int(block_table[0, col].item())]
def wrapped_preprocess_state(
self: MambaHybridModelState,
input_batch: Any,
block_tables: tuple[torch.Tensor, ...],
kv_cache_config: KVCacheConfig,
num_computed_tokens: torch.Tensor,
) -> None:
captured["block_tables"] = block_tables
captured["kv_cache_config"] = kv_cache_config
expected = (
None if cur_step_action is None else cur_step_action.preprocess_copy_idx
)
snapshots = []
if expected is not None and expected != (-1, -1):
for temporal, bt in temporal_states(self, block_tables, kv_cache_config):
snapshots.append(
(temporal, bt, temporal_block(temporal, bt, expected[0]).clone())
)
ret = original_preprocess_state(
self, input_batch, block_tables, kv_cache_config, num_computed_tokens
)
if cur_step_action is not None:
req_idx = int(input_batch.idx_mapping[0].item())
src_col = int(self._mamba_src_col_gpu[req_idx].item())
off = int(self._mamba_src_off_gpu[req_idx].item())
dst = int(self._mamba_state_idx_gpu[req_idx].item())
actual = (-1, -1) if src_col < 0 or src_col == dst else (src_col + off, dst)
assert actual == expected, (
f"V2 align preprocess copy: expected={expected}, "
f"actual={actual}, {cur_step_action=}"
)
for temporal, bt, src_state in snapshots:
torch.testing.assert_close(
temporal_block(temporal, bt, expected[1]), src_state
)
return ret
def wrapped_postprocess_state(
self: MambaHybridModelState,
idx_mapping: torch.Tensor,
num_sampled: torch.Tensor | int,
num_computed_tokens: torch.Tensor | None = None,
) -> None:
action = cur_step_action
block_tables = captured.get("block_tables")
kv_cache_config = captured.get("kv_cache_config")
# The postprocess kernel does not expose its indices, so only the copy
# case is checked, by effect: snapshot the src block, expect dst == src.
if (
action is None
or num_computed_tokens is None
or block_tables is None
or action.postprocess_copy_idx == (-1, -1)
):
return original_postprocess_state(
self, idx_mapping, num_sampled, num_computed_tokens
)
expected = action.postprocess_copy_idx
snapshots = [
(temporal, bt, temporal_block(temporal, bt, expected[0]).clone())
for temporal, bt in temporal_states(self, block_tables, kv_cache_config)
]
ret = original_postprocess_state(
self, idx_mapping, num_sampled, num_computed_tokens
)
for temporal, bt, src_state in snapshots:
torch.testing.assert_close(
temporal_block(temporal, bt, expected[1]), src_state
)
return ret
def wrapped_execute_model(
self: MRV2GPUModelRunner,
scheduler_output: SchedulerOutput,
*args: Any,
**kwargs: Any,
):
events.extend(
req.num_computed_tokens for req in scheduler_output.scheduled_new_reqs
)
events.extend(scheduler_output.scheduled_cached_reqs.num_computed_tokens)
if cur_step_action is not None:
num_scheduled_tokens = next(
iter(scheduler_output.num_scheduled_tokens.values())
)
assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens
ret = original_execute_model(self, scheduler_output, *args, **kwargs)
if cur_step_action is not None and self.execute_model_state is not None:
input_batch = self.execute_model_state.input_batch
assert (
cur_step_action.num_computed_tokens_start
== input_batch.positions[input_batch.query_start_loc[0]].item()
)
return ret
def fake_sample(
self: MRV2GPUModelRunner,
hidden_states: torch.Tensor,
input_batch: Any,
grammar_output: Any,
):
if cur_step_action is None:
return original_sample(self, hidden_states, input_batch, grammar_output)
num_reqs = input_batch.num_reqs
sampled_token_ids = torch.ones(
(num_reqs, self.num_speculative_steps + 1),
device=hidden_states.device,
dtype=torch.int64,
)
num_logits = torch.tensor(
input_batch.cu_num_logits_np[1 : num_reqs + 1]
- input_batch.cu_num_logits_np[:num_reqs],
device=hidden_states.device,
dtype=torch.int32,
)
accepted = torch.full_like(num_logits, num_accepted_tokens)
num_sampled = torch.minimum(accepted, num_logits)
prefill_lens = self.req_states.prefill_len.gpu[input_batch.idx_mapping]
is_chunked_prefill = input_batch.seq_lens[:num_reqs] < prefill_lens
num_sampled = torch.where(is_chunked_prefill, 0, num_sampled)
num_rejected = torch.where(is_chunked_prefill, 0, num_logits - num_sampled)
sampler_output = MRV2SamplerOutput(
sampled_token_ids=sampled_token_ids,
logprobs_tensors=None,
num_nans=None,
num_sampled=num_sampled,
)
return sampler_output, num_sampled, num_rejected
monkeypatch.setattr(
InprocClient,
"get_output",
get_fake_step_action_fn(original_step_action_fn),
)
monkeypatch.setattr(
KVCacheManager,
"allocate_slots",
get_fake_allocate_slots_fn(original_allocate_slots),
)
monkeypatch.setattr(MRV2GPUModelRunner, "execute_model", wrapped_execute_model)
monkeypatch.setattr(MRV2GPUModelRunner, "sample", fake_sample)
monkeypatch.setattr(
MambaHybridModelState, "preprocess_state", wrapped_preprocess_state
)
monkeypatch.setattr(
MambaHybridModelState, "postprocess_state", wrapped_postprocess_state
)
engine = LLM(
model=MODEL,
load_format="dummy",
enforce_eager=True,
skip_tokenizer_init=True,
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
},
max_num_batched_tokens=3072,
max_model_len=BLOCK_SIZE * 12,
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
seed=42,
)
try:
tests = get_mamba_prefix_cache_step_configs()
global step_actions
global cur_step_action_idx
global num_accepted_tokens
for test_name, test_config in tests.items():
num_accepted_tokens = test_config.num_accepted_tokens
cur_step_action_idx = 0
fill_following_kv_cache_block_ids(test_config)
step_actions = test_config.step_actions
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=test_config.num_generated_tokens,
ignore_eos=True,
)
_ = engine.generate(
[TokensPrompt(prompt_token_ids=[1] * test_config.num_prompt_tokens)],
sampling_params=sampling_params,
)
assert cur_step_action_idx == len(test_config.step_actions), test_name
assert (
engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
)
step_actions = []
cur_step_action_idx = 0
num_accepted_tokens = 1
prompt = TokensPrompt(prompt_token_ids=[1] * (BLOCK_SIZE * 2))
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=1,
ignore_eos=True,
)
_ = engine.generate([prompt], sampling_params=sampling_params)
first_event_count = len(events)
_ = engine.generate([prompt], sampling_params=sampling_params)
second_events = events[first_event_count:]
prefix_hits = [
num_computed_tokens
for num_computed_tokens in second_events
if num_computed_tokens >= BLOCK_SIZE
]
assert prefix_hits, (
"Expected the second identical prompt to hit prefix cache, "
f"got events={second_events!r}"
)
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
finally:
del engine
torch.accelerator.empty_cache()
cleanup_dist_env_and_memory()
-11
View File
@@ -1997,13 +1997,6 @@ class VllmConfig:
model_config = self.model_config
speculative_config = self.speculative_config
if (
model_config is not None
and model_config.has_inner_state
and self.cache_config.mamba_cache_mode == "align"
):
unsupported.append("hybrid/mamba models with align cache mode")
if self.parallel_config.prefill_context_parallel_size > 1:
unsupported.append("prefill context parallelism")
@@ -2152,10 +2145,6 @@ class VllmConfig:
"to schedule a multiple of block_size tokens even if they are "
"in the middle of a mm input"
)
# TODO: support align mamba cache mode for model runner v2
assert not envs.VLLM_USE_V2_MODEL_RUNNER, (
"Model Runner V2 has not yet supported mamba_cache_mode='align'. "
)
@model_validator(mode="after")
def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig":
@@ -973,7 +973,9 @@ class DiffusionGemmaModelState(ModelState):
# so the captured graph and runtime point to identical addresses.
return {"inputs_embeds": self._inputs_embeds_buf[:num_tokens]}
def postprocess_state(self, idx_mapping, num_sampled) -> None:
def postprocess_state(
self, idx_mapping, num_sampled, num_computed_tokens=None
) -> None:
return None
def prepare_attn(
+13 -1
View File
@@ -1111,7 +1111,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.total_len.gpu,
)
self.model_state.postprocess_state(idx_mapping, num_sampled)
self.model_state.postprocess_state(
idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu
)
@torch.inference_mode()
def execute_model(
@@ -1176,6 +1178,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Prepare all the inputs and copy to the input buffers.
input_batch = self.prepare_inputs(scheduler_output, batch_desc)
block_tables, slot_mappings = self.prepare_attn(input_batch)
# Mamba "align" pre-copy: migrate recurrent state across block
# boundaries before the forward. Runs only on real batches, and
# before model_state.prepare_attn gathers num_accepted_tokens so the
# boundary reset is visible to the attention metadata.
self.model_state.preprocess_state(
input_batch,
block_tables,
self.kv_cache_config,
self.req_states.num_computed_tokens.gpu,
)
if self.lora_config:
# Activate LoRA adapters.
+16 -1
View File
@@ -95,8 +95,23 @@ class ModelState(ABC):
def apply_staged_writes(self) -> None:
return None
def preprocess_state(
self,
input_batch: InputBatch,
block_tables: tuple[torch.Tensor, ...],
kv_cache_config: KVCacheConfig,
num_computed_tokens: torch.Tensor,
) -> None:
"""Hook run on real batches before the forward pass (after block tables
are gathered). Used by mamba "align" prefix caching to pre-copy state
across block boundaries. No-op by default."""
return None
def postprocess_state(
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor
self,
idx_mapping: torch.Tensor,
num_sampled: torch.Tensor,
num_computed_tokens: torch.Tensor | None = None,
) -> None:
return None
+174 -8
View File
@@ -9,15 +9,25 @@ import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.layers.mamba.mamba_utils import (
get_conv_copy_spec,
is_conv_state_dim_first,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.utils import CpuGpuBuffer
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata
from vllm.v1.worker.mamba_utils import (
MambaSpecDecodeGPUContext,
preprocess_mamba_align_fused_kernel,
)
from vllm.v1.worker.utils import AttentionGroup
@@ -65,9 +75,142 @@ class MambaHybridModelState(DefaultModelState):
device: torch.device,
) -> None:
super().__init__(vllm_config, model, encoder_cache, device)
self.cache_config = vllm_config.cache_config
self.num_accepted_tokens_gpu = torch.ones(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
# Pre-copy "align" prefix-cache state (V2). The migration of each
# request's mamba state across block boundaries runs as a fused GPU
# kernel reusing the postprocess copy machinery, so the per-step src
# columns and the running state_idx are kept GPU-resident.
self._align_mode = self.cache_config.mamba_cache_mode == "align"
if self._align_mode:
self._mamba_state_idx_gpu = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
self._mamba_src_col_gpu = torch.full(
(self.max_num_reqs,), -1, dtype=torch.int32, device=self.device
)
self._mamba_src_off_gpu = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
self._mamba_ctx: MambaSpecDecodeGPUContext | None = None
self._mamba_group_ids: list[int] = []
self._mamba_spec: MambaSpec | None = None
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data)
if self._align_mode:
# Seed the running state block from the resumed/prefilled position.
self._mamba_state_idx_gpu[req_index] = (
new_req_data.num_computed_tokens - 1
) // self.cache_config.block_size
self.num_accepted_tokens_gpu[req_index] = 1
def _get_mamba_group_info(
self, kv_cache_config: KVCacheConfig
) -> tuple[list[int], MambaSpec]:
if self._mamba_spec is None:
group_ids: list[int] = []
specs: list[MambaSpec] = []
for i, group in enumerate(kv_cache_config.kv_cache_groups):
spec = group.kv_cache_spec
if isinstance(spec, MambaSpec):
group_ids.append(i)
specs.append(spec)
assert specs, "no mamba layers in the model"
assert all(specs[0] == s for s in specs)
self._mamba_group_ids = group_ids
self._mamba_spec = specs[0]
return self._mamba_group_ids, self._mamba_spec
def _ensure_align_ctx(
self,
kv_cache_config: KVCacheConfig,
mamba_group_ids: list[int],
block_tables: tuple[torch.Tensor, ...],
) -> MambaSpecDecodeGPUContext:
if self._mamba_ctx is None:
copy_funcs = self.model.get_mamba_state_copy_func()
# The fused copy kernels shift conv windows assuming the SD layout;
# the DS layout cannot express a >0 spec-decode shift as a single
# contiguous copy (mirrors get_conv_copy_spec's NotImplementedError).
if get_conv_copy_spec in copy_funcs and is_conv_state_dim_first():
assert self.vllm_config.speculative_config is None, (
"DS conv state layout does not support mamba align state "
"copies with speculative decoding"
)
self._mamba_ctx = MambaSpecDecodeGPUContext.create(
max_num_reqs=self.max_num_reqs,
kv_cache_config=kv_cache_config,
num_state_types=len(copy_funcs),
device=self.device,
make_buffer=lambda n, dtype: CpuGpuBuffer(
n, dtype=dtype, device=self.device
),
)
ctx = self._mamba_ctx
if not ctx.is_initialized:
forward_context = self.vllm_config.compilation_config.static_forward_context
# block_tables are batch-order slices of the persistent
# input_block_tables (stable data_ptr), so the metadata is captured
# once here and reused across steps.
ctx.initialize_from_forward_context(
kv_cache_config,
forward_context,
self.model.get_mamba_state_copy_func(),
[block_tables[gid] for gid in mamba_group_ids],
)
return ctx
def preprocess_state(
self,
input_batch: InputBatch,
block_tables: tuple[torch.Tensor, ...],
kv_cache_config: KVCacheConfig,
num_computed_tokens: torch.Tensor,
) -> None:
"""Migrate each request's mamba state across block boundaries before the
forward (V1 align semantics, done on GPU). Runs on real batches only
(dummy DP/profiling runs skip preprocess_state), and before
``prepare_attn`` gathers ``num_accepted_tokens``, so the boundary reset
is visible to the forward kernels.
"""
if not self._align_mode:
return
num_reqs = input_batch.num_reqs
if num_reqs == 0:
return
mamba_group_ids, mamba_spec = self._get_mamba_group_info(kv_cache_config)
ctx = self._ensure_align_ctx(kv_cache_config, mamba_group_ids, block_tables)
# The state-advance + pre-copy kernels run every step; they fast-exit per
# request when src_col < 0 or src_col == dst_col, so no copy happens on
# steps that don't cross a block boundary. (Skipping the launch entirely
# would need a V1-style async-D2H of the actual num_computed, since
# num_computed_tokens_np is an optimistic mirror under async scheduling;
# the launch cost is ~0.3% of TPOT, so the GPU fast-exit suffices.)
block = 256
grid = (triton.cdiv(num_reqs, block),)
preprocess_mamba_align_fused_kernel[grid](
input_batch.idx_mapping,
self._mamba_state_idx_gpu,
num_computed_tokens,
input_batch.query_start_loc,
self.num_accepted_tokens_gpu,
self._mamba_src_col_gpu,
self._mamba_src_off_gpu,
num_reqs,
BLOCK_SIZE=block,
MAMBA_BLOCK_SIZE=mamba_spec.block_size,
)
ctx.run_fused_precopy(
num_reqs,
self._mamba_state_idx_gpu,
self._mamba_src_col_gpu,
self._mamba_src_off_gpu,
input_batch.idx_mapping,
)
def prepare_attn(
self,
@@ -145,22 +288,45 @@ class MambaHybridModelState(DefaultModelState):
)
def postprocess_state(
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor | int
self,
idx_mapping: torch.Tensor,
num_sampled: torch.Tensor | int,
num_computed_tokens: torch.Tensor | None = None,
) -> None:
# Chunked prefill does not sample a token, so num_sampled can be 0.
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
if not isinstance(num_sampled, int):
# idx_mapping may contain -1 sentinels (filtered rows) under PP; the
# kernel skips them rather than scattering with a host-side gather.
num_reqs = idx_mapping.shape[0]
if num_reqs:
_scatter_num_accepted_kernel[(num_reqs,)](
n = idx_mapping.shape[0]
if n:
_scatter_num_accepted_kernel[(n,)](
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
)
return
else:
# Fill with single value.
self.num_accepted_tokens_gpu.index_fill_(
0, idx_mapping, max(num_sampled, 1)
)
# Fill with single value.
self.num_accepted_tokens_gpu.index_fill_(0, idx_mapping, max(num_sampled, 1))
# Align: save the running state to the block-aligned position when
# spec-decode acceptance leaves the sequence non-block-aligned (mirrors
# the V1 align postprocess). num_computed_tokens already holds the
# post-step advanced count.
if (
self._align_mode
and num_computed_tokens is not None
and self._mamba_ctx is not None
):
num_reqs = idx_mapping.shape[0]
if num_reqs:
self._mamba_ctx.run_fused_postprocess_align(
num_reqs,
self.num_accepted_tokens_gpu,
self._mamba_state_idx_gpu,
num_computed_tokens,
idx_mapping,
)
@triton.jit
+12 -3
View File
@@ -17,6 +17,7 @@ from vllm.v1.core.sched.output import (
NewRequestData,
SchedulerOutput,
)
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.request import Request
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
@@ -177,9 +178,17 @@ def warmup_kernels(
num_kv_cache_groups = len(kv_cache_groups)
# Compute per-request block counts for each KV cache group.
group_block_sizes = [g.kv_cache_spec.block_size for g in kv_cache_groups]
prefill_block_counts = [cdiv(prompt_len, bs) for bs in group_block_sizes]
decode_block_counts = [cdiv(decode_len, bs) for bs in group_block_sizes]
def _warmup_block_count(num_tokens: int, spec: Any) -> int:
num_blocks = cdiv(num_tokens, spec.block_size)
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode == "align":
# Align mode reserves extra blocks beyond the token range for the
# speculative-decode running-state snapshots.
num_blocks += spec.num_speculative_blocks
return num_blocks
kv_cache_specs = [g.kv_cache_spec for g in kv_cache_groups]
prefill_block_counts = [_warmup_block_count(prompt_len, s) for s in kv_cache_specs]
decode_block_counts = [_warmup_block_count(decode_len, s) for s in kv_cache_specs]
decode_block_deltas = [
d - p for d, p in zip(decode_block_counts, prefill_block_counts)
]
+368 -92
View File
@@ -23,6 +23,112 @@ from vllm.v1.worker.gpu_input_batch import CachedRequestState
from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch
@triton.jit
def _copy_mamba_state_block(
state_idx,
bt_row_idx,
src_col,
dst_col,
token_bias,
block_table_ptrs_ptr,
block_table_stride_req,
state_base_addrs_ptr,
state_block_strides_ptr,
state_elem_sizes_ptr,
state_inner_sizes_ptr,
state_conv_widths_ptr,
state_group_indices_ptr,
# DS conv row metadata. Zero keeps the single-region copy path.
state_dim_row_count_ptr,
state_dim_row_stride_ptr,
COPY_BLOCK_SIZE: tl.constexpr,
CONV_STATE_DIM_FIRST: tl.constexpr,
):
"""Copy one (layer, state-type) mamba state block between block columns.
Shared copy body of ``postprocess_mamba_fused_kernel`` and
``precopy_mamba_align_fused_kernel``, mirroring the V1 copy specs
(``get_conv_copy_spec`` / ``get_temporal_copy_spec``):
- conv state (conv_width > 0): shift the window by ``token_bias`` tokens,
``state[bt[src_col], token_bias:] ->
state[bt[dst_col], :conv_width - token_bias]``
- temporal state: ``token_bias`` selects the accepted speculative column,
``state[bt[src_col + token_bias]] -> state[bt[dst_col]]``
The caller owns the decision logic (which columns, whether to copy); this
device function only performs the byte copy for the given metadata slot.
"""
state_base_addr = tl.load(state_base_addrs_ptr + state_idx)
state_block_stride = tl.load(state_block_strides_ptr + state_idx)
state_elem_size = tl.load(state_elem_sizes_ptr + state_idx)
state_inner_size = tl.load(state_inner_sizes_ptr + state_idx)
conv_width = tl.load(state_conv_widths_ptr + state_idx)
# Load the group index for this state, then index into the correct
# group's block table. Each mamba group has independently allocated
# physical blocks. Reinterpret as int32* since block ids are int32.
group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64)
group_base_addr = tl.load(block_table_ptrs_ptr + group_idx)
block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32))
block_table_base = block_table_typed + bt_row_idx * block_table_stride_req
# Widen block ids to int64 before they reach `block_id * state_block_stride`
# below: state_block_stride can exceed 2**31 bytes for large mamba caches,
# and Triton would otherwise do the multiply in int32 and wrap.
dest_block_id = tl.load(block_table_base + dst_col).to(tl.int64)
dst_addr = state_base_addr + dest_block_id * state_block_stride
is_conv_state = conv_width > 0
if CONV_STATE_DIM_FIRST and is_conv_state:
# DS conv layout: state_len is the slide axis; copy per dim row.
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
row_stride = tl.load(state_dim_row_stride_ptr + state_idx)
per_row_bytes = (conv_width - token_bias).to(tl.int64) * state_elem_size
bias_bytes = token_bias.to(tl.int64) * state_elem_size
src_block_addr = state_base_addr + src_block_id * state_block_stride
offsets = tl.arange(0, COPY_BLOCK_SIZE)
for d in range(0, dim_rows):
row_src = src_block_addr + d * row_stride + bias_bytes
row_dst = dst_addr + d * row_stride
for i in range(0, per_row_bytes, COPY_BLOCK_SIZE):
mask = (i + offsets) < per_row_bytes
curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8))
curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8))
data = tl.load(curr_src, mask=mask)
tl.store(curr_dst, data, mask=mask)
return
if is_conv_state:
# SD conv: copy
# state[bt[src_col], token_bias:] ->
# state[bt[dst_col], :conv_width - token_bias]
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
src_offset = token_bias.to(tl.int64) * state_inner_size * state_elem_size
src_addr = state_base_addr + src_block_id * state_block_stride + src_offset
num_elems_to_copy = (conv_width - token_bias).to(tl.int64) * state_inner_size
copy_size = num_elems_to_copy * state_elem_size
else:
# Temporal state: copy state[bt[src_col + token_bias]] -> state[bt[dst_col]]
actual_src_block_id = tl.load(block_table_base + src_col + token_bias).to(
tl.int64
)
src_addr = state_base_addr + actual_src_block_id * state_block_stride
# Use natural block data size (inner_size * elem_size), NOT
# state_block_stride which is the page stride and can exceed the
# actual data when the state tensor uses as_strided page padding.
copy_size = state_inner_size * state_elem_size
offsets = tl.arange(0, COPY_BLOCK_SIZE)
for i in range(0, copy_size, COPY_BLOCK_SIZE):
mask = (i + offsets) < copy_size
curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8))
curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8))
data = tl.load(curr_src, mask=mask)
tl.store(curr_dst, data, mask=mask)
@triton.jit
def postprocess_mamba_fused_kernel(
# Decision inputs (per-request)
@@ -49,6 +155,10 @@ def postprocess_mamba_fused_kernel(
state_dim_row_stride_ptr, # int64: bytes between rows for DS conv
# Output: num_accepted_tokens update (for src==dst case)
num_accepted_tokens_out_ptr,
# Optional: batch_idx -> req_idx mapping (V2 model runner / PP). The
# per-request decision arrays are in req-state-slot order; the block table
# is in batch order, so HAS_IDX_MAPPING splits the two indexings.
idx_mapping_ptr,
# Runtime parameter (varies per batch - NOT constexpr to avoid recompilation)
num_reqs,
# Compile-time constants (fixed after model initialization)
@@ -57,35 +167,53 @@ def postprocess_mamba_fused_kernel(
# COPY_BLOCK_SIZE: fixed tuning parameter for memory copy loop
COPY_BLOCK_SIZE: tl.constexpr,
CONV_STATE_DIM_FIRST: tl.constexpr,
# HAS_IDX_MAPPING: when True, program_id(0) is a batch index resolved to a
# req-state slot via idx_mapping_ptr (V2). When False, it is the req index.
HAS_IDX_MAPPING: tl.constexpr = False,
# PRECOMPUTED_NEW_COMPUTED: when True, num_computed_tokens_ptr already holds
# the post-step new_num_computed value (V2 supplies the advanced count).
PRECOMPUTED_NEW_COMPUTED: tl.constexpr = False,
):
"""
Fused GPU kernel for postprocess_mamba that computes decisions AND performs
mamba state copies without any CPU-GPU synchronization.
Grid: (num_reqs, num_layers * num_state_types)
- program_id(0) = request index
- program_id(0) = request/batch index
- program_id(1) = state_idx (flattened index into layer/state_type metadata)
Note: num_layers and num_state_types are not passed as kernel parameters
because the kernel indexes directly into pre-flattened metadata arrays
using program_id(1). The grid dimensions encode the total state count.
"""
req_idx = tl.program_id(0)
batch_idx = tl.program_id(0)
state_idx = tl.program_id(1)
# Bounds check
if req_idx >= num_reqs:
if batch_idx >= num_reqs:
return
if HAS_IDX_MAPPING:
req_idx = tl.load(idx_mapping_ptr + batch_idx)
if req_idx < 0:
return
else:
req_idx = batch_idx
# Compute decision logic (mirrors postprocess_mamba Python reference)
num_accepted = tl.load(num_accepted_tokens_ptr + req_idx)
src_block_idx = tl.load(mamba_state_idx_ptr + req_idx)
num_scheduled = tl.load(num_scheduled_tokens_ptr + req_idx)
num_computed = tl.load(num_computed_tokens_ptr + req_idx)
num_draft = tl.load(num_draft_tokens_ptr + req_idx)
num_tokens_running_state = num_computed + num_scheduled - num_draft
new_num_computed = num_tokens_running_state + num_accepted - 1
if PRECOMPUTED_NEW_COMPUTED:
new_num_computed = tl.load(num_computed_tokens_ptr + req_idx)
num_tokens_running_state = new_num_computed - num_accepted + 1
else:
num_scheduled = tl.load(num_scheduled_tokens_ptr + req_idx)
num_computed = tl.load(num_computed_tokens_ptr + req_idx)
num_draft = tl.load(num_draft_tokens_ptr + req_idx)
num_tokens_running_state = num_computed + num_scheduled - num_draft
new_num_computed = num_tokens_running_state + num_accepted - 1
aligned_new_computed = (new_num_computed // block_size) * block_size
needs_copy = aligned_new_computed >= num_tokens_running_state
@@ -97,99 +225,158 @@ def postprocess_mamba_fused_kernel(
accept_token_bias = aligned_new_computed - num_tokens_running_state
dest_block_idx = aligned_new_computed // block_size - 1
# Load state metadata for this layer/state_type
state_base_addr = tl.load(state_base_addrs_ptr + state_idx)
state_block_stride = tl.load(state_block_strides_ptr + state_idx)
state_elem_size = tl.load(state_elem_sizes_ptr + state_idx)
state_inner_size = tl.load(state_inner_sizes_ptr + state_idx)
conv_width = tl.load(state_conv_widths_ptr + state_idx)
# Load the group index for this state, then index into the correct
# group's block table. Each mamba group has independently allocated
# physical blocks.
group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64)
# block_table_ptrs_ptr holds one pointer per group (each group owns its own
# block table). Reinterpret as int32* since block ids are int32.
group_base_addr = tl.load(block_table_ptrs_ptr + group_idx)
block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32))
block_table_base = block_table_typed + req_idx * block_table_stride_req
# Widen block ids to int64 before they reach `block_id * state_block_stride`
# below: state_block_stride can exceed 2**31 bytes for large mamba caches,
# and Triton would otherwise do the multiply in int32 and wrap.
src_block_id = tl.load(block_table_base + src_block_idx).to(tl.int64)
dest_block_id = tl.load(block_table_base + dest_block_idx).to(tl.int64)
# Compute source and destination addresses based on state type
# conv_width > 0 means this is a conv state (get_conv_copy_spec logic)
# conv_width == 0 means this is a temporal state (get_temporal_copy_spec logic)
is_conv_state = conv_width > 0
# Update accepted-token count before early exits.
# Update accepted-token count before early exits (per-request, so only
# state_idx == 0 writes). V2 updates in place; V1 writes the _out buffer.
if src_block_idx == dest_block_idx and state_idx == 0:
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
if HAS_IDX_MAPPING:
tl.store(num_accepted_tokens_ptr + req_idx, 1)
else:
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
# Skip no-op self-copy.
if src_block_idx == dest_block_idx and accept_token_bias == 0:
return
if CONV_STATE_DIM_FIRST and is_conv_state:
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
row_stride = tl.load(state_dim_row_stride_ptr + state_idx)
per_row_bytes = (conv_width - accept_token_bias).to(tl.int64) * state_elem_size
bias_bytes = accept_token_bias.to(tl.int64) * state_elem_size
src_block_addr = state_base_addr + src_block_id * state_block_stride
dst_block_addr = state_base_addr + dest_block_id * state_block_stride
offsets = tl.arange(0, COPY_BLOCK_SIZE)
for d in range(0, dim_rows):
row_src = src_block_addr + d * row_stride + bias_bytes
row_dst = dst_block_addr + d * row_stride
for i in range(0, per_row_bytes, COPY_BLOCK_SIZE):
mask = (i + offsets) < per_row_bytes
curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8))
curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8))
data = tl.load(curr_src, mask=mask)
tl.store(curr_dst, data, mask=mask)
bt_row_idx = batch_idx if HAS_IDX_MAPPING else req_idx
_copy_mamba_state_block(
state_idx,
bt_row_idx,
src_block_idx,
dest_block_idx,
accept_token_bias,
block_table_ptrs_ptr,
block_table_stride_req,
state_base_addrs_ptr,
state_block_strides_ptr,
state_elem_sizes_ptr,
state_inner_sizes_ptr,
state_conv_widths_ptr,
state_group_indices_ptr,
state_dim_row_count_ptr,
state_dim_row_stride_ptr,
COPY_BLOCK_SIZE,
CONV_STATE_DIM_FIRST,
)
@triton.jit
def preprocess_mamba_align_fused_kernel(
idx_mapping_ptr,
state_idx_ptr,
num_computed_tokens_ptr,
query_start_loc_ptr,
num_accepted_tokens_ptr,
src_col_ptr,
src_off_ptr,
num_reqs,
BLOCK_SIZE: tl.constexpr,
MAMBA_BLOCK_SIZE: tl.constexpr,
):
"""Fused align preprocess: emit the pre-copy src column/offset AND advance
state_idx (with accepted-token reset) in a single launch (V2 align).
Per batch_idx (0..num_reqs-1), resolving req slot via idx_mapping:
1. Read pre-advance state_idx and num_accepted (last step's values).
2. Store the pre-copy src columns for ``precopy_mamba_align_fused_kernel``:
- src_col = state_idx (the previous running block column)
- src_off = max(num_accepted - 1, 0) (the accepted-token bias)
3. Advance state_idx to the new running block, and reset num_accepted to 1
when a block boundary is crossed (so the migrated state, now at the
start of the new block, is read with the neutral bias).
"""
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < num_reqs
req_indices = tl.load(idx_mapping_ptr + offsets, mask=mask, other=0)
state_idx = tl.load(state_idx_ptr + req_indices, mask=mask, other=-1)
num_accepted = tl.load(num_accepted_tokens_ptr + req_indices, mask=mask, other=1)
src_off = tl.maximum(num_accepted - 1, 0)
tl.store(src_col_ptr + req_indices, state_idx, mask=mask)
tl.store(src_off_ptr + req_indices, src_off, mask=mask)
num_computed = tl.load(num_computed_tokens_ptr + req_indices, mask=mask, other=0)
query_start = tl.load(query_start_loc_ptr + offsets, mask=mask, other=0)
query_end = tl.load(query_start_loc_ptr + offsets + 1, mask=mask, other=0)
computed_after = num_computed + query_end - query_start
new_state_idx = (computed_after + MAMBA_BLOCK_SIZE - 1) // MAMBA_BLOCK_SIZE - 1
tl.store(state_idx_ptr + req_indices, new_state_idx, mask=mask)
should_reset = (state_idx >= 0) & (state_idx != new_state_idx)
tl.store(num_accepted_tokens_ptr + req_indices, 1, mask=mask & should_reset)
@triton.jit
def precopy_mamba_align_fused_kernel(
# Per-request-slot inputs (indexed by req_idx via idx_mapping), produced by
# the V2 fused align preprocess kernel for the current step:
mamba_state_idx_ptr, # post-advance dst block column
src_col_ptr, # pre-advance src block column (-1 = fresh)
token_bias_ptr, # accepted-token bias = num_accepted - 1 (pre-reset)
# Same flattened state-layout metadata as postprocess_mamba_fused_kernel
block_table_ptrs_ptr,
block_table_stride_req: tl.int64,
state_base_addrs_ptr,
state_block_strides_ptr,
state_elem_sizes_ptr,
state_inner_sizes_ptr,
state_conv_widths_ptr,
state_group_indices_ptr,
state_dim_row_count_ptr,
state_dim_row_stride_ptr,
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip)
num_reqs,
COPY_BLOCK_SIZE: tl.constexpr,
CONV_STATE_DIM_FIRST: tl.constexpr,
):
"""Pre-copy mamba "align" state across block boundaries on the V2 runner.
Before the forward pass, copy each request's last SSM/conv state from its
previous block column into the new window block column, so the kernels read
the initial state from the write-side block as usual (V1 align semantics).
Same per-(layer, state) copy semantics as ``postprocess_mamba_fused_kernel``
(shared ``_copy_mamba_state_block`` body, i.e. the V1 ``preprocess_mamba``
copy specs), but driven by the GPU-resident src columns so it needs no
CPU-GPU sync (async-scheduling safe).
Grid: (num_reqs, num_layers * num_state_types); block tables are indexed by
batch row, per-request state by req_idx via idx_mapping (V2 layout).
"""
batch_idx = tl.program_id(0)
state_idx = tl.program_id(1)
if batch_idx >= num_reqs:
return
req_idx = tl.load(idx_mapping_ptr + batch_idx)
if req_idx < 0:
return
if is_conv_state:
# SD conv: copy
# state[block_table[req_idx, src_block_idx], accept_token_bias:]
# to
# state[block_table[req_idx, dest_block_idx], :conv_width - accept_token_bias]
src_offset = accept_token_bias.to(tl.int64) * state_inner_size * state_elem_size
src_addr = state_base_addr + src_block_id * state_block_stride + src_offset
dst_addr = state_base_addr + dest_block_id * state_block_stride
# Number of elements to copy:
# (conv_width - accept_token_bias) * inner_size
num_elems_to_copy = (conv_width - accept_token_bias).to(
tl.int64
) * state_inner_size
copy_size = num_elems_to_copy * state_elem_size
else:
# Temporal state: copy
# state[block_table[req_idx, src_block_idx + accept_token_bias]]
# to
# state[block_table[req_idx, dest_block_idx]]
actual_src_block_idx = src_block_idx + accept_token_bias
actual_src_block_id = tl.load(block_table_base + actual_src_block_idx).to(
tl.int64
)
src_addr = state_base_addr + actual_src_block_id * state_block_stride
dst_addr = state_base_addr + dest_block_id * state_block_stride
# Use natural block data size (inner_size * elem_size), NOT
# state_block_stride which is the page stride and can exceed the
# actual data when the state tensor uses as_strided page padding.
copy_size = state_inner_size * state_elem_size
src_col = tl.load(src_col_ptr + req_idx)
dst_col = tl.load(mamba_state_idx_ptr + req_idx)
# Fresh state, or still writing the same block: kernels locate the initial
# state in-block via num_accepted (preserved when no boundary is crossed),
# so there is nothing to copy.
if src_col < 0 or src_col == dst_col:
return
offsets = tl.arange(0, COPY_BLOCK_SIZE)
for i in range(0, copy_size, COPY_BLOCK_SIZE):
mask = (i + offsets) < copy_size
curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8))
curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8))
data = tl.load(curr_src, mask=mask)
tl.store(curr_dst, data, mask=mask)
token_bias = tl.load(token_bias_ptr + req_idx)
_copy_mamba_state_block(
state_idx,
batch_idx,
src_col,
dst_col,
token_bias,
block_table_ptrs_ptr,
block_table_stride_req,
state_base_addrs_ptr,
state_block_strides_ptr,
state_elem_sizes_ptr,
state_inner_sizes_ptr,
state_conv_widths_ptr,
state_group_indices_ptr,
state_dim_row_count_ptr,
state_dim_row_stride_ptr,
COPY_BLOCK_SIZE,
CONV_STATE_DIM_FIRST,
)
@triton.jit
@@ -559,12 +746,101 @@ class MambaSpecDecodeGPUContext:
self.state_dim_row_count,
self.state_dim_row_stride,
self.num_accepted_tokens_out,
None, # idx_mapping: V1 decision arrays are already in req order
num_reqs,
block_size=self.block_size,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
)
def run_fused_precopy(
self,
num_reqs: int,
state_idx_gpu: torch.Tensor,
src_col_gpu: torch.Tensor,
token_bias_gpu: torch.Tensor,
idx_mapping: torch.Tensor,
) -> None:
"""Pre-copy each request's previous running block into its new window
block before the forward pass (V2 align boundary migration).
Args:
num_reqs: Number of active requests (batch order).
state_idx_gpu: [max_reqs] post-advance dst block column per req slot.
src_col_gpu: [max_reqs] pre-advance src block column (-1 = fresh).
token_bias_gpu: [max_reqs] accepted-token bias (num_accepted - 1).
idx_mapping: [num_reqs] batch_idx -> req_state_idx (-1 to skip).
"""
if num_reqs == 0 or not self.is_initialized:
return
total_states = self.num_layers * self.num_state_types
grid = (num_reqs, total_states)
precopy_mamba_align_fused_kernel[grid](
state_idx_gpu,
src_col_gpu,
token_bias_gpu,
self.block_table_ptrs,
self.block_table_stride_req,
self.state_base_addrs,
self.state_block_strides,
self.state_elem_sizes,
self.state_inner_sizes,
self.state_conv_widths,
self.state_group_indices,
self.state_dim_row_count,
self.state_dim_row_stride,
idx_mapping,
num_reqs,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
)
def run_fused_postprocess_align(
self,
num_reqs: int,
num_accepted_tokens_gpu: torch.Tensor,
state_idx_gpu: torch.Tensor,
new_num_computed_tokens_gpu: torch.Tensor,
idx_mapping: torch.Tensor,
) -> None:
"""V2 align postprocess: save the running state to the block-aligned
position after spec-decode acceptance leaves the sequence non-aligned.
``num_accepted_tokens_gpu`` is updated in place (reset to 1 when the
accepted position stays in the running block); ``new_num_computed_tokens``
already holds the post-step computed count (PRECOMPUTED_NEW_COMPUTED).
``idx_mapping`` maps batch row -> req-state slot (HAS_IDX_MAPPING).
"""
if num_reqs == 0 or not self.is_initialized:
return
total_states = self.num_layers * self.num_state_types
grid = (num_reqs, total_states)
postprocess_mamba_fused_kernel[grid](
num_accepted_tokens_gpu,
state_idx_gpu,
None, # num_scheduled: unused under PRECOMPUTED_NEW_COMPUTED
new_num_computed_tokens_gpu,
None, # num_draft: unused under PRECOMPUTED_NEW_COMPUTED
self.block_table_ptrs,
self.block_table_stride_req,
self.state_base_addrs,
self.state_block_strides,
self.state_elem_sizes,
self.state_inner_sizes,
self.state_conv_widths,
self.state_group_indices,
self.state_dim_row_count,
self.state_dim_row_stride,
None, # num_accepted_out: V2 updates num_accepted in place
idx_mapping,
num_reqs,
block_size=self.block_size,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
HAS_IDX_MAPPING=True,
PRECOMPUTED_NEW_COMPUTED=True,
)
@dataclasses.dataclass
class MambaBuffers: