diff --git a/tests/kernels/mamba/test_precopy_mamba_align.py b/tests/kernels/mamba/test_precopy_mamba_align.py new file mode 100644 index 0000000000..be1e455948 --- /dev/null +++ b/tests/kernels/mamba/test_precopy_mamba_align.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Equivalence test for ``precopy_mamba_align_fused_kernel``. + +The V2 "align" pre-copy must migrate mamba state across block boundaries with +byte-identical semantics to the V1 copy specs (``get_conv_copy_spec`` / +``get_temporal_copy_spec``): + +* conv state (SD layout, conv_width > 0): shift the sliding window by + ``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` -> + ``state[bt[dst_col], :conv_width - token_bias]``. +* temporal state (conv_width == 0): ``token_bias`` selects the accepted + speculative column -- ``state[bt[src_col + token_bias]]`` -> + ``state[bt[dst_col]]``. + +The kernel must also no-op when ``src_col < 0`` (fresh request) or +``src_col == dst_col`` (no boundary crossed). +""" + +from __future__ import annotations + +import torch + +from vllm.platforms import current_platform +from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel + +try: + import pytest + + pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), + reason="precopy_mamba_align_fused_kernel needs CUDA/Triton", + ) + _parametrize = pytest.mark.parametrize +except ModuleNotFoundError: # allow running directly as ``python `` + pytest = None + + def _parametrize(_name, _values): + def _deco(fn): + return fn + + return _deco + + +NUM_LAYERS = 3 +CONV_WIDTH = 4 # conv_kernel - 1 + num_spec +CONV_DIM = 96 +SSM_SHAPE = (4, 16, 16) +MAX_COLS = 8 + + +def _build_state(num_blocks, device): + """Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools.""" + convs, ssms = [], [] + for _ in range(NUM_LAYERS): + convs.append( + torch.randn( + num_blocks, CONV_WIDTH, CONV_DIM, dtype=torch.bfloat16, device=device + ) + ) + ssms.append( + torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device) + ) + return convs, ssms + + +def _build_meta(convs, ssms, device): + """Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer.""" + n = NUM_LAYERS * 2 + base = torch.zeros(n, dtype=torch.int64, device=device) + blk_stride = torch.zeros(n, dtype=torch.int64, device=device) + elem = torch.zeros(n, dtype=torch.int32, device=device) + inner = torch.zeros(n, dtype=torch.int64, device=device) + width = torch.zeros(n, dtype=torch.int32, device=device) + group = torch.zeros(n, dtype=torch.int32, device=device) + drc = torch.zeros(n, dtype=torch.int32, device=device) # DS rows (unused, SD) + drs = torch.zeros(n, dtype=torch.int64, device=device) + i = 0 + for layer in range(NUM_LAYERS): + conv, ssm = convs[layer], ssms[layer] + # conv (SD): width = size(1), inner = stride(1) + base[i] = conv.data_ptr() + blk_stride[i] = conv.stride(0) * conv.element_size() + elem[i] = conv.element_size() + width[i] = conv.size(1) + inner[i] = conv.stride(1) + i += 1 + # ssm (temporal): width = 0, inner = elems per block + base[i] = ssm.data_ptr() + blk_stride[i] = ssm.stride(0) * ssm.element_size() + elem[i] = ssm.element_size() + width[i] = 0 + inner[i] = ssm[0].numel() + i += 1 + return base, blk_stride, elem, inner, width, group, drc, drs + + +def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs): + """Apply the V1 copy semantics on clones, reading from the pre-copy state.""" + conv_pre = [c.clone() for c in convs] + ssm_pre = [s.clone() for s in ssms] + conv_ref = [c.clone() for c in convs] + ssm_ref = [s.clone() for s in ssms] + for r in range(num_reqs): + sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r]) + if sc < 0 or sc == dc: + continue + sblk, dblk = int(bt[r, sc]), int(bt[r, dc]) + tblk = int(bt[r, sc + tb]) # temporal src column shifted by bias + for layer in range(NUM_LAYERS): + conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:] + ssm_ref[layer][dblk] = ssm_pre[layer][tblk] + return conv_ref, ssm_ref + + +@_parametrize("num_reqs", [1, 4, 16]) +@_parametrize("token_bias", [0, 1, 2]) +def test_precopy_matches_v1_copy_specs(num_reqs, token_bias): + device = torch.device("cuda") + torch.manual_seed(0) + # Distinct physical block per (req, col) so copies never alias. + num_blocks = num_reqs * MAX_COLS + 1 + bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device) + for r in range(num_reqs): + bt[r] = torch.arange( + 1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device + ) + + # Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip), + # the rest cross from col 1 -> col 0 with the given spec token bias. + src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device) + dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device) + bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device) + if num_reqs >= 1: + src_col[0] = -1 # fresh -> no copy + if num_reqs >= 2: + dst_col[1] = 1 # src_col == dst_col -> no copy + + convs, ssms = _build_state(num_blocks, device) + conv_ref, ssm_ref = _reference( + convs, ssms, bt.cpu(), src_col.cpu(), dst_col.cpu(), bias.cpu(), num_reqs + ) + + base, blk_stride, elem, inner, width, group, drc, drs = _build_meta( + convs, ssms, device + ) + bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device) + idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device) + grid = (num_reqs, NUM_LAYERS * 2) + precopy_mamba_align_fused_kernel[grid]( + dst_col, + src_col, + bias, + bt_ptrs, + bt.stride(0), + base, + blk_stride, + elem, + inner, + width, + group, + drc, + drs, + idx_mapping, + num_reqs, + COPY_BLOCK_SIZE=1024, + CONV_STATE_DIM_FIRST=False, + ) + torch.accelerator.synchronize() + + for layer in range(NUM_LAYERS): + torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0) + torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0) + + +if __name__ == "__main__": + for nr in (1, 4, 16): + for tb in (0, 1, 2): + test_precopy_matches_v1_copy_specs(nr, tb) + print(f"OK num_reqs={nr} token_bias={tb}") diff --git a/tests/v1/e2e/general/test_mamba_prefix_cache.py b/tests/v1/e2e/general/test_mamba_prefix_cache.py index e857b12728..4644a6cc7e 100644 --- a/tests/v1/e2e/general/test_mamba_prefix_cache.py +++ b/tests/v1/e2e/general/test_mamba_prefix_cache.py @@ -11,6 +11,7 @@ import datasets import pytest import torch +import vllm.envs as envs from tests.utils import create_new_process_for_each_test from vllm import LLM, SamplingParams, TokensPrompt from vllm.config import CacheConfig @@ -494,12 +495,7 @@ def apply_patch(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(mamba_utils, "do_mamba_copy_block", fake_copy_fn) -@create_new_process_for_each_test() -def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch): - run_ref_mamba_state_in_subprocess() - apply_patch(monkeypatch) - prompt_dataset = datasets.load_dataset("heheda/a_long_article") - full_prompt = prompt_dataset["train"][0]["text"] +def get_mamba_prefix_cache_step_configs() -> dict[str, TestConfig]: tests = { "accept_1": TestConfig( num_prompt_tokens=554, @@ -731,6 +727,27 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch): ), } + return tests + + +def fill_following_kv_cache_block_ids(test_config: TestConfig) -> None: + for step_action_prev, step_action_next in zip( + test_config.step_actions[:-1], test_config.step_actions[1:] + ): + if len(step_action_next.kv_cache_block_ids) == 0: + step_action_next.kv_cache_block_ids = ( + step_action_prev.kv_cache_block_ids.copy() + ) + + +@create_new_process_for_each_test() +def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch): + run_ref_mamba_state_in_subprocess() + apply_patch(monkeypatch) + prompt_dataset = datasets.load_dataset("heheda/a_long_article") + full_prompt = prompt_dataset["train"][0]["text"] + tests = get_mamba_prefix_cache_step_configs() + engine = LLM( model=MODEL, enable_prefix_caching=True, @@ -758,16 +775,7 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch): ) global cur_step_action_idx cur_step_action_idx = 0 - for step_action_prev, step_action_next in zip( - test_config.step_actions[:-1], test_config.step_actions[1:] - ): - if ( - step_action_next.kv_cache_block_ids is not None - and len(step_action_next.kv_cache_block_ids) == 0 - ): - prev_block_ids = step_action_prev.kv_cache_block_ids - if prev_block_ids is not None: - step_action_next.kv_cache_block_ids = prev_block_ids.copy() + fill_following_kv_cache_block_ids(test_config) global step_actions step_actions = test_config.step_actions _ = engine.generate( @@ -787,3 +795,259 @@ def test_mamba_prefix_cache(monkeypatch: pytest.MonkeyPatch): del engine torch.accelerator.empty_cache() cleanup_dist_env_and_memory() + + +@create_new_process_for_each_test() +def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + envs.disable_envs_cache() + + from vllm.v1.worker.gpu.model_runner import GPUModelRunner as MRV2GPUModelRunner + from vllm.v1.worker.gpu.model_states.mamba_hybrid import ( + MambaHybridModelState, + ) + from vllm.v1.worker.gpu.sample.output import SamplerOutput as MRV2SamplerOutput + + events: list[int] = [] + original_execute_model = MRV2GPUModelRunner.execute_model + original_sample = MRV2GPUModelRunner.sample + original_preprocess_state = MambaHybridModelState.preprocess_state + original_postprocess_state = MambaHybridModelState.postprocess_state + original_step_action_fn = InprocClient.get_output + original_allocate_slots = KVCacheManager.allocate_slots + captured: dict[str, Any] = {} + + def temporal_states(model_state, block_tables, kv_cache_config): + # Qwen3-Next keeps the temporal (ssm) state as the last Mamba cache. + forward_context = ( + model_state.vllm_config.compilation_config.static_forward_context + ) + group_ids, _ = get_mamba_groups(kv_cache_config) + for group_id in group_ids: + block_table = block_tables[group_id] + for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names: + yield forward_context[layer_name].kv_cache[-1], block_table + + def temporal_block(temporal_state, block_table, col): + return temporal_state[int(block_table[0, col].item())] + + def wrapped_preprocess_state( + self: MambaHybridModelState, + input_batch: Any, + block_tables: tuple[torch.Tensor, ...], + kv_cache_config: KVCacheConfig, + num_computed_tokens: torch.Tensor, + ) -> None: + captured["block_tables"] = block_tables + captured["kv_cache_config"] = kv_cache_config + expected = ( + None if cur_step_action is None else cur_step_action.preprocess_copy_idx + ) + snapshots = [] + if expected is not None and expected != (-1, -1): + for temporal, bt in temporal_states(self, block_tables, kv_cache_config): + snapshots.append( + (temporal, bt, temporal_block(temporal, bt, expected[0]).clone()) + ) + ret = original_preprocess_state( + self, input_batch, block_tables, kv_cache_config, num_computed_tokens + ) + if cur_step_action is not None: + req_idx = int(input_batch.idx_mapping[0].item()) + src_col = int(self._mamba_src_col_gpu[req_idx].item()) + off = int(self._mamba_src_off_gpu[req_idx].item()) + dst = int(self._mamba_state_idx_gpu[req_idx].item()) + actual = (-1, -1) if src_col < 0 or src_col == dst else (src_col + off, dst) + assert actual == expected, ( + f"V2 align preprocess copy: expected={expected}, " + f"actual={actual}, {cur_step_action=}" + ) + for temporal, bt, src_state in snapshots: + torch.testing.assert_close( + temporal_block(temporal, bt, expected[1]), src_state + ) + return ret + + def wrapped_postprocess_state( + self: MambaHybridModelState, + idx_mapping: torch.Tensor, + num_sampled: torch.Tensor | int, + num_computed_tokens: torch.Tensor | None = None, + ) -> None: + action = cur_step_action + block_tables = captured.get("block_tables") + kv_cache_config = captured.get("kv_cache_config") + # The postprocess kernel does not expose its indices, so only the copy + # case is checked, by effect: snapshot the src block, expect dst == src. + if ( + action is None + or num_computed_tokens is None + or block_tables is None + or action.postprocess_copy_idx == (-1, -1) + ): + return original_postprocess_state( + self, idx_mapping, num_sampled, num_computed_tokens + ) + expected = action.postprocess_copy_idx + snapshots = [ + (temporal, bt, temporal_block(temporal, bt, expected[0]).clone()) + for temporal, bt in temporal_states(self, block_tables, kv_cache_config) + ] + ret = original_postprocess_state( + self, idx_mapping, num_sampled, num_computed_tokens + ) + for temporal, bt, src_state in snapshots: + torch.testing.assert_close( + temporal_block(temporal, bt, expected[1]), src_state + ) + return ret + + def wrapped_execute_model( + self: MRV2GPUModelRunner, + scheduler_output: SchedulerOutput, + *args: Any, + **kwargs: Any, + ): + events.extend( + req.num_computed_tokens for req in scheduler_output.scheduled_new_reqs + ) + events.extend(scheduler_output.scheduled_cached_reqs.num_computed_tokens) + if cur_step_action is not None: + num_scheduled_tokens = next( + iter(scheduler_output.num_scheduled_tokens.values()) + ) + assert num_scheduled_tokens == cur_step_action.num_scheduled_tokens + ret = original_execute_model(self, scheduler_output, *args, **kwargs) + if cur_step_action is not None and self.execute_model_state is not None: + input_batch = self.execute_model_state.input_batch + assert ( + cur_step_action.num_computed_tokens_start + == input_batch.positions[input_batch.query_start_loc[0]].item() + ) + return ret + + def fake_sample( + self: MRV2GPUModelRunner, + hidden_states: torch.Tensor, + input_batch: Any, + grammar_output: Any, + ): + if cur_step_action is None: + return original_sample(self, hidden_states, input_batch, grammar_output) + + num_reqs = input_batch.num_reqs + sampled_token_ids = torch.ones( + (num_reqs, self.num_speculative_steps + 1), + device=hidden_states.device, + dtype=torch.int64, + ) + num_logits = torch.tensor( + input_batch.cu_num_logits_np[1 : num_reqs + 1] + - input_batch.cu_num_logits_np[:num_reqs], + device=hidden_states.device, + dtype=torch.int32, + ) + accepted = torch.full_like(num_logits, num_accepted_tokens) + num_sampled = torch.minimum(accepted, num_logits) + prefill_lens = self.req_states.prefill_len.gpu[input_batch.idx_mapping] + is_chunked_prefill = input_batch.seq_lens[:num_reqs] < prefill_lens + num_sampled = torch.where(is_chunked_prefill, 0, num_sampled) + num_rejected = torch.where(is_chunked_prefill, 0, num_logits - num_sampled) + sampler_output = MRV2SamplerOutput( + sampled_token_ids=sampled_token_ids, + logprobs_tensors=None, + num_nans=None, + num_sampled=num_sampled, + ) + return sampler_output, num_sampled, num_rejected + + monkeypatch.setattr( + InprocClient, + "get_output", + get_fake_step_action_fn(original_step_action_fn), + ) + monkeypatch.setattr( + KVCacheManager, + "allocate_slots", + get_fake_allocate_slots_fn(original_allocate_slots), + ) + monkeypatch.setattr(MRV2GPUModelRunner, "execute_model", wrapped_execute_model) + monkeypatch.setattr(MRV2GPUModelRunner, "sample", fake_sample) + monkeypatch.setattr( + MambaHybridModelState, "preprocess_state", wrapped_preprocess_state + ) + monkeypatch.setattr( + MambaHybridModelState, "postprocess_state", wrapped_postprocess_state + ) + + engine = LLM( + model=MODEL, + load_format="dummy", + enforce_eager=True, + skip_tokenizer_init=True, + enable_prefix_caching=True, + block_size=BLOCK_SIZE, + mamba_cache_mode="align", + speculative_config={ + "method": "qwen3_next_mtp", + "num_speculative_tokens": num_speculative_tokens, + }, + max_num_batched_tokens=3072, + max_model_len=BLOCK_SIZE * 12, + hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS}, + seed=42, + ) + + try: + tests = get_mamba_prefix_cache_step_configs() + + global step_actions + global cur_step_action_idx + global num_accepted_tokens + for test_name, test_config in tests.items(): + num_accepted_tokens = test_config.num_accepted_tokens + cur_step_action_idx = 0 + fill_following_kv_cache_block_ids(test_config) + step_actions = test_config.step_actions + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=test_config.num_generated_tokens, + ignore_eos=True, + ) + _ = engine.generate( + [TokensPrompt(prompt_token_ids=[1] * test_config.num_prompt_tokens)], + sampling_params=sampling_params, + ) + assert cur_step_action_idx == len(test_config.step_actions), test_name + assert ( + engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache() + ) + + step_actions = [] + cur_step_action_idx = 0 + num_accepted_tokens = 1 + prompt = TokensPrompt(prompt_token_ids=[1] * (BLOCK_SIZE * 2)) + sampling_params = SamplingParams( + temperature=0.0, + max_tokens=1, + ignore_eos=True, + ) + _ = engine.generate([prompt], sampling_params=sampling_params) + first_event_count = len(events) + _ = engine.generate([prompt], sampling_params=sampling_params) + second_events = events[first_event_count:] + prefix_hits = [ + num_computed_tokens + for num_computed_tokens in second_events + if num_computed_tokens >= BLOCK_SIZE + ] + assert prefix_hits, ( + "Expected the second identical prompt to hit prefix cache, " + f"got events={second_events!r}" + ) + assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache() + finally: + del engine + torch.accelerator.empty_cache() + cleanup_dist_env_and_memory() diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index ba7d26c93b..b093b1788a 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1997,13 +1997,6 @@ class VllmConfig: model_config = self.model_config speculative_config = self.speculative_config - if ( - model_config is not None - and model_config.has_inner_state - and self.cache_config.mamba_cache_mode == "align" - ): - unsupported.append("hybrid/mamba models with align cache mode") - if self.parallel_config.prefill_context_parallel_size > 1: unsupported.append("prefill context parallelism") @@ -2152,10 +2145,6 @@ class VllmConfig: "to schedule a multiple of block_size tokens even if they are " "in the middle of a mm input" ) - # TODO: support align mamba cache mode for model runner v2 - assert not envs.VLLM_USE_V2_MODEL_RUNNER, ( - "Model Runner V2 has not yet supported mamba_cache_mode='align'. " - ) @model_validator(mode="after") def validate_nvfp4_kv_cache_with_mla(self) -> "VllmConfig": diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 6121e55dab..eebb5ef148 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -973,7 +973,9 @@ class DiffusionGemmaModelState(ModelState): # so the captured graph and runtime point to identical addresses. return {"inputs_embeds": self._inputs_embeds_buf[:num_tokens]} - def postprocess_state(self, idx_mapping, num_sampled) -> None: + def postprocess_state( + self, idx_mapping, num_sampled, num_computed_tokens=None + ) -> None: return None def prepare_attn( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 8869e93f9f..c9f4362a6f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1111,7 +1111,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.req_states.total_len.gpu, ) - self.model_state.postprocess_state(idx_mapping, num_sampled) + self.model_state.postprocess_state( + idx_mapping, num_sampled, self.req_states.num_computed_tokens.gpu + ) @torch.inference_mode() def execute_model( @@ -1176,6 +1178,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Prepare all the inputs and copy to the input buffers. input_batch = self.prepare_inputs(scheduler_output, batch_desc) block_tables, slot_mappings = self.prepare_attn(input_batch) + # Mamba "align" pre-copy: migrate recurrent state across block + # boundaries before the forward. Runs only on real batches, and + # before model_state.prepare_attn gathers num_accepted_tokens so the + # boundary reset is visible to the attention metadata. + self.model_state.preprocess_state( + input_batch, + block_tables, + self.kv_cache_config, + self.req_states.num_computed_tokens.gpu, + ) if self.lora_config: # Activate LoRA adapters. diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index c80e19547c..df86efa4a7 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -95,8 +95,23 @@ class ModelState(ABC): def apply_staged_writes(self) -> None: return None + def preprocess_state( + self, + input_batch: InputBatch, + block_tables: tuple[torch.Tensor, ...], + kv_cache_config: KVCacheConfig, + num_computed_tokens: torch.Tensor, + ) -> None: + """Hook run on real batches before the forward pass (after block tables + are gathered). Used by mamba "align" prefix caching to pre-copy state + across block boundaries. No-op by default.""" + return None + def postprocess_state( - self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor + self, + idx_mapping: torch.Tensor, + num_sampled: torch.Tensor, + num_computed_tokens: torch.Tensor | None = None, ) -> None: return None diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index a0f5968361..c6a0632c2d 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -9,15 +9,25 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.layers.mamba.mamba_utils import ( + get_conv_copy_spec, + is_conv_state_dim_first, +) from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder -from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.core.sched.output import NewRequestData +from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec +from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu.attn_utils import build_attn_metadata from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.model_states.default import DefaultModelState from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata +from vllm.v1.worker.mamba_utils import ( + MambaSpecDecodeGPUContext, + preprocess_mamba_align_fused_kernel, +) from vllm.v1.worker.utils import AttentionGroup @@ -65,9 +75,142 @@ class MambaHybridModelState(DefaultModelState): device: torch.device, ) -> None: super().__init__(vllm_config, model, encoder_cache, device) + self.cache_config = vllm_config.cache_config self.num_accepted_tokens_gpu = torch.ones( self.max_num_reqs, dtype=torch.int32, device=self.device ) + # Pre-copy "align" prefix-cache state (V2). The migration of each + # request's mamba state across block boundaries runs as a fused GPU + # kernel reusing the postprocess copy machinery, so the per-step src + # columns and the running state_idx are kept GPU-resident. + self._align_mode = self.cache_config.mamba_cache_mode == "align" + if self._align_mode: + self._mamba_state_idx_gpu = torch.zeros( + self.max_num_reqs, dtype=torch.int32, device=self.device + ) + self._mamba_src_col_gpu = torch.full( + (self.max_num_reqs,), -1, dtype=torch.int32, device=self.device + ) + self._mamba_src_off_gpu = torch.zeros( + self.max_num_reqs, dtype=torch.int32, device=self.device + ) + self._mamba_ctx: MambaSpecDecodeGPUContext | None = None + self._mamba_group_ids: list[int] = [] + self._mamba_spec: MambaSpec | None = None + + def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: + super().add_request(req_index, new_req_data) + if self._align_mode: + # Seed the running state block from the resumed/prefilled position. + self._mamba_state_idx_gpu[req_index] = ( + new_req_data.num_computed_tokens - 1 + ) // self.cache_config.block_size + self.num_accepted_tokens_gpu[req_index] = 1 + + def _get_mamba_group_info( + self, kv_cache_config: KVCacheConfig + ) -> tuple[list[int], MambaSpec]: + if self._mamba_spec is None: + group_ids: list[int] = [] + specs: list[MambaSpec] = [] + for i, group in enumerate(kv_cache_config.kv_cache_groups): + spec = group.kv_cache_spec + if isinstance(spec, MambaSpec): + group_ids.append(i) + specs.append(spec) + assert specs, "no mamba layers in the model" + assert all(specs[0] == s for s in specs) + self._mamba_group_ids = group_ids + self._mamba_spec = specs[0] + return self._mamba_group_ids, self._mamba_spec + + def _ensure_align_ctx( + self, + kv_cache_config: KVCacheConfig, + mamba_group_ids: list[int], + block_tables: tuple[torch.Tensor, ...], + ) -> MambaSpecDecodeGPUContext: + if self._mamba_ctx is None: + copy_funcs = self.model.get_mamba_state_copy_func() + # The fused copy kernels shift conv windows assuming the SD layout; + # the DS layout cannot express a >0 spec-decode shift as a single + # contiguous copy (mirrors get_conv_copy_spec's NotImplementedError). + if get_conv_copy_spec in copy_funcs and is_conv_state_dim_first(): + assert self.vllm_config.speculative_config is None, ( + "DS conv state layout does not support mamba align state " + "copies with speculative decoding" + ) + self._mamba_ctx = MambaSpecDecodeGPUContext.create( + max_num_reqs=self.max_num_reqs, + kv_cache_config=kv_cache_config, + num_state_types=len(copy_funcs), + device=self.device, + make_buffer=lambda n, dtype: CpuGpuBuffer( + n, dtype=dtype, device=self.device + ), + ) + ctx = self._mamba_ctx + if not ctx.is_initialized: + forward_context = self.vllm_config.compilation_config.static_forward_context + # block_tables are batch-order slices of the persistent + # input_block_tables (stable data_ptr), so the metadata is captured + # once here and reused across steps. + ctx.initialize_from_forward_context( + kv_cache_config, + forward_context, + self.model.get_mamba_state_copy_func(), + [block_tables[gid] for gid in mamba_group_ids], + ) + return ctx + + def preprocess_state( + self, + input_batch: InputBatch, + block_tables: tuple[torch.Tensor, ...], + kv_cache_config: KVCacheConfig, + num_computed_tokens: torch.Tensor, + ) -> None: + """Migrate each request's mamba state across block boundaries before the + forward (V1 align semantics, done on GPU). Runs on real batches only + (dummy DP/profiling runs skip preprocess_state), and before + ``prepare_attn`` gathers ``num_accepted_tokens``, so the boundary reset + is visible to the forward kernels. + """ + if not self._align_mode: + return + num_reqs = input_batch.num_reqs + if num_reqs == 0: + return + mamba_group_ids, mamba_spec = self._get_mamba_group_info(kv_cache_config) + ctx = self._ensure_align_ctx(kv_cache_config, mamba_group_ids, block_tables) + + # The state-advance + pre-copy kernels run every step; they fast-exit per + # request when src_col < 0 or src_col == dst_col, so no copy happens on + # steps that don't cross a block boundary. (Skipping the launch entirely + # would need a V1-style async-D2H of the actual num_computed, since + # num_computed_tokens_np is an optimistic mirror under async scheduling; + # the launch cost is ~0.3% of TPOT, so the GPU fast-exit suffices.) + block = 256 + grid = (triton.cdiv(num_reqs, block),) + preprocess_mamba_align_fused_kernel[grid]( + input_batch.idx_mapping, + self._mamba_state_idx_gpu, + num_computed_tokens, + input_batch.query_start_loc, + self.num_accepted_tokens_gpu, + self._mamba_src_col_gpu, + self._mamba_src_off_gpu, + num_reqs, + BLOCK_SIZE=block, + MAMBA_BLOCK_SIZE=mamba_spec.block_size, + ) + ctx.run_fused_precopy( + num_reqs, + self._mamba_state_idx_gpu, + self._mamba_src_col_gpu, + self._mamba_src_off_gpu, + input_batch.idx_mapping, + ) def prepare_attn( self, @@ -145,22 +288,45 @@ class MambaHybridModelState(DefaultModelState): ) def postprocess_state( - self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor | int + self, + idx_mapping: torch.Tensor, + num_sampled: torch.Tensor | int, + num_computed_tokens: torch.Tensor | None = None, ) -> None: # Chunked prefill does not sample a token, so num_sampled can be 0. # Mamba treats num_accepted_tokens=1 as the neutral non-spec value. if not isinstance(num_sampled, int): # idx_mapping may contain -1 sentinels (filtered rows) under PP; the # kernel skips them rather than scattering with a host-side gather. - num_reqs = idx_mapping.shape[0] - if num_reqs: - _scatter_num_accepted_kernel[(num_reqs,)]( + n = idx_mapping.shape[0] + if n: + _scatter_num_accepted_kernel[(n,)]( idx_mapping, num_sampled, self.num_accepted_tokens_gpu ) - return + else: + # Fill with single value. + self.num_accepted_tokens_gpu.index_fill_( + 0, idx_mapping, max(num_sampled, 1) + ) - # Fill with single value. - self.num_accepted_tokens_gpu.index_fill_(0, idx_mapping, max(num_sampled, 1)) + # Align: save the running state to the block-aligned position when + # spec-decode acceptance leaves the sequence non-block-aligned (mirrors + # the V1 align postprocess). num_computed_tokens already holds the + # post-step advanced count. + if ( + self._align_mode + and num_computed_tokens is not None + and self._mamba_ctx is not None + ): + num_reqs = idx_mapping.shape[0] + if num_reqs: + self._mamba_ctx.run_fused_postprocess_align( + num_reqs, + self.num_accepted_tokens_gpu, + self._mamba_state_idx_gpu, + num_computed_tokens, + idx_mapping, + ) @triton.jit diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index 3192b9aeaa..ff9a75f0ef 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -17,6 +17,7 @@ from vllm.v1.core.sched.output import ( NewRequestData, SchedulerOutput, ) +from vllm.v1.kv_cache_interface import MambaSpec from vllm.v1.request import Request from vllm.v1.worker.gpu.model_runner import GPUModelRunner @@ -177,9 +178,17 @@ def warmup_kernels( num_kv_cache_groups = len(kv_cache_groups) # Compute per-request block counts for each KV cache group. - group_block_sizes = [g.kv_cache_spec.block_size for g in kv_cache_groups] - prefill_block_counts = [cdiv(prompt_len, bs) for bs in group_block_sizes] - decode_block_counts = [cdiv(decode_len, bs) for bs in group_block_sizes] + def _warmup_block_count(num_tokens: int, spec: Any) -> int: + num_blocks = cdiv(num_tokens, spec.block_size) + if isinstance(spec, MambaSpec) and spec.mamba_cache_mode == "align": + # Align mode reserves extra blocks beyond the token range for the + # speculative-decode running-state snapshots. + num_blocks += spec.num_speculative_blocks + return num_blocks + + kv_cache_specs = [g.kv_cache_spec for g in kv_cache_groups] + prefill_block_counts = [_warmup_block_count(prompt_len, s) for s in kv_cache_specs] + decode_block_counts = [_warmup_block_count(decode_len, s) for s in kv_cache_specs] decode_block_deltas = [ d - p for d, p in zip(decode_block_counts, prefill_block_counts) ] diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 45166ef9a3..8d8d3e62a9 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -23,6 +23,112 @@ from vllm.v1.worker.gpu_input_batch import CachedRequestState from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch +@triton.jit +def _copy_mamba_state_block( + state_idx, + bt_row_idx, + src_col, + dst_col, + token_bias, + block_table_ptrs_ptr, + block_table_stride_req, + state_base_addrs_ptr, + state_block_strides_ptr, + state_elem_sizes_ptr, + state_inner_sizes_ptr, + state_conv_widths_ptr, + state_group_indices_ptr, + # DS conv row metadata. Zero keeps the single-region copy path. + state_dim_row_count_ptr, + state_dim_row_stride_ptr, + COPY_BLOCK_SIZE: tl.constexpr, + CONV_STATE_DIM_FIRST: tl.constexpr, +): + """Copy one (layer, state-type) mamba state block between block columns. + + Shared copy body of ``postprocess_mamba_fused_kernel`` and + ``precopy_mamba_align_fused_kernel``, mirroring the V1 copy specs + (``get_conv_copy_spec`` / ``get_temporal_copy_spec``): + - conv state (conv_width > 0): shift the window by ``token_bias`` tokens, + ``state[bt[src_col], token_bias:] -> + state[bt[dst_col], :conv_width - token_bias]`` + - temporal state: ``token_bias`` selects the accepted speculative column, + ``state[bt[src_col + token_bias]] -> state[bt[dst_col]]`` + + The caller owns the decision logic (which columns, whether to copy); this + device function only performs the byte copy for the given metadata slot. + """ + state_base_addr = tl.load(state_base_addrs_ptr + state_idx) + state_block_stride = tl.load(state_block_strides_ptr + state_idx) + state_elem_size = tl.load(state_elem_sizes_ptr + state_idx) + state_inner_size = tl.load(state_inner_sizes_ptr + state_idx) + conv_width = tl.load(state_conv_widths_ptr + state_idx) + + # Load the group index for this state, then index into the correct + # group's block table. Each mamba group has independently allocated + # physical blocks. Reinterpret as int32* since block ids are int32. + group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64) + group_base_addr = tl.load(block_table_ptrs_ptr + group_idx) + block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32)) + block_table_base = block_table_typed + bt_row_idx * block_table_stride_req + + # Widen block ids to int64 before they reach `block_id * state_block_stride` + # below: state_block_stride can exceed 2**31 bytes for large mamba caches, + # and Triton would otherwise do the multiply in int32 and wrap. + dest_block_id = tl.load(block_table_base + dst_col).to(tl.int64) + dst_addr = state_base_addr + dest_block_id * state_block_stride + + is_conv_state = conv_width > 0 + + if CONV_STATE_DIM_FIRST and is_conv_state: + # DS conv layout: state_len is the slide axis; copy per dim row. + src_block_id = tl.load(block_table_base + src_col).to(tl.int64) + dim_rows = tl.load(state_dim_row_count_ptr + state_idx) + row_stride = tl.load(state_dim_row_stride_ptr + state_idx) + per_row_bytes = (conv_width - token_bias).to(tl.int64) * state_elem_size + bias_bytes = token_bias.to(tl.int64) * state_elem_size + src_block_addr = state_base_addr + src_block_id * state_block_stride + offsets = tl.arange(0, COPY_BLOCK_SIZE) + for d in range(0, dim_rows): + row_src = src_block_addr + d * row_stride + bias_bytes + row_dst = dst_addr + d * row_stride + for i in range(0, per_row_bytes, COPY_BLOCK_SIZE): + mask = (i + offsets) < per_row_bytes + curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8)) + curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8)) + data = tl.load(curr_src, mask=mask) + tl.store(curr_dst, data, mask=mask) + return + + if is_conv_state: + # SD conv: copy + # state[bt[src_col], token_bias:] -> + # state[bt[dst_col], :conv_width - token_bias] + src_block_id = tl.load(block_table_base + src_col).to(tl.int64) + src_offset = token_bias.to(tl.int64) * state_inner_size * state_elem_size + src_addr = state_base_addr + src_block_id * state_block_stride + src_offset + num_elems_to_copy = (conv_width - token_bias).to(tl.int64) * state_inner_size + copy_size = num_elems_to_copy * state_elem_size + else: + # Temporal state: copy state[bt[src_col + token_bias]] -> state[bt[dst_col]] + actual_src_block_id = tl.load(block_table_base + src_col + token_bias).to( + tl.int64 + ) + src_addr = state_base_addr + actual_src_block_id * state_block_stride + # Use natural block data size (inner_size * elem_size), NOT + # state_block_stride which is the page stride and can exceed the + # actual data when the state tensor uses as_strided page padding. + copy_size = state_inner_size * state_elem_size + + offsets = tl.arange(0, COPY_BLOCK_SIZE) + for i in range(0, copy_size, COPY_BLOCK_SIZE): + mask = (i + offsets) < copy_size + curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8)) + curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8)) + data = tl.load(curr_src, mask=mask) + tl.store(curr_dst, data, mask=mask) + + @triton.jit def postprocess_mamba_fused_kernel( # Decision inputs (per-request) @@ -49,6 +155,10 @@ def postprocess_mamba_fused_kernel( state_dim_row_stride_ptr, # int64: bytes between rows for DS conv # Output: num_accepted_tokens update (for src==dst case) num_accepted_tokens_out_ptr, + # Optional: batch_idx -> req_idx mapping (V2 model runner / PP). The + # per-request decision arrays are in req-state-slot order; the block table + # is in batch order, so HAS_IDX_MAPPING splits the two indexings. + idx_mapping_ptr, # Runtime parameter (varies per batch - NOT constexpr to avoid recompilation) num_reqs, # Compile-time constants (fixed after model initialization) @@ -57,35 +167,53 @@ def postprocess_mamba_fused_kernel( # COPY_BLOCK_SIZE: fixed tuning parameter for memory copy loop COPY_BLOCK_SIZE: tl.constexpr, CONV_STATE_DIM_FIRST: tl.constexpr, + # HAS_IDX_MAPPING: when True, program_id(0) is a batch index resolved to a + # req-state slot via idx_mapping_ptr (V2). When False, it is the req index. + HAS_IDX_MAPPING: tl.constexpr = False, + # PRECOMPUTED_NEW_COMPUTED: when True, num_computed_tokens_ptr already holds + # the post-step new_num_computed value (V2 supplies the advanced count). + PRECOMPUTED_NEW_COMPUTED: tl.constexpr = False, ): """ Fused GPU kernel for postprocess_mamba that computes decisions AND performs mamba state copies without any CPU-GPU synchronization. Grid: (num_reqs, num_layers * num_state_types) - - program_id(0) = request index + - program_id(0) = request/batch index - program_id(1) = state_idx (flattened index into layer/state_type metadata) Note: num_layers and num_state_types are not passed as kernel parameters because the kernel indexes directly into pre-flattened metadata arrays using program_id(1). The grid dimensions encode the total state count. """ - req_idx = tl.program_id(0) + batch_idx = tl.program_id(0) state_idx = tl.program_id(1) # Bounds check - if req_idx >= num_reqs: + if batch_idx >= num_reqs: return + if HAS_IDX_MAPPING: + req_idx = tl.load(idx_mapping_ptr + batch_idx) + if req_idx < 0: + return + else: + req_idx = batch_idx + # Compute decision logic (mirrors postprocess_mamba Python reference) num_accepted = tl.load(num_accepted_tokens_ptr + req_idx) src_block_idx = tl.load(mamba_state_idx_ptr + req_idx) - num_scheduled = tl.load(num_scheduled_tokens_ptr + req_idx) - num_computed = tl.load(num_computed_tokens_ptr + req_idx) - num_draft = tl.load(num_draft_tokens_ptr + req_idx) - num_tokens_running_state = num_computed + num_scheduled - num_draft - new_num_computed = num_tokens_running_state + num_accepted - 1 + if PRECOMPUTED_NEW_COMPUTED: + new_num_computed = tl.load(num_computed_tokens_ptr + req_idx) + num_tokens_running_state = new_num_computed - num_accepted + 1 + else: + num_scheduled = tl.load(num_scheduled_tokens_ptr + req_idx) + num_computed = tl.load(num_computed_tokens_ptr + req_idx) + num_draft = tl.load(num_draft_tokens_ptr + req_idx) + num_tokens_running_state = num_computed + num_scheduled - num_draft + new_num_computed = num_tokens_running_state + num_accepted - 1 + aligned_new_computed = (new_num_computed // block_size) * block_size needs_copy = aligned_new_computed >= num_tokens_running_state @@ -97,99 +225,158 @@ def postprocess_mamba_fused_kernel( accept_token_bias = aligned_new_computed - num_tokens_running_state dest_block_idx = aligned_new_computed // block_size - 1 - # Load state metadata for this layer/state_type - state_base_addr = tl.load(state_base_addrs_ptr + state_idx) - state_block_stride = tl.load(state_block_strides_ptr + state_idx) - state_elem_size = tl.load(state_elem_sizes_ptr + state_idx) - state_inner_size = tl.load(state_inner_sizes_ptr + state_idx) - conv_width = tl.load(state_conv_widths_ptr + state_idx) - - # Load the group index for this state, then index into the correct - # group's block table. Each mamba group has independently allocated - # physical blocks. - group_idx = tl.load(state_group_indices_ptr + state_idx).to(tl.int64) - - # block_table_ptrs_ptr holds one pointer per group (each group owns its own - # block table). Reinterpret as int32* since block ids are int32. - group_base_addr = tl.load(block_table_ptrs_ptr + group_idx) - block_table_typed = group_base_addr.to(tl.pointer_type(tl.int32)) - block_table_base = block_table_typed + req_idx * block_table_stride_req - - # Widen block ids to int64 before they reach `block_id * state_block_stride` - # below: state_block_stride can exceed 2**31 bytes for large mamba caches, - # and Triton would otherwise do the multiply in int32 and wrap. - src_block_id = tl.load(block_table_base + src_block_idx).to(tl.int64) - dest_block_id = tl.load(block_table_base + dest_block_idx).to(tl.int64) - - # Compute source and destination addresses based on state type - # conv_width > 0 means this is a conv state (get_conv_copy_spec logic) - # conv_width == 0 means this is a temporal state (get_temporal_copy_spec logic) - is_conv_state = conv_width > 0 - - # Update accepted-token count before early exits. + # Update accepted-token count before early exits (per-request, so only + # state_idx == 0 writes). V2 updates in place; V1 writes the _out buffer. if src_block_idx == dest_block_idx and state_idx == 0: - tl.store(num_accepted_tokens_out_ptr + req_idx, 1) + if HAS_IDX_MAPPING: + tl.store(num_accepted_tokens_ptr + req_idx, 1) + else: + tl.store(num_accepted_tokens_out_ptr + req_idx, 1) # Skip no-op self-copy. if src_block_idx == dest_block_idx and accept_token_bias == 0: return - if CONV_STATE_DIM_FIRST and is_conv_state: - dim_rows = tl.load(state_dim_row_count_ptr + state_idx) - row_stride = tl.load(state_dim_row_stride_ptr + state_idx) - per_row_bytes = (conv_width - accept_token_bias).to(tl.int64) * state_elem_size - bias_bytes = accept_token_bias.to(tl.int64) * state_elem_size - src_block_addr = state_base_addr + src_block_id * state_block_stride - dst_block_addr = state_base_addr + dest_block_id * state_block_stride - offsets = tl.arange(0, COPY_BLOCK_SIZE) - for d in range(0, dim_rows): - row_src = src_block_addr + d * row_stride + bias_bytes - row_dst = dst_block_addr + d * row_stride - for i in range(0, per_row_bytes, COPY_BLOCK_SIZE): - mask = (i + offsets) < per_row_bytes - curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8)) - curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8)) - data = tl.load(curr_src, mask=mask) - tl.store(curr_dst, data, mask=mask) + bt_row_idx = batch_idx if HAS_IDX_MAPPING else req_idx + _copy_mamba_state_block( + state_idx, + bt_row_idx, + src_block_idx, + dest_block_idx, + accept_token_bias, + block_table_ptrs_ptr, + block_table_stride_req, + state_base_addrs_ptr, + state_block_strides_ptr, + state_elem_sizes_ptr, + state_inner_sizes_ptr, + state_conv_widths_ptr, + state_group_indices_ptr, + state_dim_row_count_ptr, + state_dim_row_stride_ptr, + COPY_BLOCK_SIZE, + CONV_STATE_DIM_FIRST, + ) + + +@triton.jit +def preprocess_mamba_align_fused_kernel( + idx_mapping_ptr, + state_idx_ptr, + num_computed_tokens_ptr, + query_start_loc_ptr, + num_accepted_tokens_ptr, + src_col_ptr, + src_off_ptr, + num_reqs, + BLOCK_SIZE: tl.constexpr, + MAMBA_BLOCK_SIZE: tl.constexpr, +): + """Fused align preprocess: emit the pre-copy src column/offset AND advance + state_idx (with accepted-token reset) in a single launch (V2 align). + + Per batch_idx (0..num_reqs-1), resolving req slot via idx_mapping: + 1. Read pre-advance state_idx and num_accepted (last step's values). + 2. Store the pre-copy src columns for ``precopy_mamba_align_fused_kernel``: + - src_col = state_idx (the previous running block column) + - src_off = max(num_accepted - 1, 0) (the accepted-token bias) + 3. Advance state_idx to the new running block, and reset num_accepted to 1 + when a block boundary is crossed (so the migrated state, now at the + start of the new block, is read with the neutral bias). + """ + offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offsets < num_reqs + req_indices = tl.load(idx_mapping_ptr + offsets, mask=mask, other=0) + + state_idx = tl.load(state_idx_ptr + req_indices, mask=mask, other=-1) + num_accepted = tl.load(num_accepted_tokens_ptr + req_indices, mask=mask, other=1) + + src_off = tl.maximum(num_accepted - 1, 0) + tl.store(src_col_ptr + req_indices, state_idx, mask=mask) + tl.store(src_off_ptr + req_indices, src_off, mask=mask) + + num_computed = tl.load(num_computed_tokens_ptr + req_indices, mask=mask, other=0) + query_start = tl.load(query_start_loc_ptr + offsets, mask=mask, other=0) + query_end = tl.load(query_start_loc_ptr + offsets + 1, mask=mask, other=0) + computed_after = num_computed + query_end - query_start + new_state_idx = (computed_after + MAMBA_BLOCK_SIZE - 1) // MAMBA_BLOCK_SIZE - 1 + tl.store(state_idx_ptr + req_indices, new_state_idx, mask=mask) + should_reset = (state_idx >= 0) & (state_idx != new_state_idx) + tl.store(num_accepted_tokens_ptr + req_indices, 1, mask=mask & should_reset) + + +@triton.jit +def precopy_mamba_align_fused_kernel( + # Per-request-slot inputs (indexed by req_idx via idx_mapping), produced by + # the V2 fused align preprocess kernel for the current step: + mamba_state_idx_ptr, # post-advance dst block column + src_col_ptr, # pre-advance src block column (-1 = fresh) + token_bias_ptr, # accepted-token bias = num_accepted - 1 (pre-reset) + # Same flattened state-layout metadata as postprocess_mamba_fused_kernel + block_table_ptrs_ptr, + block_table_stride_req: tl.int64, + state_base_addrs_ptr, + state_block_strides_ptr, + state_elem_sizes_ptr, + state_inner_sizes_ptr, + state_conv_widths_ptr, + state_group_indices_ptr, + state_dim_row_count_ptr, + state_dim_row_stride_ptr, + idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip) + num_reqs, + COPY_BLOCK_SIZE: tl.constexpr, + CONV_STATE_DIM_FIRST: tl.constexpr, +): + """Pre-copy mamba "align" state across block boundaries on the V2 runner. + + Before the forward pass, copy each request's last SSM/conv state from its + previous block column into the new window block column, so the kernels read + the initial state from the write-side block as usual (V1 align semantics). + Same per-(layer, state) copy semantics as ``postprocess_mamba_fused_kernel`` + (shared ``_copy_mamba_state_block`` body, i.e. the V1 ``preprocess_mamba`` + copy specs), but driven by the GPU-resident src columns so it needs no + CPU-GPU sync (async-scheduling safe). + + Grid: (num_reqs, num_layers * num_state_types); block tables are indexed by + batch row, per-request state by req_idx via idx_mapping (V2 layout). + """ + batch_idx = tl.program_id(0) + state_idx = tl.program_id(1) + if batch_idx >= num_reqs: + return + req_idx = tl.load(idx_mapping_ptr + batch_idx) + if req_idx < 0: return - if is_conv_state: - # SD conv: copy - # state[block_table[req_idx, src_block_idx], accept_token_bias:] - # to - # state[block_table[req_idx, dest_block_idx], :conv_width - accept_token_bias] - src_offset = accept_token_bias.to(tl.int64) * state_inner_size * state_elem_size - src_addr = state_base_addr + src_block_id * state_block_stride + src_offset - dst_addr = state_base_addr + dest_block_id * state_block_stride - # Number of elements to copy: - # (conv_width - accept_token_bias) * inner_size - num_elems_to_copy = (conv_width - accept_token_bias).to( - tl.int64 - ) * state_inner_size - copy_size = num_elems_to_copy * state_elem_size - else: - # Temporal state: copy - # state[block_table[req_idx, src_block_idx + accept_token_bias]] - # to - # state[block_table[req_idx, dest_block_idx]] - actual_src_block_idx = src_block_idx + accept_token_bias - actual_src_block_id = tl.load(block_table_base + actual_src_block_idx).to( - tl.int64 - ) - src_addr = state_base_addr + actual_src_block_id * state_block_stride - dst_addr = state_base_addr + dest_block_id * state_block_stride - # Use natural block data size (inner_size * elem_size), NOT - # state_block_stride which is the page stride and can exceed the - # actual data when the state tensor uses as_strided page padding. - copy_size = state_inner_size * state_elem_size + src_col = tl.load(src_col_ptr + req_idx) + dst_col = tl.load(mamba_state_idx_ptr + req_idx) + # Fresh state, or still writing the same block: kernels locate the initial + # state in-block via num_accepted (preserved when no boundary is crossed), + # so there is nothing to copy. + if src_col < 0 or src_col == dst_col: + return - offsets = tl.arange(0, COPY_BLOCK_SIZE) - for i in range(0, copy_size, COPY_BLOCK_SIZE): - mask = (i + offsets) < copy_size - curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8)) - curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8)) - data = tl.load(curr_src, mask=mask) - tl.store(curr_dst, data, mask=mask) + token_bias = tl.load(token_bias_ptr + req_idx) + _copy_mamba_state_block( + state_idx, + batch_idx, + src_col, + dst_col, + token_bias, + block_table_ptrs_ptr, + block_table_stride_req, + state_base_addrs_ptr, + state_block_strides_ptr, + state_elem_sizes_ptr, + state_inner_sizes_ptr, + state_conv_widths_ptr, + state_group_indices_ptr, + state_dim_row_count_ptr, + state_dim_row_stride_ptr, + COPY_BLOCK_SIZE, + CONV_STATE_DIM_FIRST, + ) @triton.jit @@ -559,12 +746,101 @@ class MambaSpecDecodeGPUContext: self.state_dim_row_count, self.state_dim_row_stride, self.num_accepted_tokens_out, + None, # idx_mapping: V1 decision arrays are already in req order num_reqs, block_size=self.block_size, COPY_BLOCK_SIZE=1024, CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), ) + def run_fused_precopy( + self, + num_reqs: int, + state_idx_gpu: torch.Tensor, + src_col_gpu: torch.Tensor, + token_bias_gpu: torch.Tensor, + idx_mapping: torch.Tensor, + ) -> None: + """Pre-copy each request's previous running block into its new window + block before the forward pass (V2 align boundary migration). + + Args: + num_reqs: Number of active requests (batch order). + state_idx_gpu: [max_reqs] post-advance dst block column per req slot. + src_col_gpu: [max_reqs] pre-advance src block column (-1 = fresh). + token_bias_gpu: [max_reqs] accepted-token bias (num_accepted - 1). + idx_mapping: [num_reqs] batch_idx -> req_state_idx (-1 to skip). + """ + if num_reqs == 0 or not self.is_initialized: + return + total_states = self.num_layers * self.num_state_types + grid = (num_reqs, total_states) + precopy_mamba_align_fused_kernel[grid]( + state_idx_gpu, + src_col_gpu, + token_bias_gpu, + self.block_table_ptrs, + self.block_table_stride_req, + self.state_base_addrs, + self.state_block_strides, + self.state_elem_sizes, + self.state_inner_sizes, + self.state_conv_widths, + self.state_group_indices, + self.state_dim_row_count, + self.state_dim_row_stride, + idx_mapping, + num_reqs, + COPY_BLOCK_SIZE=1024, + CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), + ) + + def run_fused_postprocess_align( + self, + num_reqs: int, + num_accepted_tokens_gpu: torch.Tensor, + state_idx_gpu: torch.Tensor, + new_num_computed_tokens_gpu: torch.Tensor, + idx_mapping: torch.Tensor, + ) -> None: + """V2 align postprocess: save the running state to the block-aligned + position after spec-decode acceptance leaves the sequence non-aligned. + + ``num_accepted_tokens_gpu`` is updated in place (reset to 1 when the + accepted position stays in the running block); ``new_num_computed_tokens`` + already holds the post-step computed count (PRECOMPUTED_NEW_COMPUTED). + ``idx_mapping`` maps batch row -> req-state slot (HAS_IDX_MAPPING). + """ + if num_reqs == 0 or not self.is_initialized: + return + total_states = self.num_layers * self.num_state_types + grid = (num_reqs, total_states) + postprocess_mamba_fused_kernel[grid]( + num_accepted_tokens_gpu, + state_idx_gpu, + None, # num_scheduled: unused under PRECOMPUTED_NEW_COMPUTED + new_num_computed_tokens_gpu, + None, # num_draft: unused under PRECOMPUTED_NEW_COMPUTED + self.block_table_ptrs, + self.block_table_stride_req, + self.state_base_addrs, + self.state_block_strides, + self.state_elem_sizes, + self.state_inner_sizes, + self.state_conv_widths, + self.state_group_indices, + self.state_dim_row_count, + self.state_dim_row_stride, + None, # num_accepted_out: V2 updates num_accepted in place + idx_mapping, + num_reqs, + block_size=self.block_size, + COPY_BLOCK_SIZE=1024, + CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), + HAS_IDX_MAPPING=True, + PRECOMPUTED_NEW_COMPUTED=True, + ) + @dataclasses.dataclass class MambaBuffers: