diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index dece9db00c..4bbab10b48 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -2132,3 +2132,118 @@ class TestPostprocessMambaFusedKernel: expected_accepted, msg="num_accepted_tokens mismatch at accept_token_bias=2", ) + + def test_ds_conv_layout_bias_gt_0_byte_equal_to_sd( + self, device, test_config, monkeypatch + ): + """DS conv postprocess should match SD when accept_token_bias > 0.""" + from vllm.model_executor.layers.mamba import mamba_utils as model_mamba_utils + + cfg = test_config + torch.manual_seed(38898) + + req_ids = ["req_0"] + num_computed_tokens = [30] + num_scheduled_tokens = {"req_0": 1} + num_draft_tokens: dict[str, int] = {} + num_accepted_tokens = [2] # Results in accept_token_bias = 1 + mamba_state_idx = [1] # src_block_idx = 1 = dest_block_idx + block_ids_per_req = [list(range(8))] + + layer_names = ["layer_0"] + kv_cache_config = _make_kv_cache_config(cfg, layer_names) + + num_reqs = len(req_ids) + block_table_gpu = torch.zeros(num_reqs, 8, dtype=torch.int32, device=device) + block_table_gpu[0, :8] = torch.tensor(block_ids_per_req[0], dtype=torch.int32) + + # Same logical conv state in SD and DS layouts. + sd_source_conv = torch.randn( + cfg.num_blocks, + cfg.conv_width, + cfg.conv_inner_dim, + dtype=cfg.dtype, + device=device, + ) + ds_source_conv = sd_source_conv.permute(0, 2, 1).contiguous() + sd_source_temporal = torch.randn( + cfg.num_blocks, cfg.temporal_state_dim, dtype=cfg.dtype, device=device + ) + + # SD GPU path. Default layout is SD. + model_mamba_utils.get_conv_state_layout.cache_clear() + sd_conv = sd_source_conv.clone() + sd_temporal = sd_source_temporal.clone() + forward_context_sd = { + "layer_0": _make_mock_attention(sd_conv, sd_temporal), + } + gpu_ctx_sd = _make_gpu_ctx(cfg, kv_cache_config, device) + _run_gpu_postprocess( + gpu_ctx_sd, + kv_cache_config=kv_cache_config, + forward_context=forward_context_sd, + copy_funcs=_COPY_FUNCS, + block_table=block_table_gpu, + req_ids=req_ids, + num_accepted_tokens=num_accepted_tokens, + mamba_state_idx=mamba_state_idx, + num_scheduled_tokens=num_scheduled_tokens, + num_computed_tokens=num_computed_tokens, + num_draft_tokens=num_draft_tokens, + device=device, + ) + torch.accelerator.synchronize() + + # Sanity: SD path actually modified the state (copy was performed). + assert not torch.equal(sd_conv, sd_source_conv), ( + "SD baseline did not modify conv state; test setup is wrong" + ) + + # DS GPU path on the DS twin. + monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS") + model_mamba_utils.get_conv_state_layout.cache_clear() + try: + ds_conv = ds_source_conv.clone() + ds_temporal = sd_source_temporal.clone() + forward_context_ds = { + "layer_0": _make_mock_attention(ds_conv, ds_temporal), + } + gpu_ctx_ds = _make_gpu_ctx(cfg, kv_cache_config, device) + _run_gpu_postprocess( + gpu_ctx_ds, + kv_cache_config=kv_cache_config, + forward_context=forward_context_ds, + copy_funcs=_COPY_FUNCS, + block_table=block_table_gpu, + req_ids=req_ids, + num_accepted_tokens=num_accepted_tokens, + mamba_state_idx=mamba_state_idx, + num_scheduled_tokens=num_scheduled_tokens, + num_computed_tokens=num_computed_tokens, + num_draft_tokens=num_draft_tokens, + device=device, + ) + torch.accelerator.synchronize() + finally: + # Reset the lru cache so other tests see the default layout again. + model_mamba_utils.get_conv_state_layout.cache_clear() + + # DS bytes, un-permuted, should match the SD result. + torch.testing.assert_close( + ds_conv.permute(0, 2, 1).contiguous(), + sd_conv, + msg=( + "DS conv post-kernel does not match SD baseline; the DS " + "row-loop in postprocess_mamba_fused_kernel is wrong." + ), + ) + torch.testing.assert_close( + ds_temporal, + sd_temporal, + msg="DS temporal state diverged from SD", + ) + torch.testing.assert_close( + gpu_ctx_ds.num_accepted_tokens_out[:num_reqs], + gpu_ctx_sd.num_accepted_tokens_out[:num_reqs], + msg="DS num_accepted_tokens diverged from SD", + ) diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 0c86c78791..9e78b82228 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -303,18 +303,14 @@ def get_conv_copy_spec( src_block_id = block_ids[cur_block_idx] offset = num_accepted_tokens - 1 if is_conv_state_dim_first(): - # DS layout: (num_blocks, dim, state_len) — state_len is last. - if offset > 0: - # Slicing along the last dim yields a non-contiguous view - # because features (dim) are strided by state_len. - raise NotImplementedError( - "DS conv state layout does not yet support speculative " - "decoding with mamba_cache_mode='align' " - "(num_accepted_tokens > 1)." - ) + # DS offset > 0 is handled by the fused postprocess kernel. + assert offset == 0, ( + "DS conv state with num_accepted_tokens > 1 must be handled by " + "the fused postprocess kernel, not get_conv_copy_spec" + ) src_state = state[src_block_id] else: - # SD layout: (num_blocks, state_len, dim) — dim contiguous. + # SD layout: (num_blocks, state_len, dim), with dim contiguous. src_state = state[src_block_id, offset:] return MambaCopySpec( start_addr=src_state.data_ptr(), num_elements=src_state.numel() diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 485b274eab..a2718b7260 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -12,6 +12,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, get_conv_copy_spec, get_temporal_copy_spec, + is_conv_state_dim_first, ) from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv @@ -43,6 +44,9 @@ def postprocess_mamba_fused_kernel( state_inner_sizes_ptr, # number of elements in inner dimensions state_conv_widths_ptr, # conv width for conv states (0 for temporal) state_group_indices_ptr, # maps state_idx to group index in block table + # DS conv row metadata. Zero keeps the single-region copy path. + state_dim_row_count_ptr, # int32: per-block dim row count for DS conv + state_dim_row_stride_ptr, # int64: bytes between rows for DS conv # Output: num_accepted_tokens update (for src==dst case) num_accepted_tokens_out_ptr, # Runtime parameter (varies per batch - NOT constexpr to avoid recompilation) @@ -52,6 +56,7 @@ def postprocess_mamba_fused_kernel( block_size: tl.constexpr, # COPY_BLOCK_SIZE: fixed tuning parameter for memory copy loop COPY_BLOCK_SIZE: tl.constexpr, + CONV_STATE_DIM_FIRST: tl.constexpr, ): """ Fused GPU kernel for postprocess_mamba that computes decisions AND performs @@ -121,8 +126,35 @@ def postprocess_mamba_fused_kernel( # conv_width == 0 means this is a temporal state (get_temporal_copy_spec logic) is_conv_state = conv_width > 0 + # Update accepted-token count before early exits. + if src_block_idx == dest_block_idx and state_idx == 0: + tl.store(num_accepted_tokens_out_ptr + req_idx, 1) + + # Skip no-op self-copy. + if src_block_idx == dest_block_idx and accept_token_bias == 0: + return + + if CONV_STATE_DIM_FIRST and is_conv_state: + dim_rows = tl.load(state_dim_row_count_ptr + state_idx) + row_stride = tl.load(state_dim_row_stride_ptr + state_idx) + per_row_bytes = (conv_width - accept_token_bias).to(tl.int64) * state_elem_size + bias_bytes = accept_token_bias.to(tl.int64) * state_elem_size + src_block_addr = state_base_addr + src_block_id * state_block_stride + dst_block_addr = state_base_addr + dest_block_id * state_block_stride + offsets = tl.arange(0, COPY_BLOCK_SIZE) + for d in range(0, dim_rows): + row_src = src_block_addr + d * row_stride + bias_bytes + row_dst = dst_block_addr + d * row_stride + for i in range(0, per_row_bytes, COPY_BLOCK_SIZE): + mask = (i + offsets) < per_row_bytes + curr_src = (row_src + i + offsets).to(tl.pointer_type(tl.uint8)) + curr_dst = (row_dst + i + offsets).to(tl.pointer_type(tl.uint8)) + data = tl.load(curr_src, mask=mask) + tl.store(curr_dst, data, mask=mask) + return + if is_conv_state: - # Conv state: copy + # SD conv: copy # state[block_table[req_idx, src_block_idx], accept_token_bias:] # to # state[block_table[req_idx, dest_block_idx], :conv_width - accept_token_bias] @@ -151,19 +183,6 @@ def postprocess_mamba_fused_kernel( # actual data when the state tensor uses as_strided page padding. copy_size = state_inner_size * state_elem_size - # Mirror postprocess_mamba's trailing - # if src_block_idx == dest_block_idx: num_accepted_tokens_cpu[i] = 1 - # This runs whether or not the copy below is skipped (it's per-request, so - # only state_idx == 0 writes). - if src_block_idx == dest_block_idx and state_idx == 0: - tl.store(num_accepted_tokens_out_ptr + req_idx, 1) - - # Mirror collect_mamba_copy_meta's early return: src==dst with no token - # bias means source and destination ranges coincide, so the copy is a - # no-op. - if src_block_idx == dest_block_idx and accept_token_bias == 0: - return - offsets = tl.arange(0, COPY_BLOCK_SIZE) for i in range(0, copy_size, COPY_BLOCK_SIZE): mask = (i + offsets) < copy_size @@ -271,6 +290,9 @@ class MambaSpecDecodeGPUContext: state_inner_sizes: torch.Tensor # int64: elements in inner dimensions state_conv_widths: torch.Tensor # int32: conv width (0 for temporal states) state_group_indices: torch.Tensor # int32: maps state_idx to group index + # DS conv row metadata. Zero keeps the single-region copy path. + state_dim_row_count: torch.Tensor # int32: per-block dim row count + state_dim_row_stride: torch.Tensor # int64: bytes between rows # Configuration block_size: int @@ -338,6 +360,12 @@ class MambaSpecDecodeGPUContext: state_group_indices=torch.zeros( total_states, dtype=torch.int32, device=device ), + state_dim_row_count=torch.zeros( + total_states, dtype=torch.int32, device=device + ), + state_dim_row_stride=torch.zeros( + total_states, dtype=torch.int64, device=device + ), block_size=mamba_spec.block_size, num_layers=num_layers, num_state_types=num_state_types, @@ -430,17 +458,23 @@ class MambaSpecDecodeGPUContext: or copy_func is get_temporal_copy_spec ), f"unexpected copy func: {copy_func}" if copy_func is get_conv_copy_spec: - # Conv state: conv_width is state.size(1) - # inner_size is stride(1) = elements per conv position, - # used to compute byte offset for state[block, offset:] - conv_w = state.size(1) if state.dim() > 1 else 0 - self.state_conv_widths[idx] = conv_w - if state.dim() > 2: - # stride(1) = product of dims[2:] for contiguous tensor - self.state_inner_sizes[idx] = state.stride(1) - else: - # 2D tensor: [num_blocks, conv_dim], no inner dims + if state.dim() != 3: + raise ValueError( + "Expected 3D conv state cache, got " + f"shape {tuple(state.shape)}" + ) + if is_conv_state_dim_first(): + # DS layout: state_len is the slide axis. + self.state_conv_widths[idx] = state.size(2) self.state_inner_sizes[idx] = 1 + self.state_dim_row_count[idx] = state.size(1) + self.state_dim_row_stride[idx] = ( + state.stride(1) * state.element_size() + ) + else: + # SD layout: dim is contiguous. + self.state_conv_widths[idx] = state.size(1) + self.state_inner_sizes[idx] = state.stride(1) else: # Temporal state: inner_size = natural elements per # block (prod of inner dims). The kernel uses this @@ -521,10 +555,13 @@ class MambaSpecDecodeGPUContext: self.state_inner_sizes, self.state_conv_widths, self.state_group_indices, + self.state_dim_row_count, + self.state_dim_row_stride, self.num_accepted_tokens_out, num_reqs, block_size=self.block_size, COPY_BLOCK_SIZE=1024, + CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), )