diff --git a/tests/kernels/mamba/test_memcpy_u64_tiled.py b/tests/kernels/mamba/test_memcpy_u64_tiled.py new file mode 100644 index 0000000000..c48270e78d --- /dev/null +++ b/tests/kernels/mamba/test_memcpy_u64_tiled.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit test for ``_memcpy_u64_tiled``. + +Guards the head/body/tail alignment handling and the u64 body's tile +partitioning used by ``postprocess_mamba_fused_kernel`` and +``precopy_mamba_align_fused_kernel``. The device function must be +byte-identical to a ``dst[dst_off:dst_off+copy_size] = src[src_off:src_off+ +copy_size]`` slice copy for every combination of: + +* ``copy_size``: the degenerate 0-byte case, sub-8B (head-only), 8B-aligned + bodies, and a multi-tile body that spans several ``COPY_BLOCK_SIZE`` + iterations. +* ``(src_off, dst_off)`` pairs: aligned, shared sub-8B misalignment (fast + path exercising head/tail masking), and mismatched src/dst alignment + (byte-fallback path). The kernel branches on ``(src ^ dst) & 7``, so a + full 5x5 product would re-exercise the same two codepaths. +* ``NUM_TILES``: 1 (SD conv callsite, single-CTA memcpy) and + ``_TEMPORAL_TILES`` (temporal callsite, u64 range partitioned across + CTAs). +* ``COPY_BLOCK_SIZE``: the production value 1024 (single-tile at these + sizes) and a small value 8 so the existing ``copy_size`` cases actually + cross tile boundaries under ``NUM_TILES=_TEMPORAL_TILES``. The + partitioning math is COPY_BLOCK_SIZE-agnostic, so testing at a small + value covers boundary arithmetic for all values. +""" + +from __future__ import annotations + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton +from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, _memcpy_u64_tiled + +try: + import pytest + + pytestmark = pytest.mark.skipif( + not current_platform.is_cuda(), + reason="_memcpy_u64_tiled needs CUDA/Triton", + ) + _parametrize = pytest.mark.parametrize +except ModuleNotFoundError: # allow running directly as ``python `` + pytest = None + + def _parametrize(_name, _values): + def _deco(fn): + return fn + + return _deco + + +@triton.jit +def _memcpy_wrapper_kernel( + src_ptr, + dst_ptr, + copy_size, + src_off, + dst_off, + COPY_BLOCK_SIZE: tl.constexpr, + NUM_TILES: tl.constexpr, +): + """Thin harness: one CTA per tile, exercising the device function.""" + tile_idx = tl.program_id(0) + src_addr = src_ptr.to(tl.int64) + src_off + dst_addr = dst_ptr.to(tl.int64) + dst_off + _memcpy_u64_tiled( + src_addr, + dst_addr, + copy_size, + tile_idx, + COPY_BLOCK_SIZE=COPY_BLOCK_SIZE, + NUM_TILES=NUM_TILES, + ) + + +# Copy sizes: 0/1/7 (all-head), 8 (all-body when dst-aligned), 15 (body+tail, +# tail-mask path), 16 (small tiled body), 1024 and 4 KiB (spans all 16 tiles +# at COPY_BLOCK_SIZE=8, with 4 KiB adding multi-iteration-per-tile coverage; +# both single-tile at COPY_BLOCK_SIZE=1024). +_COPY_SIZES = [0, 1, 7, 8, 15, 16, 1024, 4 * 1024] +# (src_off, dst_off) pairs. Torch tensors are 256B-aligned at data_ptr, so +# slicing by these bytes yields a controlled sub-8B alignment. The kernel +# branches on ``(src ^ dst) & 7``, so we cover both sides plus the aligned +# baseline; the full 5x5 product added no coverage. +_ALIGN_PAIRS = [ + (0, 0), # fully aligned; head/tail masked out + (3, 3), # shared misalignment; fast path with head_bytes=5 + (7, 7), # shared, extreme; head_bytes=1 with non-empty body + (1, 3), # mismatched; byte load/store fallback + (3, 1), # mismatched, opposite direction +] +_MAX_ALIGN_OFF = max(o for pair in _ALIGN_PAIRS for o in pair) +# (NUM_TILES, COPY_BLOCK_SIZE) configs. NUM_TILES=1 is the SD conv callsite +# (single-CTA memcpy) where tile partitioning collapses, so COPY_BLOCK_SIZE +# is not observable — one value suffices. NUM_TILES=_TEMPORAL_TILES is the +# temporal callsite: 1024 matches the production launch; 8 shrinks per-tile +# rounding so ``_COPY_SIZES``' 1024/4096 cases cross tile boundaries +# end-to-end (partitioning math is COPY_BLOCK_SIZE-agnostic, so a small +# value proves it for all values). +_TILE_BLOCK_CONFIGS = [ + (1, 1024), + (_TEMPORAL_TILES, 1024), + (_TEMPORAL_TILES, 8), +] + + +@_parametrize("copy_size", _COPY_SIZES) +@_parametrize("src_off,dst_off", _ALIGN_PAIRS) +@_parametrize("num_tiles,copy_block_size", _TILE_BLOCK_CONFIGS) +def test_memcpy_u64_tiled_matches_slice_copy( + copy_size, src_off, dst_off, num_tiles, copy_block_size +): + device = torch.device("cuda") + torch.manual_seed(0) + + # Pad by the max possible offset so the reference slice is in-bounds. + slack = _MAX_ALIGN_OFF + 8 + src = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device) + # Start dst from a distinct random pattern so the "unchanged region" + # check catches any accidental out-of-range writes. + dst = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device) + ref = dst.clone() + ref[dst_off : dst_off + copy_size] = src[src_off : src_off + copy_size] + + _memcpy_wrapper_kernel[(num_tiles,)]( + src, + dst, + copy_size, + src_off, + dst_off, + COPY_BLOCK_SIZE=copy_block_size, + NUM_TILES=num_tiles, + ) + torch.accelerator.synchronize() + + # Byte-identical: the memcpy is dtype-agnostic; any diff is a bug. + torch.testing.assert_close(dst, ref, rtol=0, atol=0) + + +if __name__ == "__main__": + for cs in _COPY_SIZES: + for so, do in _ALIGN_PAIRS: + for nt, cbs in _TILE_BLOCK_CONFIGS: + test_memcpy_u64_tiled_matches_slice_copy(cs, so, do, nt, cbs) + print("ok") diff --git a/tests/kernels/mamba/test_precopy_mamba_align.py b/tests/kernels/mamba/test_precopy_mamba_align.py index b7a3da37d5..d5a3f56f13 100644 --- a/tests/kernels/mamba/test_precopy_mamba_align.py +++ b/tests/kernels/mamba/test_precopy_mamba_align.py @@ -31,7 +31,7 @@ import torch from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils from vllm.platforms import current_platform from vllm.v1.worker import mamba_utils as worker_mamba_utils -from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel +from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, precopy_mamba_align_fused_kernel _parametrize: Callable[..., Callable[[Any], Any]] @@ -150,9 +150,14 @@ def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs, conv_dim_first @_parametrize("num_reqs", [1, 4, 16]) @_parametrize("token_bias", [0, 1, 2]) @_parametrize("has_idx_mapping", [True, False]) +@_parametrize("temporal_tiles", [1, _TEMPORAL_TILES]) @_cuda_required def test_precopy_matches_v1_copy_specs( - num_reqs, token_bias, has_idx_mapping, conv_state_dim_first + num_reqs, + token_bias, + has_idx_mapping, + conv_state_dim_first, + temporal_tiles, ): device = torch.device("cuda") torch.manual_seed(0) @@ -191,7 +196,7 @@ def test_precopy_matches_v1_copy_specs( ) bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device) idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device) - grid = (num_reqs, NUM_LAYERS * 2) + grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles) precopy_mamba_align_fused_kernel[grid]( dst_col, src_col, @@ -211,6 +216,7 @@ def test_precopy_matches_v1_copy_specs( COPY_BLOCK_SIZE=1024, CONV_STATE_DIM_FIRST=conv_state_dim_first, HAS_IDX_MAPPING=has_idx_mapping, + TEMPORAL_TILES=temporal_tiles, ) torch.accelerator.synchronize() @@ -413,12 +419,18 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi if __name__ == "__main__": - for nr in (1, 4, 16): - for tb in (0, 1, 2): - for mapping in (True, False): - for dim_first in (False, True): - test_precopy_matches_v1_copy_specs(nr, tb, mapping, dim_first) - print( - f"OK num_reqs={nr} token_bias={tb} " - f"has_idx_mapping={mapping} conv_dim_first={dim_first}" - ) + from itertools import product + + _CASES = product( + (1, 4, 16), # num_reqs + (0, 1, 2), # token_bias + (True, False), # has_idx_mapping + (False, True), # conv_state_dim_first + (1, _TEMPORAL_TILES), # temporal_tiles + ) + for nr, tb, mapping, dim_first, tt in _CASES: + test_precopy_matches_v1_copy_specs(nr, tb, mapping, dim_first, tt) + print( + f"OK num_reqs={nr} token_bias={tb} has_idx_mapping={mapping} " + f"conv_dim_first={dim_first} temporal_tiles={tt}" + ) diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index ea0a9b8a1b..f87a34a364 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -8,6 +8,7 @@ from typing import Any, NamedTuple import torch from vllm.config import CacheConfig +from vllm.logger import init_logger from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, get_conv_copy_spec, @@ -22,6 +23,95 @@ from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu_input_batch import CachedRequestState from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch +logger = init_logger(__name__) + +# 16 saturates HBM on H100/GB200 across the reqs=8..128 range in +# microbenchmarks +_TEMPORAL_TILES = 16 + + +@triton.jit +def _memcpy_u64_tiled( + src_addr, + dst_addr, + copy_size, + tile_idx, + COPY_BLOCK_SIZE: tl.constexpr, + NUM_TILES: tl.constexpr, +): + """Head/body/tail memcpy with the u64 body split across ``NUM_TILES`` CTAs. + + Fast path (``src`` and ``dst`` share sub-8B alignment): tile 0 owns the + byte head that lifts dst to 8B and the 0-7 byte tail; body tiles vectorize + as u64 over the aligned interior. ``NUM_TILES=1`` collapses to a single- + CTA memcpy. Production callers derive both addresses from the same + ``state_base_addr + block_id * stride`` and always take this path. + + Slow path (mismatched sub-8B alignment): byte-wide tiled copy. Some + NVIDIA parts (e.g. GB200) reject misaligned u64 loads with + ``cudaErrorMisalignedAddress`` instead of accepting the misaligned-sector + cost, so we can't just let the fast path run with a misaligned src. + """ + src_addr_i = src_addr.to(tl.int64) + dst_addr_i = dst_addr.to(tl.int64) + + if ((src_addr_i ^ dst_addr_i) & 7) == 0: + head_bytes = tl.minimum(((-dst_addr_i) & 7).to(tl.int64), copy_size) + if tile_idx == 0: + head_off = tl.arange(0, 8) + head_mask = head_off < head_bytes + head_src = src_addr.to(tl.pointer_type(tl.uint8)) + head_dst = dst_addr.to(tl.pointer_type(tl.uint8)) + tl.store( + head_dst + head_off, + tl.load(head_src + head_off, mask=head_mask), + mask=head_mask, + ) + + # Body: u64 tiled. Rounding per_tile up to COPY_BLOCK_SIZE keeps every + # inner-loop iteration full-width vectorized; only the last non-empty + # tile can be masked. Late tiles fall off the end and iterate zero + # times. + body_bytes = copy_size - head_bytes + body_u64 = body_bytes // 8 + per_tile_u64_raw = tl.cdiv(body_u64, NUM_TILES) + per_tile_u64 = tl.cdiv(per_tile_u64_raw, COPY_BLOCK_SIZE) * COPY_BLOCK_SIZE + tile_start = tile_idx.to(tl.int64) * per_tile_u64 + tile_end = tl.minimum(tile_start + per_tile_u64, body_u64) + + src_body_u64 = (src_addr + head_bytes).to(tl.pointer_type(tl.uint64)) + dst_body_u64 = (dst_addr + head_bytes).to(tl.pointer_type(tl.uint64)) + offsets = tl.arange(0, COPY_BLOCK_SIZE) + for i in range(tile_start, tile_end, COPY_BLOCK_SIZE): + mask = (i + offsets) < tile_end + data = tl.load(src_body_u64 + i + offsets, mask=mask) + tl.store(dst_body_u64 + i + offsets, data, mask=mask) + + if tile_idx == 0: + tail_start = head_bytes + body_u64 * 8 + tail_bytes = copy_size - tail_start + tail_off = tl.arange(0, 8) + tail_src = (src_addr + tail_start).to(tl.pointer_type(tl.uint8)) + tail_dst = (dst_addr + tail_start).to(tl.pointer_type(tl.uint8)) + tail_mask = tail_off < tail_bytes + tl.store( + tail_dst + tail_off, + tl.load(tail_src + tail_off, mask=tail_mask), + mask=tail_mask, + ) + else: + src_u8 = src_addr.to(tl.pointer_type(tl.uint8)) + dst_u8 = dst_addr.to(tl.pointer_type(tl.uint8)) + per_tile_bytes_raw = tl.cdiv(copy_size, NUM_TILES) + per_tile_bytes = tl.cdiv(per_tile_bytes_raw, COPY_BLOCK_SIZE) * COPY_BLOCK_SIZE + tile_start = tile_idx.to(tl.int64) * per_tile_bytes + tile_end = tl.minimum(tile_start + per_tile_bytes, copy_size) + offsets = tl.arange(0, COPY_BLOCK_SIZE) + for i in range(tile_start, tile_end, COPY_BLOCK_SIZE): + mask = (i + offsets) < tile_end + data = tl.load(src_u8 + i + offsets, mask=mask) + tl.store(dst_u8 + i + offsets, data, mask=mask) + @triton.jit def _copy_mamba_state_block( @@ -41,8 +131,10 @@ def _copy_mamba_state_block( # DS conv row metadata. Zero keeps the single-region copy path. state_dim_row_count_ptr, state_dim_row_stride_ptr, + tile_idx, COPY_BLOCK_SIZE: tl.constexpr, CONV_STATE_DIM_FIRST: tl.constexpr, + TEMPORAL_TILES: tl.constexpr, ): """Copy one (layer, state-type) mamba state block between block columns. @@ -57,6 +149,13 @@ def _copy_mamba_state_block( The caller owns the decision logic (which columns, whether to copy); this device function only performs the byte copy for the given metadata slot. + + ``tile_idx`` in ``[0, TEMPORAL_TILES)`` partitions the temporal state's + u64 range into ``TEMPORAL_TILES`` contiguous, COPY_BLOCK_SIZE-aligned + slices, giving more CTAs to fill the SMs at small batch (multi-MiB + temporal copies otherwise leave the GPU under-filled). Conv states are + small; only ``tile_idx == 0`` copies them. ``TEMPORAL_TILES == 1`` and + ``tile_idx == 0`` reproduces the untiled behavior. """ state_base_addr = tl.load(state_base_addrs_ptr + state_idx) state_block_stride = tl.load(state_block_strides_ptr + state_idx) @@ -81,6 +180,10 @@ def _copy_mamba_state_block( is_conv_state = conv_width > 0 if CONV_STATE_DIM_FIRST and is_conv_state: + # Conv states are small; only tile 0 does the copy. Higher tiles + # early-return so they contribute nothing beyond a bounds check. + if tile_idx > 0: + return # DS conv layout: state_len is the slide axis; copy per dim row. src_block_id = tl.load(block_table_base + src_col).to(tl.int64) dim_rows = tl.load(state_dim_row_count_ptr + state_idx) @@ -101,53 +204,46 @@ def _copy_mamba_state_block( return if is_conv_state: + if tile_idx > 0: + return # SD conv: copy # state[bt[src_col], token_bias:] -> # state[bt[dst_col], :conv_width - token_bias] + # Small per-block bytes (~60-80 KiB) make tiling degenerate, so + # conv runs as a single-CTA memcpy (NUM_TILES=1). src_block_id = tl.load(block_table_base + src_col).to(tl.int64) src_offset = token_bias.to(tl.int64) * state_inner_size * state_elem_size src_addr = state_base_addr + src_block_id * state_block_stride + src_offset - num_elems_to_copy = (conv_width - token_bias).to(tl.int64) * state_inner_size - copy_size = num_elems_to_copy * state_elem_size - offsets = tl.arange(0, COPY_BLOCK_SIZE) - for i in range(0, copy_size, COPY_BLOCK_SIZE): - mask = (i + offsets) < copy_size - curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8)) - curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8)) - data = tl.load(curr_src, mask=mask) - tl.store(curr_dst, data, mask=mask) + copy_size = ( + (conv_width - token_bias).to(tl.int64) * state_inner_size * state_elem_size + ) + _memcpy_u64_tiled( + src_addr, + dst_addr, + copy_size, + tile_idx, + COPY_BLOCK_SIZE=COPY_BLOCK_SIZE, + NUM_TILES=1, + ) return # Temporal state: copy state[bt[src_col + token_bias]] -> state[bt[dst_col]] + # Body u64 range is partitioned across TEMPORAL_TILES CTAs to keep the + # SMs filled at small batch. actual_src_block_id = tl.load(block_table_base + src_col + token_bias).to(tl.int64) src_addr = state_base_addr + actual_src_block_id * state_block_stride # Use natural block data size (inner_size * elem_size), NOT # state_block_stride which is the page stride and can exceed the # actual data when the state tensor uses as_strided page padding. copy_size = state_inner_size * state_elem_size - - # Vectorize via uint64 (8B per thread → LDG.64/STG.64): both temporal - # and SD conv produce src/dst addresses aligned to a full token slice - # (inner_size * elem_size) and a copy_size that's a multiple of it, - # which is 8B-aligned for all state dtypes in use. A masked byte tail - # covers any remaining 0-7 bytes (only reachable for sub-8B slices). - copy_size_u64 = copy_size // 8 - src_u64 = src_addr.to(tl.pointer_type(tl.uint64)) - dst_u64 = dst_addr.to(tl.pointer_type(tl.uint64)) - offsets = tl.arange(0, COPY_BLOCK_SIZE) - for i in range(0, copy_size_u64, COPY_BLOCK_SIZE): - mask = (i + offsets) < copy_size_u64 - data = tl.load(src_u64 + i + offsets, mask=mask) - tl.store(dst_u64 + i + offsets, data, mask=mask) - - tail_start = copy_size_u64 * 8 - tail_bytes = copy_size - tail_start - tail_off = tl.arange(0, 8) - tail_src = (src_addr + tail_start).to(tl.pointer_type(tl.uint8)) - tail_dst = (dst_addr + tail_start).to(tl.pointer_type(tl.uint8)) - tail_mask = tail_off < tail_bytes - tail_data = tl.load(tail_src + tail_off, mask=tail_mask) - tl.store(tail_dst + tail_off, tail_data, mask=tail_mask) + _memcpy_u64_tiled( + src_addr, + dst_addr, + copy_size, + tile_idx, + COPY_BLOCK_SIZE=COPY_BLOCK_SIZE, + NUM_TILES=TEMPORAL_TILES, + ) @triton.jit(do_not_specialize=["num_reqs"]) @@ -194,14 +290,20 @@ def postprocess_mamba_fused_kernel( # PRECOMPUTED_NEW_COMPUTED: when True, num_computed_tokens_ptr already holds # the post-step new_num_computed value (V2 supplies the advanced count). PRECOMPUTED_NEW_COMPUTED: tl.constexpr = False, + # TEMPORAL_TILES: when > 1, the temporal copy body is partitioned across + # TEMPORAL_TILES CTAs along the u64 inner range. Callers must launch a + # 3D grid (num_reqs, total_states, TEMPORAL_TILES). Default 1 preserves + # the existing 2D-grid contract. + TEMPORAL_TILES: tl.constexpr = 1, ): """ Fused GPU kernel for postprocess_mamba that computes decisions AND performs mamba state copies without any CPU-GPU synchronization. - Grid: (num_reqs, num_layers * num_state_types) + Grid: (num_reqs, num_layers * num_state_types [, TEMPORAL_TILES]) - program_id(0) = request/batch index - program_id(1) = state_idx (flattened index into layer/state_type metadata) + - program_id(2) = temporal-copy tile index (0 when TEMPORAL_TILES == 1) Note: num_layers and num_state_types are not passed as kernel parameters because the kernel indexes directly into pre-flattened metadata arrays @@ -209,6 +311,7 @@ def postprocess_mamba_fused_kernel( """ batch_idx = tl.program_id(0) state_idx = tl.program_id(1) + tile_idx = tl.program_id(2) # Bounds check if batch_idx >= num_reqs: @@ -247,8 +350,9 @@ def postprocess_mamba_fused_kernel( dest_block_idx = aligned_new_computed // block_size - 1 # Update accepted-token count before early exits (per-request, so only - # state_idx == 0 writes). - if src_block_idx == dest_block_idx and state_idx == 0: + # state_idx == 0 writes). Also guard on tile_idx == 0 so tiles > 0 + # (when TEMPORAL_TILES > 1) do not duplicate the store. + if src_block_idx == dest_block_idx and state_idx == 0 and tile_idx == 0: tl.store(num_accepted_tokens_out_ptr + req_idx, 1) # Skip no-op self-copy. @@ -272,8 +376,10 @@ def postprocess_mamba_fused_kernel( state_group_indices_ptr, state_dim_row_count_ptr, state_dim_row_stride_ptr, + tile_idx, COPY_BLOCK_SIZE, CONV_STATE_DIM_FIRST, + TEMPORAL_TILES, ) @@ -346,6 +452,9 @@ def precopy_mamba_align_fused_kernel( COPY_BLOCK_SIZE: tl.constexpr, CONV_STATE_DIM_FIRST: tl.constexpr, HAS_IDX_MAPPING: tl.constexpr = True, + # TEMPORAL_TILES: see postprocess_mamba_fused_kernel. Default 1 preserves + # the 2D-grid contract; > 1 requires a 3D grid. + TEMPORAL_TILES: tl.constexpr = 1, ): """Pre-copy mamba "align" state across block boundaries. @@ -357,12 +466,13 @@ def precopy_mamba_align_fused_kernel( copy specs), but driven by the GPU-resident src columns so it needs no CPU-GPU sync (async-scheduling safe). - Grid: (num_reqs, num_layers * num_state_types). V2 passes a batch-to-state - idx_mapping; V1 already stores the staged arrays in batch order and uses - HAS_IDX_MAPPING=False. + Grid: (num_reqs, num_layers * num_state_types [, TEMPORAL_TILES]). V2 passes + a batch-to-state idx_mapping; V1 already stores the staged arrays in batch + order and uses HAS_IDX_MAPPING=False. """ batch_idx = tl.program_id(0) state_idx = tl.program_id(1) + tile_idx = tl.program_id(2) if batch_idx >= num_reqs: return if HAS_IDX_MAPPING: @@ -397,8 +507,10 @@ def precopy_mamba_align_fused_kernel( state_group_indices_ptr, state_dim_row_count_ptr, state_dim_row_stride_ptr, + tile_idx, COPY_BLOCK_SIZE, CONV_STATE_DIM_FIRST, + TEMPORAL_TILES, ) @@ -701,23 +813,28 @@ class MambaSpecDecodeGPUContext: self.state_inner_sizes[idx] = ( state[0].numel() if state.dim() > 1 else 1 ) - # Temporal copies are vectorized with uint64 - # loads/stores; base pointer and block stride must - # be 8B-aligned (tail loop handles copy_size % 8). + # Temporal copies vectorize with uint64 loads/stores. + # The kernel's head/tail handles misalignment for + # correctness, but unaligned base/stride costs in + # throughput. base_addr = state.data_ptr() block_stride_bytes = block_stride_elems * state.element_size() - assert base_addr % 8 == 0, ( - f"layer {layer_name}: state.data_ptr() = " - f"{base_addr:#x} is not 8B-aligned; " - f"_copy_mamba_state_block uint64 " - f"vectorization requires it" - ) - assert block_stride_bytes % 8 == 0, ( - f"layer {layer_name}: block stride = " - f"{block_stride_bytes}B is not 8B-aligned; " - f"_copy_mamba_state_block uint64 " - f"vectorization requires it" - ) + if base_addr % 8 != 0: + logger.warning_once( + "layer %s: state.data_ptr() = %#x is not " + "8B-aligned; _memcpy_u64_tiled uint64 " + "vectorization will pay misaligned load cost", + layer_name, + base_addr, + ) + if block_stride_bytes % 8 != 0: + logger.warning_once( + "layer %s: block stride = %dB is not " + "8B-aligned; _memcpy_u64_tiled uint64 " + "vectorization will pay misaligned load cost", + layer_name, + block_stride_bytes, + ) self.state_group_indices[idx] = group_local_idx idx += 1 @@ -771,7 +888,7 @@ class MambaSpecDecodeGPUContext: ) total_states = self.num_layers * self.num_state_types - grid = (num_reqs, total_states) + grid = (num_reqs, total_states, _TEMPORAL_TILES) postprocess_mamba_fused_kernel[grid]( num_accepted_tokens_gpu, @@ -795,6 +912,7 @@ class MambaSpecDecodeGPUContext: block_size=self.block_size, COPY_BLOCK_SIZE=1024, CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), + TEMPORAL_TILES=_TEMPORAL_TILES, ) def run_fused_precopy( @@ -819,7 +937,7 @@ class MambaSpecDecodeGPUContext: if num_reqs == 0 or not self.is_initialized: return total_states = self.num_layers * self.num_state_types - grid = (num_reqs, total_states) + grid = (num_reqs, total_states, _TEMPORAL_TILES) precopy_mamba_align_fused_kernel[grid]( state_idx_gpu, src_col_gpu, @@ -839,6 +957,7 @@ class MambaSpecDecodeGPUContext: COPY_BLOCK_SIZE=1024, CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), HAS_IDX_MAPPING=idx_mapping is not None, + TEMPORAL_TILES=_TEMPORAL_TILES, ) def run_fused_postprocess_align( @@ -868,7 +987,7 @@ class MambaSpecDecodeGPUContext: num_accepted_tokens_snapshot.copy_(num_accepted_tokens_gpu) total_states = self.num_layers * self.num_state_types - grid = (num_reqs, total_states) + grid = (num_reqs, total_states, _TEMPORAL_TILES) postprocess_mamba_fused_kernel[grid]( num_accepted_tokens_snapshot, state_idx_gpu, @@ -893,6 +1012,7 @@ class MambaSpecDecodeGPUContext: CONV_STATE_DIM_FIRST=is_conv_state_dim_first(), HAS_IDX_MAPPING=True, PRECOMPUTED_NEW_COMPUTED=True, + TEMPORAL_TILES=_TEMPORAL_TILES, )