[Perf][Hybrid] 3D-grid tiling of the state-copy Triton kernels (#49436)
Signed-off-by: Francesco Fusco <ffu@zurich.ibm.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
bd6536071c
commit
fac808b36f
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for ``_memcpy_u64_tiled``.
|
||||
|
||||
Guards the head/body/tail alignment handling and the u64 body's tile
|
||||
partitioning used by ``postprocess_mamba_fused_kernel`` and
|
||||
``precopy_mamba_align_fused_kernel``. The device function must be
|
||||
byte-identical to a ``dst[dst_off:dst_off+copy_size] = src[src_off:src_off+
|
||||
copy_size]`` slice copy for every combination of:
|
||||
|
||||
* ``copy_size``: the degenerate 0-byte case, sub-8B (head-only), 8B-aligned
|
||||
bodies, and a multi-tile body that spans several ``COPY_BLOCK_SIZE``
|
||||
iterations.
|
||||
* ``(src_off, dst_off)`` pairs: aligned, shared sub-8B misalignment (fast
|
||||
path exercising head/tail masking), and mismatched src/dst alignment
|
||||
(byte-fallback path). The kernel branches on ``(src ^ dst) & 7``, so a
|
||||
full 5x5 product would re-exercise the same two codepaths.
|
||||
* ``NUM_TILES``: 1 (SD conv callsite, single-CTA memcpy) and
|
||||
``_TEMPORAL_TILES`` (temporal callsite, u64 range partitioned across
|
||||
CTAs).
|
||||
* ``COPY_BLOCK_SIZE``: the production value 1024 (single-tile at these
|
||||
sizes) and a small value 8 so the existing ``copy_size`` cases actually
|
||||
cross tile boundaries under ``NUM_TILES=_TEMPORAL_TILES``. The
|
||||
partitioning math is COPY_BLOCK_SIZE-agnostic, so testing at a small
|
||||
value covers boundary arithmetic for all values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, _memcpy_u64_tiled
|
||||
|
||||
try:
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="_memcpy_u64_tiled needs CUDA/Triton",
|
||||
)
|
||||
_parametrize = pytest.mark.parametrize
|
||||
except ModuleNotFoundError: # allow running directly as ``python <thisfile>``
|
||||
pytest = None
|
||||
|
||||
def _parametrize(_name, _values):
|
||||
def _deco(fn):
|
||||
return fn
|
||||
|
||||
return _deco
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _memcpy_wrapper_kernel(
|
||||
src_ptr,
|
||||
dst_ptr,
|
||||
copy_size,
|
||||
src_off,
|
||||
dst_off,
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
NUM_TILES: tl.constexpr,
|
||||
):
|
||||
"""Thin harness: one CTA per tile, exercising the device function."""
|
||||
tile_idx = tl.program_id(0)
|
||||
src_addr = src_ptr.to(tl.int64) + src_off
|
||||
dst_addr = dst_ptr.to(tl.int64) + dst_off
|
||||
_memcpy_u64_tiled(
|
||||
src_addr,
|
||||
dst_addr,
|
||||
copy_size,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE=COPY_BLOCK_SIZE,
|
||||
NUM_TILES=NUM_TILES,
|
||||
)
|
||||
|
||||
|
||||
# Copy sizes: 0/1/7 (all-head), 8 (all-body when dst-aligned), 15 (body+tail,
|
||||
# tail-mask path), 16 (small tiled body), 1024 and 4 KiB (spans all 16 tiles
|
||||
# at COPY_BLOCK_SIZE=8, with 4 KiB adding multi-iteration-per-tile coverage;
|
||||
# both single-tile at COPY_BLOCK_SIZE=1024).
|
||||
_COPY_SIZES = [0, 1, 7, 8, 15, 16, 1024, 4 * 1024]
|
||||
# (src_off, dst_off) pairs. Torch tensors are 256B-aligned at data_ptr, so
|
||||
# slicing by these bytes yields a controlled sub-8B alignment. The kernel
|
||||
# branches on ``(src ^ dst) & 7``, so we cover both sides plus the aligned
|
||||
# baseline; the full 5x5 product added no coverage.
|
||||
_ALIGN_PAIRS = [
|
||||
(0, 0), # fully aligned; head/tail masked out
|
||||
(3, 3), # shared misalignment; fast path with head_bytes=5
|
||||
(7, 7), # shared, extreme; head_bytes=1 with non-empty body
|
||||
(1, 3), # mismatched; byte load/store fallback
|
||||
(3, 1), # mismatched, opposite direction
|
||||
]
|
||||
_MAX_ALIGN_OFF = max(o for pair in _ALIGN_PAIRS for o in pair)
|
||||
# (NUM_TILES, COPY_BLOCK_SIZE) configs. NUM_TILES=1 is the SD conv callsite
|
||||
# (single-CTA memcpy) where tile partitioning collapses, so COPY_BLOCK_SIZE
|
||||
# is not observable — one value suffices. NUM_TILES=_TEMPORAL_TILES is the
|
||||
# temporal callsite: 1024 matches the production launch; 8 shrinks per-tile
|
||||
# rounding so ``_COPY_SIZES``' 1024/4096 cases cross tile boundaries
|
||||
# end-to-end (partitioning math is COPY_BLOCK_SIZE-agnostic, so a small
|
||||
# value proves it for all values).
|
||||
_TILE_BLOCK_CONFIGS = [
|
||||
(1, 1024),
|
||||
(_TEMPORAL_TILES, 1024),
|
||||
(_TEMPORAL_TILES, 8),
|
||||
]
|
||||
|
||||
|
||||
@_parametrize("copy_size", _COPY_SIZES)
|
||||
@_parametrize("src_off,dst_off", _ALIGN_PAIRS)
|
||||
@_parametrize("num_tiles,copy_block_size", _TILE_BLOCK_CONFIGS)
|
||||
def test_memcpy_u64_tiled_matches_slice_copy(
|
||||
copy_size, src_off, dst_off, num_tiles, copy_block_size
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Pad by the max possible offset so the reference slice is in-bounds.
|
||||
slack = _MAX_ALIGN_OFF + 8
|
||||
src = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device)
|
||||
# Start dst from a distinct random pattern so the "unchanged region"
|
||||
# check catches any accidental out-of-range writes.
|
||||
dst = torch.randint(0, 256, (copy_size + slack,), dtype=torch.uint8, device=device)
|
||||
ref = dst.clone()
|
||||
ref[dst_off : dst_off + copy_size] = src[src_off : src_off + copy_size]
|
||||
|
||||
_memcpy_wrapper_kernel[(num_tiles,)](
|
||||
src,
|
||||
dst,
|
||||
copy_size,
|
||||
src_off,
|
||||
dst_off,
|
||||
COPY_BLOCK_SIZE=copy_block_size,
|
||||
NUM_TILES=num_tiles,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Byte-identical: the memcpy is dtype-agnostic; any diff is a bug.
|
||||
torch.testing.assert_close(dst, ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for cs in _COPY_SIZES:
|
||||
for so, do in _ALIGN_PAIRS:
|
||||
for nt, cbs in _TILE_BLOCK_CONFIGS:
|
||||
test_memcpy_u64_tiled_matches_slice_copy(cs, so, do, nt, cbs)
|
||||
print("ok")
|
||||
@@ -31,7 +31,7 @@ import torch
|
||||
from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker import mamba_utils as worker_mamba_utils
|
||||
from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel
|
||||
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, precopy_mamba_align_fused_kernel
|
||||
|
||||
_parametrize: Callable[..., Callable[[Any], Any]]
|
||||
|
||||
@@ -150,9 +150,14 @@ def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs, conv_dim_first
|
||||
@_parametrize("num_reqs", [1, 4, 16])
|
||||
@_parametrize("token_bias", [0, 1, 2])
|
||||
@_parametrize("has_idx_mapping", [True, False])
|
||||
@_parametrize("temporal_tiles", [1, _TEMPORAL_TILES])
|
||||
@_cuda_required
|
||||
def test_precopy_matches_v1_copy_specs(
|
||||
num_reqs, token_bias, has_idx_mapping, conv_state_dim_first
|
||||
num_reqs,
|
||||
token_bias,
|
||||
has_idx_mapping,
|
||||
conv_state_dim_first,
|
||||
temporal_tiles,
|
||||
):
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
@@ -191,7 +196,7 @@ def test_precopy_matches_v1_copy_specs(
|
||||
)
|
||||
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
grid = (num_reqs, NUM_LAYERS * 2)
|
||||
grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles)
|
||||
precopy_mamba_align_fused_kernel[grid](
|
||||
dst_col,
|
||||
src_col,
|
||||
@@ -211,6 +216,7 @@ def test_precopy_matches_v1_copy_specs(
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=conv_state_dim_first,
|
||||
HAS_IDX_MAPPING=has_idx_mapping,
|
||||
TEMPORAL_TILES=temporal_tiles,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -413,12 +419,18 @@ def test_preprocess_fused_align_matches_scalar_bookkeeping(monkeypatch, token_bi
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for nr in (1, 4, 16):
|
||||
for tb in (0, 1, 2):
|
||||
for mapping in (True, False):
|
||||
for dim_first in (False, True):
|
||||
test_precopy_matches_v1_copy_specs(nr, tb, mapping, dim_first)
|
||||
print(
|
||||
f"OK num_reqs={nr} token_bias={tb} "
|
||||
f"has_idx_mapping={mapping} conv_dim_first={dim_first}"
|
||||
)
|
||||
from itertools import product
|
||||
|
||||
_CASES = product(
|
||||
(1, 4, 16), # num_reqs
|
||||
(0, 1, 2), # token_bias
|
||||
(True, False), # has_idx_mapping
|
||||
(False, True), # conv_state_dim_first
|
||||
(1, _TEMPORAL_TILES), # temporal_tiles
|
||||
)
|
||||
for nr, tb, mapping, dim_first, tt in _CASES:
|
||||
test_precopy_matches_v1_copy_specs(nr, tb, mapping, dim_first, tt)
|
||||
print(
|
||||
f"OK num_reqs={nr} token_bias={tb} has_idx_mapping={mapping} "
|
||||
f"conv_dim_first={dim_first} temporal_tiles={tt}"
|
||||
)
|
||||
|
||||
+176
-56
@@ -8,6 +8,7 @@ from typing import Any, NamedTuple
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
get_conv_copy_spec,
|
||||
@@ -22,6 +23,95 @@ from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.gpu_input_batch import CachedRequestState
|
||||
from vllm.v1.worker.lora_model_runner_mixin import GPUInputBatch
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# 16 saturates HBM on H100/GB200 across the reqs=8..128 range in
|
||||
# microbenchmarks
|
||||
_TEMPORAL_TILES = 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _memcpy_u64_tiled(
|
||||
src_addr,
|
||||
dst_addr,
|
||||
copy_size,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
NUM_TILES: tl.constexpr,
|
||||
):
|
||||
"""Head/body/tail memcpy with the u64 body split across ``NUM_TILES`` CTAs.
|
||||
|
||||
Fast path (``src`` and ``dst`` share sub-8B alignment): tile 0 owns the
|
||||
byte head that lifts dst to 8B and the 0-7 byte tail; body tiles vectorize
|
||||
as u64 over the aligned interior. ``NUM_TILES=1`` collapses to a single-
|
||||
CTA memcpy. Production callers derive both addresses from the same
|
||||
``state_base_addr + block_id * stride`` and always take this path.
|
||||
|
||||
Slow path (mismatched sub-8B alignment): byte-wide tiled copy. Some
|
||||
NVIDIA parts (e.g. GB200) reject misaligned u64 loads with
|
||||
``cudaErrorMisalignedAddress`` instead of accepting the misaligned-sector
|
||||
cost, so we can't just let the fast path run with a misaligned src.
|
||||
"""
|
||||
src_addr_i = src_addr.to(tl.int64)
|
||||
dst_addr_i = dst_addr.to(tl.int64)
|
||||
|
||||
if ((src_addr_i ^ dst_addr_i) & 7) == 0:
|
||||
head_bytes = tl.minimum(((-dst_addr_i) & 7).to(tl.int64), copy_size)
|
||||
if tile_idx == 0:
|
||||
head_off = tl.arange(0, 8)
|
||||
head_mask = head_off < head_bytes
|
||||
head_src = src_addr.to(tl.pointer_type(tl.uint8))
|
||||
head_dst = dst_addr.to(tl.pointer_type(tl.uint8))
|
||||
tl.store(
|
||||
head_dst + head_off,
|
||||
tl.load(head_src + head_off, mask=head_mask),
|
||||
mask=head_mask,
|
||||
)
|
||||
|
||||
# Body: u64 tiled. Rounding per_tile up to COPY_BLOCK_SIZE keeps every
|
||||
# inner-loop iteration full-width vectorized; only the last non-empty
|
||||
# tile can be masked. Late tiles fall off the end and iterate zero
|
||||
# times.
|
||||
body_bytes = copy_size - head_bytes
|
||||
body_u64 = body_bytes // 8
|
||||
per_tile_u64_raw = tl.cdiv(body_u64, NUM_TILES)
|
||||
per_tile_u64 = tl.cdiv(per_tile_u64_raw, COPY_BLOCK_SIZE) * COPY_BLOCK_SIZE
|
||||
tile_start = tile_idx.to(tl.int64) * per_tile_u64
|
||||
tile_end = tl.minimum(tile_start + per_tile_u64, body_u64)
|
||||
|
||||
src_body_u64 = (src_addr + head_bytes).to(tl.pointer_type(tl.uint64))
|
||||
dst_body_u64 = (dst_addr + head_bytes).to(tl.pointer_type(tl.uint64))
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for i in range(tile_start, tile_end, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < tile_end
|
||||
data = tl.load(src_body_u64 + i + offsets, mask=mask)
|
||||
tl.store(dst_body_u64 + i + offsets, data, mask=mask)
|
||||
|
||||
if tile_idx == 0:
|
||||
tail_start = head_bytes + body_u64 * 8
|
||||
tail_bytes = copy_size - tail_start
|
||||
tail_off = tl.arange(0, 8)
|
||||
tail_src = (src_addr + tail_start).to(tl.pointer_type(tl.uint8))
|
||||
tail_dst = (dst_addr + tail_start).to(tl.pointer_type(tl.uint8))
|
||||
tail_mask = tail_off < tail_bytes
|
||||
tl.store(
|
||||
tail_dst + tail_off,
|
||||
tl.load(tail_src + tail_off, mask=tail_mask),
|
||||
mask=tail_mask,
|
||||
)
|
||||
else:
|
||||
src_u8 = src_addr.to(tl.pointer_type(tl.uint8))
|
||||
dst_u8 = dst_addr.to(tl.pointer_type(tl.uint8))
|
||||
per_tile_bytes_raw = tl.cdiv(copy_size, NUM_TILES)
|
||||
per_tile_bytes = tl.cdiv(per_tile_bytes_raw, COPY_BLOCK_SIZE) * COPY_BLOCK_SIZE
|
||||
tile_start = tile_idx.to(tl.int64) * per_tile_bytes
|
||||
tile_end = tl.minimum(tile_start + per_tile_bytes, copy_size)
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for i in range(tile_start, tile_end, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < tile_end
|
||||
data = tl.load(src_u8 + i + offsets, mask=mask)
|
||||
tl.store(dst_u8 + i + offsets, data, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _copy_mamba_state_block(
|
||||
@@ -41,8 +131,10 @@ def _copy_mamba_state_block(
|
||||
# DS conv row metadata. Zero keeps the single-region copy path.
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
CONV_STATE_DIM_FIRST: tl.constexpr,
|
||||
TEMPORAL_TILES: tl.constexpr,
|
||||
):
|
||||
"""Copy one (layer, state-type) mamba state block between block columns.
|
||||
|
||||
@@ -57,6 +149,13 @@ def _copy_mamba_state_block(
|
||||
|
||||
The caller owns the decision logic (which columns, whether to copy); this
|
||||
device function only performs the byte copy for the given metadata slot.
|
||||
|
||||
``tile_idx`` in ``[0, TEMPORAL_TILES)`` partitions the temporal state's
|
||||
u64 range into ``TEMPORAL_TILES`` contiguous, COPY_BLOCK_SIZE-aligned
|
||||
slices, giving more CTAs to fill the SMs at small batch (multi-MiB
|
||||
temporal copies otherwise leave the GPU under-filled). Conv states are
|
||||
small; only ``tile_idx == 0`` copies them. ``TEMPORAL_TILES == 1`` and
|
||||
``tile_idx == 0`` reproduces the untiled behavior.
|
||||
"""
|
||||
state_base_addr = tl.load(state_base_addrs_ptr + state_idx)
|
||||
state_block_stride = tl.load(state_block_strides_ptr + state_idx)
|
||||
@@ -81,6 +180,10 @@ def _copy_mamba_state_block(
|
||||
is_conv_state = conv_width > 0
|
||||
|
||||
if CONV_STATE_DIM_FIRST and is_conv_state:
|
||||
# Conv states are small; only tile 0 does the copy. Higher tiles
|
||||
# early-return so they contribute nothing beyond a bounds check.
|
||||
if tile_idx > 0:
|
||||
return
|
||||
# DS conv layout: state_len is the slide axis; copy per dim row.
|
||||
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
|
||||
dim_rows = tl.load(state_dim_row_count_ptr + state_idx)
|
||||
@@ -101,53 +204,46 @@ def _copy_mamba_state_block(
|
||||
return
|
||||
|
||||
if is_conv_state:
|
||||
if tile_idx > 0:
|
||||
return
|
||||
# SD conv: copy
|
||||
# state[bt[src_col], token_bias:] ->
|
||||
# state[bt[dst_col], :conv_width - token_bias]
|
||||
# Small per-block bytes (~60-80 KiB) make tiling degenerate, so
|
||||
# conv runs as a single-CTA memcpy (NUM_TILES=1).
|
||||
src_block_id = tl.load(block_table_base + src_col).to(tl.int64)
|
||||
src_offset = token_bias.to(tl.int64) * state_inner_size * state_elem_size
|
||||
src_addr = state_base_addr + src_block_id * state_block_stride + src_offset
|
||||
num_elems_to_copy = (conv_width - token_bias).to(tl.int64) * state_inner_size
|
||||
copy_size = num_elems_to_copy * state_elem_size
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for i in range(0, copy_size, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < copy_size
|
||||
curr_src = (src_addr + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
curr_dst = (dst_addr + i + offsets).to(tl.pointer_type(tl.uint8))
|
||||
data = tl.load(curr_src, mask=mask)
|
||||
tl.store(curr_dst, data, mask=mask)
|
||||
copy_size = (
|
||||
(conv_width - token_bias).to(tl.int64) * state_inner_size * state_elem_size
|
||||
)
|
||||
_memcpy_u64_tiled(
|
||||
src_addr,
|
||||
dst_addr,
|
||||
copy_size,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE=COPY_BLOCK_SIZE,
|
||||
NUM_TILES=1,
|
||||
)
|
||||
return
|
||||
|
||||
# Temporal state: copy state[bt[src_col + token_bias]] -> state[bt[dst_col]]
|
||||
# Body u64 range is partitioned across TEMPORAL_TILES CTAs to keep the
|
||||
# SMs filled at small batch.
|
||||
actual_src_block_id = tl.load(block_table_base + src_col + token_bias).to(tl.int64)
|
||||
src_addr = state_base_addr + actual_src_block_id * state_block_stride
|
||||
# Use natural block data size (inner_size * elem_size), NOT
|
||||
# state_block_stride which is the page stride and can exceed the
|
||||
# actual data when the state tensor uses as_strided page padding.
|
||||
copy_size = state_inner_size * state_elem_size
|
||||
|
||||
# Vectorize via uint64 (8B per thread → LDG.64/STG.64): both temporal
|
||||
# and SD conv produce src/dst addresses aligned to a full token slice
|
||||
# (inner_size * elem_size) and a copy_size that's a multiple of it,
|
||||
# which is 8B-aligned for all state dtypes in use. A masked byte tail
|
||||
# covers any remaining 0-7 bytes (only reachable for sub-8B slices).
|
||||
copy_size_u64 = copy_size // 8
|
||||
src_u64 = src_addr.to(tl.pointer_type(tl.uint64))
|
||||
dst_u64 = dst_addr.to(tl.pointer_type(tl.uint64))
|
||||
offsets = tl.arange(0, COPY_BLOCK_SIZE)
|
||||
for i in range(0, copy_size_u64, COPY_BLOCK_SIZE):
|
||||
mask = (i + offsets) < copy_size_u64
|
||||
data = tl.load(src_u64 + i + offsets, mask=mask)
|
||||
tl.store(dst_u64 + i + offsets, data, mask=mask)
|
||||
|
||||
tail_start = copy_size_u64 * 8
|
||||
tail_bytes = copy_size - tail_start
|
||||
tail_off = tl.arange(0, 8)
|
||||
tail_src = (src_addr + tail_start).to(tl.pointer_type(tl.uint8))
|
||||
tail_dst = (dst_addr + tail_start).to(tl.pointer_type(tl.uint8))
|
||||
tail_mask = tail_off < tail_bytes
|
||||
tail_data = tl.load(tail_src + tail_off, mask=tail_mask)
|
||||
tl.store(tail_dst + tail_off, tail_data, mask=tail_mask)
|
||||
_memcpy_u64_tiled(
|
||||
src_addr,
|
||||
dst_addr,
|
||||
copy_size,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE=COPY_BLOCK_SIZE,
|
||||
NUM_TILES=TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_reqs"])
|
||||
@@ -194,14 +290,20 @@ def postprocess_mamba_fused_kernel(
|
||||
# PRECOMPUTED_NEW_COMPUTED: when True, num_computed_tokens_ptr already holds
|
||||
# the post-step new_num_computed value (V2 supplies the advanced count).
|
||||
PRECOMPUTED_NEW_COMPUTED: tl.constexpr = False,
|
||||
# TEMPORAL_TILES: when > 1, the temporal copy body is partitioned across
|
||||
# TEMPORAL_TILES CTAs along the u64 inner range. Callers must launch a
|
||||
# 3D grid (num_reqs, total_states, TEMPORAL_TILES). Default 1 preserves
|
||||
# the existing 2D-grid contract.
|
||||
TEMPORAL_TILES: tl.constexpr = 1,
|
||||
):
|
||||
"""
|
||||
Fused GPU kernel for postprocess_mamba that computes decisions AND performs
|
||||
mamba state copies without any CPU-GPU synchronization.
|
||||
|
||||
Grid: (num_reqs, num_layers * num_state_types)
|
||||
Grid: (num_reqs, num_layers * num_state_types [, TEMPORAL_TILES])
|
||||
- program_id(0) = request/batch index
|
||||
- program_id(1) = state_idx (flattened index into layer/state_type metadata)
|
||||
- program_id(2) = temporal-copy tile index (0 when TEMPORAL_TILES == 1)
|
||||
|
||||
Note: num_layers and num_state_types are not passed as kernel parameters
|
||||
because the kernel indexes directly into pre-flattened metadata arrays
|
||||
@@ -209,6 +311,7 @@ def postprocess_mamba_fused_kernel(
|
||||
"""
|
||||
batch_idx = tl.program_id(0)
|
||||
state_idx = tl.program_id(1)
|
||||
tile_idx = tl.program_id(2)
|
||||
|
||||
# Bounds check
|
||||
if batch_idx >= num_reqs:
|
||||
@@ -247,8 +350,9 @@ def postprocess_mamba_fused_kernel(
|
||||
dest_block_idx = aligned_new_computed // block_size - 1
|
||||
|
||||
# Update accepted-token count before early exits (per-request, so only
|
||||
# state_idx == 0 writes).
|
||||
if src_block_idx == dest_block_idx and state_idx == 0:
|
||||
# state_idx == 0 writes). Also guard on tile_idx == 0 so tiles > 0
|
||||
# (when TEMPORAL_TILES > 1) do not duplicate the store.
|
||||
if src_block_idx == dest_block_idx and state_idx == 0 and tile_idx == 0:
|
||||
tl.store(num_accepted_tokens_out_ptr + req_idx, 1)
|
||||
|
||||
# Skip no-op self-copy.
|
||||
@@ -272,8 +376,10 @@ def postprocess_mamba_fused_kernel(
|
||||
state_group_indices_ptr,
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE,
|
||||
CONV_STATE_DIM_FIRST,
|
||||
TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
|
||||
@@ -346,6 +452,9 @@ def precopy_mamba_align_fused_kernel(
|
||||
COPY_BLOCK_SIZE: tl.constexpr,
|
||||
CONV_STATE_DIM_FIRST: tl.constexpr,
|
||||
HAS_IDX_MAPPING: tl.constexpr = True,
|
||||
# TEMPORAL_TILES: see postprocess_mamba_fused_kernel. Default 1 preserves
|
||||
# the 2D-grid contract; > 1 requires a 3D grid.
|
||||
TEMPORAL_TILES: tl.constexpr = 1,
|
||||
):
|
||||
"""Pre-copy mamba "align" state across block boundaries.
|
||||
|
||||
@@ -357,12 +466,13 @@ def precopy_mamba_align_fused_kernel(
|
||||
copy specs), but driven by the GPU-resident src columns so it needs no
|
||||
CPU-GPU sync (async-scheduling safe).
|
||||
|
||||
Grid: (num_reqs, num_layers * num_state_types). V2 passes a batch-to-state
|
||||
idx_mapping; V1 already stores the staged arrays in batch order and uses
|
||||
HAS_IDX_MAPPING=False.
|
||||
Grid: (num_reqs, num_layers * num_state_types [, TEMPORAL_TILES]). V2 passes
|
||||
a batch-to-state idx_mapping; V1 already stores the staged arrays in batch
|
||||
order and uses HAS_IDX_MAPPING=False.
|
||||
"""
|
||||
batch_idx = tl.program_id(0)
|
||||
state_idx = tl.program_id(1)
|
||||
tile_idx = tl.program_id(2)
|
||||
if batch_idx >= num_reqs:
|
||||
return
|
||||
if HAS_IDX_MAPPING:
|
||||
@@ -397,8 +507,10 @@ def precopy_mamba_align_fused_kernel(
|
||||
state_group_indices_ptr,
|
||||
state_dim_row_count_ptr,
|
||||
state_dim_row_stride_ptr,
|
||||
tile_idx,
|
||||
COPY_BLOCK_SIZE,
|
||||
CONV_STATE_DIM_FIRST,
|
||||
TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
|
||||
@@ -701,23 +813,28 @@ class MambaSpecDecodeGPUContext:
|
||||
self.state_inner_sizes[idx] = (
|
||||
state[0].numel() if state.dim() > 1 else 1
|
||||
)
|
||||
# Temporal copies are vectorized with uint64
|
||||
# loads/stores; base pointer and block stride must
|
||||
# be 8B-aligned (tail loop handles copy_size % 8).
|
||||
# Temporal copies vectorize with uint64 loads/stores.
|
||||
# The kernel's head/tail handles misalignment for
|
||||
# correctness, but unaligned base/stride costs in
|
||||
# throughput.
|
||||
base_addr = state.data_ptr()
|
||||
block_stride_bytes = block_stride_elems * state.element_size()
|
||||
assert base_addr % 8 == 0, (
|
||||
f"layer {layer_name}: state.data_ptr() = "
|
||||
f"{base_addr:#x} is not 8B-aligned; "
|
||||
f"_copy_mamba_state_block uint64 "
|
||||
f"vectorization requires it"
|
||||
)
|
||||
assert block_stride_bytes % 8 == 0, (
|
||||
f"layer {layer_name}: block stride = "
|
||||
f"{block_stride_bytes}B is not 8B-aligned; "
|
||||
f"_copy_mamba_state_block uint64 "
|
||||
f"vectorization requires it"
|
||||
)
|
||||
if base_addr % 8 != 0:
|
||||
logger.warning_once(
|
||||
"layer %s: state.data_ptr() = %#x is not "
|
||||
"8B-aligned; _memcpy_u64_tiled uint64 "
|
||||
"vectorization will pay misaligned load cost",
|
||||
layer_name,
|
||||
base_addr,
|
||||
)
|
||||
if block_stride_bytes % 8 != 0:
|
||||
logger.warning_once(
|
||||
"layer %s: block stride = %dB is not "
|
||||
"8B-aligned; _memcpy_u64_tiled uint64 "
|
||||
"vectorization will pay misaligned load cost",
|
||||
layer_name,
|
||||
block_stride_bytes,
|
||||
)
|
||||
|
||||
self.state_group_indices[idx] = group_local_idx
|
||||
idx += 1
|
||||
@@ -771,7 +888,7 @@ class MambaSpecDecodeGPUContext:
|
||||
)
|
||||
|
||||
total_states = self.num_layers * self.num_state_types
|
||||
grid = (num_reqs, total_states)
|
||||
grid = (num_reqs, total_states, _TEMPORAL_TILES)
|
||||
|
||||
postprocess_mamba_fused_kernel[grid](
|
||||
num_accepted_tokens_gpu,
|
||||
@@ -795,6 +912,7 @@ class MambaSpecDecodeGPUContext:
|
||||
block_size=self.block_size,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
TEMPORAL_TILES=_TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
def run_fused_precopy(
|
||||
@@ -819,7 +937,7 @@ class MambaSpecDecodeGPUContext:
|
||||
if num_reqs == 0 or not self.is_initialized:
|
||||
return
|
||||
total_states = self.num_layers * self.num_state_types
|
||||
grid = (num_reqs, total_states)
|
||||
grid = (num_reqs, total_states, _TEMPORAL_TILES)
|
||||
precopy_mamba_align_fused_kernel[grid](
|
||||
state_idx_gpu,
|
||||
src_col_gpu,
|
||||
@@ -839,6 +957,7 @@ class MambaSpecDecodeGPUContext:
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
HAS_IDX_MAPPING=idx_mapping is not None,
|
||||
TEMPORAL_TILES=_TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
def run_fused_postprocess_align(
|
||||
@@ -868,7 +987,7 @@ class MambaSpecDecodeGPUContext:
|
||||
num_accepted_tokens_snapshot.copy_(num_accepted_tokens_gpu)
|
||||
|
||||
total_states = self.num_layers * self.num_state_types
|
||||
grid = (num_reqs, total_states)
|
||||
grid = (num_reqs, total_states, _TEMPORAL_TILES)
|
||||
postprocess_mamba_fused_kernel[grid](
|
||||
num_accepted_tokens_snapshot,
|
||||
state_idx_gpu,
|
||||
@@ -893,6 +1012,7 @@ class MambaSpecDecodeGPUContext:
|
||||
CONV_STATE_DIM_FIRST=is_conv_state_dim_first(),
|
||||
HAS_IDX_MAPPING=True,
|
||||
PRECOMPUTED_NEW_COMPUTED=True,
|
||||
TEMPORAL_TILES=_TEMPORAL_TILES,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user