[ROCm] [DSv4] [Perf] Support DeepSeek v4 MTP (#43385)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2026-05-24 18:43:08 +08:00
committed by GitHub
parent 5940590855
commit 1806d1adfc
6 changed files with 2340 additions and 52 deletions
+46 -16
View File
@@ -144,35 +144,57 @@ def _cp_gather_indexer_quant_cache_kernel(
HEAD_DIM: tl.constexpr,
BLOCK_TILE_SIZE: tl.constexpr,
HEAD_TILE_SIZE: tl.constexpr,
NUM_TOKENS: tl.constexpr,
NUM_BATCHES: tl.constexpr,
BLOCK_TABLE_WIDTH: tl.constexpr,
NUM_BLOCKS: tl.constexpr,
):
tid = tl.program_id(0)
offset = tl.arange(0, HEAD_DIM)
batch_id = tl.load(token_to_seq_ptr + tid)
batch_start = tl.load(cu_seqlen_ptr + batch_id)
batch_end = tl.load(cu_seqlen_ptr + batch_id + 1)
valid_tid = tid < NUM_TOKENS
batch_id = tl.load(token_to_seq_ptr + tid, mask=valid_tid, other=-1)
valid_batch = (batch_id >= 0) & (batch_id < NUM_BATCHES)
safe_batch_id = tl.where(valid_batch, batch_id, 0)
batch_start = tl.load(cu_seqlen_ptr + safe_batch_id, mask=valid_batch, other=0)
batch_end = tl.load(cu_seqlen_ptr + safe_batch_id + 1, mask=valid_batch, other=0)
batch_offset = tid - batch_start
if tid >= batch_end:
valid_token = valid_tid & valid_batch & (tid >= batch_start) & (tid < batch_end)
if not valid_token:
return
block_table_id = batch_offset // block_size
block_offset = batch_offset % block_size
block_table_offset = batch_id * block_table_stride + block_table_id
block_id = tl.load(block_table_ptr + block_table_offset)
tiled_block_id = block_offset // BLOCK_TILE_SIZE
tiled_block_offset = block_offset % BLOCK_TILE_SIZE
valid_block_table = (
valid_token
& (block_table_id >= 0)
& (block_table_id < BLOCK_TABLE_WIDTH)
& (block_offset >= 0)
& (block_offset < block_size)
)
safe_block_table_id = tl.where(valid_block_table, block_table_id, 0)
block_table_offset = safe_batch_id * block_table_stride + safe_block_table_id
block_id = tl.load(
block_table_ptr + block_table_offset, mask=valid_block_table, other=-1
)
valid_block = valid_block_table & (block_id >= 0) & (block_id < NUM_BLOCKS)
safe_block_id = tl.where(valid_block, block_id, 0)
safe_block_offset = tl.where(valid_block, block_offset, 0)
tiled_block_offset = safe_block_offset % BLOCK_TILE_SIZE
if LAYOUT == "SHUFFLE":
src_cache_offset = (
block_id * kv_cache_stride
+ tiled_block_id * HEAD_DIM * BLOCK_TILE_SIZE
safe_block_id * kv_cache_stride
+ (safe_block_offset // BLOCK_TILE_SIZE) * HEAD_DIM * BLOCK_TILE_SIZE
+ tiled_block_offset * HEAD_TILE_SIZE
)
else:
src_cache_offset = block_id * kv_cache_stride + block_offset * HEAD_DIM
src_scale_offset = block_id * kv_cache_scale_stride + block_offset
src_cache_offset = (
safe_block_id * kv_cache_stride + safe_block_offset * HEAD_DIM
)
src_scale_offset = safe_block_id * kv_cache_scale_stride + safe_block_offset
dst_offset = tid * HEAD_DIM
src_scale_ptr = kv_cache_scale_ptr + src_scale_offset
src_cache_ptr = kv_cache_ptr + src_cache_offset
dst_k_ptr = k_fp8_ptr + dst_offset
scale_val = tl.load(src_scale_ptr)
scale_val = tl.load(src_scale_ptr, mask=valid_block, other=0.0)
tl.store(k_scale_ptr + tid, scale_val)
if LAYOUT == "SHUFFLE":
tiled_src_offset = (
@@ -182,7 +204,7 @@ def _cp_gather_indexer_quant_cache_kernel(
else:
tiled_src_offset = offset
val = tl.load(src_cache_ptr + tiled_src_offset)
tl.store(dst_k_ptr + offset, val)
tl.store(dst_k_ptr + offset, val, mask=valid_block)
def cp_gather_indexer_k_quant_cache_triton(
@@ -223,6 +245,10 @@ def cp_gather_indexer_k_quant_cache_triton(
head_dim,
block_tile_size,
head_tile_size,
num_tokens,
cu_seqlen.shape[0] - 1,
block_table.shape[1],
num_blocks,
)
@@ -943,8 +969,9 @@ def _pack_dense_prefix_to_ragged_kernel(
return
mask = offsets < row_len
safe_offsets = tl.where(offsets < row_width, offsets, 0)
vals = tl.load(
indices_ptr + row_idx * indices_stride0 + offsets,
indices_ptr + row_idx * indices_stride0 + safe_offsets,
mask=mask & (offsets < row_width),
other=-1,
).to(tl.int32)
@@ -1060,9 +1087,12 @@ def _sparse_attn_prefill_ragged_kernel(
in_range = k_pos < kv_len
slot = tl.load(kv_indices_ptr + kv_start + k_pos, mask=in_range, other=-1)
valid = in_range & (slot >= 0) & (slot < num_kv)
safe_slot = tl.where(valid, slot, 0)
kv = tl.load(
kv_ptr + slot[:, None] * kv_stride_n + dim_offsets[None, :] * kv_stride_d,
kv_ptr
+ safe_slot[:, None] * kv_stride_n
+ dim_offsets[None, :] * kv_stride_d,
mask=valid[:, None] & dim_mask[None, :],
other=0.0,
)
+6
View File
@@ -240,6 +240,10 @@ class SpecDecodeBaseProposer:
# Determine allowed attention backends once during initialization.
self.allowed_attn_types: tuple | None = None
if current_platform.is_rocm():
from vllm.models.deepseek_v4.amd.rocm import (
DeepseekV4ROCMAiterMLASparseMetadata,
DeepseekV4ROCMAiterSparseSWAMetadata,
)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerMetadata,
)
@@ -252,6 +256,8 @@ class SpecDecodeBaseProposer:
TritonAttentionMetadata,
RocmAttentionMetadata,
ROCMAiterMLASparseMetadata,
DeepseekV4ROCMAiterMLASparseMetadata,
DeepseekV4ROCMAiterSparseSWAMetadata,
DeepseekV32IndexerMetadata,
]
# ROCM_AITER_FA is an optional backend