[ROCm] [DSv4] [Perf] Support DeepSeek v4 MTP (#43385)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -144,35 +144,57 @@ def _cp_gather_indexer_quant_cache_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_TILE_SIZE: tl.constexpr,
|
||||
HEAD_TILE_SIZE: tl.constexpr,
|
||||
NUM_TOKENS: tl.constexpr,
|
||||
NUM_BATCHES: tl.constexpr,
|
||||
BLOCK_TABLE_WIDTH: tl.constexpr,
|
||||
NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
tid = tl.program_id(0)
|
||||
offset = tl.arange(0, HEAD_DIM)
|
||||
batch_id = tl.load(token_to_seq_ptr + tid)
|
||||
batch_start = tl.load(cu_seqlen_ptr + batch_id)
|
||||
batch_end = tl.load(cu_seqlen_ptr + batch_id + 1)
|
||||
valid_tid = tid < NUM_TOKENS
|
||||
batch_id = tl.load(token_to_seq_ptr + tid, mask=valid_tid, other=-1)
|
||||
valid_batch = (batch_id >= 0) & (batch_id < NUM_BATCHES)
|
||||
safe_batch_id = tl.where(valid_batch, batch_id, 0)
|
||||
batch_start = tl.load(cu_seqlen_ptr + safe_batch_id, mask=valid_batch, other=0)
|
||||
batch_end = tl.load(cu_seqlen_ptr + safe_batch_id + 1, mask=valid_batch, other=0)
|
||||
batch_offset = tid - batch_start
|
||||
if tid >= batch_end:
|
||||
valid_token = valid_tid & valid_batch & (tid >= batch_start) & (tid < batch_end)
|
||||
if not valid_token:
|
||||
return
|
||||
block_table_id = batch_offset // block_size
|
||||
block_offset = batch_offset % block_size
|
||||
block_table_offset = batch_id * block_table_stride + block_table_id
|
||||
block_id = tl.load(block_table_ptr + block_table_offset)
|
||||
tiled_block_id = block_offset // BLOCK_TILE_SIZE
|
||||
tiled_block_offset = block_offset % BLOCK_TILE_SIZE
|
||||
valid_block_table = (
|
||||
valid_token
|
||||
& (block_table_id >= 0)
|
||||
& (block_table_id < BLOCK_TABLE_WIDTH)
|
||||
& (block_offset >= 0)
|
||||
& (block_offset < block_size)
|
||||
)
|
||||
safe_block_table_id = tl.where(valid_block_table, block_table_id, 0)
|
||||
block_table_offset = safe_batch_id * block_table_stride + safe_block_table_id
|
||||
block_id = tl.load(
|
||||
block_table_ptr + block_table_offset, mask=valid_block_table, other=-1
|
||||
)
|
||||
valid_block = valid_block_table & (block_id >= 0) & (block_id < NUM_BLOCKS)
|
||||
safe_block_id = tl.where(valid_block, block_id, 0)
|
||||
safe_block_offset = tl.where(valid_block, block_offset, 0)
|
||||
tiled_block_offset = safe_block_offset % BLOCK_TILE_SIZE
|
||||
if LAYOUT == "SHUFFLE":
|
||||
src_cache_offset = (
|
||||
block_id * kv_cache_stride
|
||||
+ tiled_block_id * HEAD_DIM * BLOCK_TILE_SIZE
|
||||
safe_block_id * kv_cache_stride
|
||||
+ (safe_block_offset // BLOCK_TILE_SIZE) * HEAD_DIM * BLOCK_TILE_SIZE
|
||||
+ tiled_block_offset * HEAD_TILE_SIZE
|
||||
)
|
||||
else:
|
||||
src_cache_offset = block_id * kv_cache_stride + block_offset * HEAD_DIM
|
||||
src_scale_offset = block_id * kv_cache_scale_stride + block_offset
|
||||
src_cache_offset = (
|
||||
safe_block_id * kv_cache_stride + safe_block_offset * HEAD_DIM
|
||||
)
|
||||
src_scale_offset = safe_block_id * kv_cache_scale_stride + safe_block_offset
|
||||
dst_offset = tid * HEAD_DIM
|
||||
src_scale_ptr = kv_cache_scale_ptr + src_scale_offset
|
||||
src_cache_ptr = kv_cache_ptr + src_cache_offset
|
||||
dst_k_ptr = k_fp8_ptr + dst_offset
|
||||
scale_val = tl.load(src_scale_ptr)
|
||||
scale_val = tl.load(src_scale_ptr, mask=valid_block, other=0.0)
|
||||
tl.store(k_scale_ptr + tid, scale_val)
|
||||
if LAYOUT == "SHUFFLE":
|
||||
tiled_src_offset = (
|
||||
@@ -182,7 +204,7 @@ def _cp_gather_indexer_quant_cache_kernel(
|
||||
else:
|
||||
tiled_src_offset = offset
|
||||
val = tl.load(src_cache_ptr + tiled_src_offset)
|
||||
tl.store(dst_k_ptr + offset, val)
|
||||
tl.store(dst_k_ptr + offset, val, mask=valid_block)
|
||||
|
||||
|
||||
def cp_gather_indexer_k_quant_cache_triton(
|
||||
@@ -223,6 +245,10 @@ def cp_gather_indexer_k_quant_cache_triton(
|
||||
head_dim,
|
||||
block_tile_size,
|
||||
head_tile_size,
|
||||
num_tokens,
|
||||
cu_seqlen.shape[0] - 1,
|
||||
block_table.shape[1],
|
||||
num_blocks,
|
||||
)
|
||||
|
||||
|
||||
@@ -943,8 +969,9 @@ def _pack_dense_prefix_to_ragged_kernel(
|
||||
return
|
||||
|
||||
mask = offsets < row_len
|
||||
safe_offsets = tl.where(offsets < row_width, offsets, 0)
|
||||
vals = tl.load(
|
||||
indices_ptr + row_idx * indices_stride0 + offsets,
|
||||
indices_ptr + row_idx * indices_stride0 + safe_offsets,
|
||||
mask=mask & (offsets < row_width),
|
||||
other=-1,
|
||||
).to(tl.int32)
|
||||
@@ -1060,9 +1087,12 @@ def _sparse_attn_prefill_ragged_kernel(
|
||||
in_range = k_pos < kv_len
|
||||
slot = tl.load(kv_indices_ptr + kv_start + k_pos, mask=in_range, other=-1)
|
||||
valid = in_range & (slot >= 0) & (slot < num_kv)
|
||||
safe_slot = tl.where(valid, slot, 0)
|
||||
|
||||
kv = tl.load(
|
||||
kv_ptr + slot[:, None] * kv_stride_n + dim_offsets[None, :] * kv_stride_d,
|
||||
kv_ptr
|
||||
+ safe_slot[:, None] * kv_stride_n
|
||||
+ dim_offsets[None, :] * kv_stride_d,
|
||||
mask=valid[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
@@ -240,6 +240,10 @@ class SpecDecodeBaseProposer:
|
||||
# Determine allowed attention backends once during initialization.
|
||||
self.allowed_attn_types: tuple | None = None
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
DeepseekV4ROCMAiterMLASparseMetadata,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerMetadata,
|
||||
)
|
||||
@@ -252,6 +256,8 @@ class SpecDecodeBaseProposer:
|
||||
TritonAttentionMetadata,
|
||||
RocmAttentionMetadata,
|
||||
ROCMAiterMLASparseMetadata,
|
||||
DeepseekV4ROCMAiterMLASparseMetadata,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
DeepseekV32IndexerMetadata,
|
||||
]
|
||||
# ROCM_AITER_FA is an optional backend
|
||||
|
||||
Reference in New Issue
Block a user