[ROCm] [DSv4] [Perf] Support DeepSeek v4 MTP (#43385)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -44,6 +44,127 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
||||
return indptr
|
||||
|
||||
|
||||
# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes
|
||||
# do not touch the shared DeepSeek V4 cache utilities.
|
||||
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _combine_topk_swa_indices_kernel(
|
||||
combined_indices_ptr,
|
||||
combined_indices_stride,
|
||||
combined_lens_ptr,
|
||||
topk_indices_ptr,
|
||||
topk_indices_stride,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
gather_lens_ptr,
|
||||
M,
|
||||
N,
|
||||
TOP_K: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
TOPK_WIDTH: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
query_len = query_end - query_start
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
start_pos = seq_len - query_len
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
token_idx_in_query = token_idx - query_start
|
||||
pos = start_pos + token_idx_in_query
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
|
||||
topk_offset = tl.arange(0, PADDED_TOP_K)
|
||||
topk_mask = topk_offset < topk_len
|
||||
safe_topk_offset = tl.where(topk_offset < TOPK_WIDTH, topk_offset, 0)
|
||||
topk_indices = tl.load(
|
||||
topk_indices_ptr + token_idx * topk_indices_stride + safe_topk_offset,
|
||||
mask=topk_mask,
|
||||
other=-1,
|
||||
)
|
||||
valid_topk = (topk_indices >= 0) & (topk_indices < N)
|
||||
topk_indices = tl.where(valid_topk, topk_indices + M * batch_idx, -1)
|
||||
tl.store(
|
||||
combined_indices_ptr + token_idx * combined_indices_stride + topk_offset,
|
||||
topk_indices,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
swa_offset = tl.arange(0, WINDOW_SIZE)
|
||||
tl.store(
|
||||
combined_indices_ptr
|
||||
+ token_idx * combined_indices_stride
|
||||
+ topk_len
|
||||
+ swa_offset,
|
||||
M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start,
|
||||
mask=swa_offset < swa_len,
|
||||
)
|
||||
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
|
||||
|
||||
def combine_topk_swa_indices(
|
||||
topk_indices: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
gather_lens: torch.Tensor,
|
||||
window_size: int,
|
||||
compress_ratio: int,
|
||||
topk: int,
|
||||
M: int,
|
||||
N: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
|
||||
num_tokens = topk_indices.shape[0]
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_topk = (
|
||||
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
|
||||
// _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||
* _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||
)
|
||||
combined_indices = torch.full(
|
||||
(num_tokens, combined_topk),
|
||||
fill_value=-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
|
||||
num_workers = 128
|
||||
_combine_topk_swa_indices_kernel[(num_reqs, num_workers)](
|
||||
combined_indices,
|
||||
combined_indices.stride(0),
|
||||
combined_lens,
|
||||
topk_indices,
|
||||
topk_indices.stride(0),
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
M,
|
||||
N,
|
||||
TOP_K=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
TOPK_WIDTH=topk_indices.shape[-1],
|
||||
PADDED_TOP_K=triton.next_power_of_2(topk_indices.shape[-1]),
|
||||
)
|
||||
return combined_indices, combined_lens
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_topk_lens_kernel(
|
||||
topk_lens_ptr,
|
||||
@@ -704,31 +825,23 @@ class DeepseekV4ROCMAiterMLASparseImpl(DeepseekV4SparseMLAAttentionImpl):
|
||||
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
||||
)
|
||||
|
||||
combined_ragged_indices, combined_ragged_indptr, combined_lens = (
|
||||
combine_topk_swa_indices_ragged(
|
||||
topk_indices[query_start:query_end],
|
||||
query_start_loc[
|
||||
num_decodes + chunk_start : num_decodes + chunk_end + 1
|
||||
],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
layer.window_size,
|
||||
layer.compress_ratio,
|
||||
top_k,
|
||||
M,
|
||||
N,
|
||||
)
|
||||
combined_indices, combined_lens = combine_topk_swa_indices(
|
||||
topk_indices[query_start:query_end],
|
||||
query_start_loc[
|
||||
num_decodes + chunk_start : num_decodes + chunk_end + 1
|
||||
],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
layer.window_size,
|
||||
layer.compress_ratio,
|
||||
top_k,
|
||||
M,
|
||||
N,
|
||||
)
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=torch.empty(
|
||||
q[query_start:query_end].shape[0],
|
||||
1,
|
||||
0,
|
||||
dtype=torch.int32,
|
||||
device=q.device,
|
||||
),
|
||||
indices=combined_indices,
|
||||
topk_length=combined_lens,
|
||||
scale=layer.scale,
|
||||
head_dim=layer.head_dim,
|
||||
@@ -736,6 +849,4 @@ class DeepseekV4ROCMAiterMLASparseImpl(DeepseekV4SparseMLAAttentionImpl):
|
||||
rope_head_dim=layer.rope_head_dim,
|
||||
attn_sink=layer.attn_sink,
|
||||
output=output[query_start:query_end],
|
||||
ragged_indices=combined_ragged_indices,
|
||||
ragged_indptr=combined_ragged_indptr,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user