[DSv4 Perf] Adaptive topk width, 1.0% E2E throughput improvement (#50004)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -261,6 +261,13 @@ class DeepseekV4FlashMLAMetadataBuilder(
|
||||
assert cm.positions is not None, (
|
||||
"positions is required for C128A metadata build"
|
||||
)
|
||||
active_topk_width = min(
|
||||
max(
|
||||
triton.next_power_of_2(max(cm.max_seq_len // self.compress_ratio, 1)),
|
||||
_C128A_TOPK_ALIGNMENT,
|
||||
),
|
||||
self.c128a_max_compressed,
|
||||
)
|
||||
block_size = self.kv_cache_spec.block_size // self.compress_ratio
|
||||
global_decode, decode_lens, prefill_local = build_c128a_topk_metadata(
|
||||
cm.positions[:num_total],
|
||||
@@ -273,7 +280,7 @@ class DeepseekV4FlashMLAMetadataBuilder(
|
||||
self.c128a_global_decode_buffer,
|
||||
self.c128a_decode_lens_buffer,
|
||||
self.c128a_prefill_buffer,
|
||||
max_compressed_tokens=self.c128a_max_compressed,
|
||||
max_compressed_tokens=active_topk_width,
|
||||
)
|
||||
|
||||
result: dict[str, torch.Tensor | None] = {}
|
||||
@@ -305,25 +312,30 @@ def build_c128a_topk_metadata(
|
||||
Decode tokens: position → block_table lookup → global slot ids + topk_lens.
|
||||
Prefill tokens: position → local indices [0, ..., n-1, -1, ...].
|
||||
|
||||
Writes into pre-allocated buffers for CUDA graph address stability.
|
||||
Returns slices of the buffers.
|
||||
Writes into packed views of pre-allocated buffers for CUDA graph stability.
|
||||
"""
|
||||
num_tokens = positions.shape[0]
|
||||
num_prefill_tokens = num_tokens - num_decode_tokens
|
||||
|
||||
global_decode = global_decode_buffer[:num_decode_tokens]
|
||||
# view(-1) as 1-d array and then expanded to
|
||||
# [num_decode_tokens, max_compressed_tokens]
|
||||
global_decode = global_decode_buffer.view(-1)[
|
||||
: num_decode_tokens * max_compressed_tokens
|
||||
].view(num_decode_tokens, max_compressed_tokens)
|
||||
decode_lens = decode_lens_buffer[:num_decode_tokens]
|
||||
prefill_local = prefill_buffer[:num_prefill_tokens]
|
||||
prefill_local = prefill_buffer.view(-1)[
|
||||
: num_prefill_tokens * max_compressed_tokens
|
||||
].view(num_prefill_tokens, max_compressed_tokens)
|
||||
|
||||
if num_tokens == 0:
|
||||
return global_decode, decode_lens, prefill_local
|
||||
|
||||
_build_c128a_topk_metadata_kernel[(num_tokens,)](
|
||||
global_decode_buffer,
|
||||
global_decode_buffer.stride(0),
|
||||
max_compressed_tokens,
|
||||
decode_lens_buffer,
|
||||
prefill_buffer,
|
||||
prefill_buffer.stride(0),
|
||||
max_compressed_tokens,
|
||||
positions,
|
||||
compress_ratio,
|
||||
max_compressed_tokens,
|
||||
|
||||
Reference in New Issue
Block a user