[DSv4 Perf] Adaptive topk width for dsv4, making #50004 back (#52823)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-21 12:11:45 -07:00
committed by GitHub
parent d6c2fec9fd
commit e6f35d3c69
2 changed files with 78 additions and 3 deletions
+21 -3
View File
@@ -257,6 +257,15 @@ class DeepseekV4SparseMLAMetadataBuilder(
assert cm.positions is not None, (
"positions is required for C128A metadata build"
)
active_topk_width = min(
max(
triton.next_power_of_2(max(cm.max_seq_len // self.compress_ratio, 1)),
_C128A_TOPK_ALIGNMENT,
),
self.c128a_max_compressed,
)
assert active_topk_width >= cm.max_seq_len // self.compress_ratio
assert active_topk_width % _C128A_TOPK_ALIGNMENT == 0
block_size = self.kv_cache_spec.block_size // self.compress_ratio
global_decode, decode_lens, prefill_local = build_c128a_topk_metadata(
cm.positions[:num_total],
@@ -269,7 +278,7 @@ class DeepseekV4SparseMLAMetadataBuilder(
self.c128a_global_decode_buffer,
self.c128a_decode_lens_buffer,
self.c128a_prefill_buffer,
max_compressed_tokens=self.c128a_max_compressed,
max_compressed_tokens=active_topk_width,
)
result: dict[str, torch.Tensor | None] = {}
@@ -320,10 +329,19 @@ def build_c128a_topk_metadata(
"""
num_tokens = positions.shape[0]
num_prefill_tokens = num_tokens - num_decode_tokens
assert max_compressed_tokens % _C128A_TOPK_ALIGNMENT == 0
assert (
0
< max_compressed_tokens
<= min(global_decode_buffer.shape[1], prefill_buffer.shape[1])
)
assert global_decode_buffer.stride(-1) == prefill_buffer.stride(-1) == 1
global_decode = global_decode_buffer[:num_decode_tokens]
global_decode = global_decode_buffer[:num_decode_tokens, :max_compressed_tokens]
decode_lens = decode_lens_buffer[:num_decode_tokens]
prefill_local = prefill_buffer[:num_prefill_tokens]
prefill_local = prefill_buffer[:num_prefill_tokens, :max_compressed_tokens]
assert global_decode.stride(0) == global_decode_buffer.stride(0)
assert prefill_local.stride(0) == prefill_buffer.stride(0)
if num_tokens == 0:
return global_decode, decode_lens, prefill_local