[Perf][DSv4/DSv3.2] Add cluster-cooperative topK kernel for low-latency scenarios (#43008)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
This commit is contained in:
Roberto L. Castro
2026-06-23 16:11:00 -07:00
committed by GitHub
parent 3cc871aaf1
commit 855cd4d787
10 changed files with 1568 additions and 45 deletions
@@ -334,7 +334,32 @@ def sparse_attn_indexer(
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
if current_platform.is_cuda() and topk_tokens in (512, 1024, 2048):
use_cooperative_topk = (
current_platform.is_cuda()
and topk_tokens in (512, 1024, 2048)
and num_rows <= 32
and logits.stride(0) % 4 == 0 # TMA 16-byte alignment
and current_platform.has_device_capability(90)
)
use_persistent_topk = current_platform.is_cuda() and topk_tokens in (
512,
1024,
2048,
)
if use_cooperative_topk:
workspace_manager = current_workspace_manager()
(topk_workspace,) = workspace_manager.get_simultaneous(
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
torch.ops._C.cooperative_topk(
logits,
seq_lens,
topk_indices,
topk_workspace,
topk_tokens,
attn_metadata_narrowed.max_seq_len,
)
elif use_persistent_topk:
workspace_manager = current_workspace_manager()
(topk_workspace,) = workspace_manager.get_simultaneous(
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),