[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-11 20:36:03 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 53181384e0
commit bbee532988
7 changed files with 49 additions and 33 deletions
+4 -2
View File
@@ -283,8 +283,10 @@ def bincount(
output_bin_counts: torch.Tensor,
max_prefill_len: int,
) -> None:
prompt_bin_mask[expanded_idx_mapping] = 0
output_bin_counts[expanded_idx_mapping] = 0
# Use index_fill_ instead of `tensor[idx] = 0` to avoid sync.
idx_long = expanded_idx_mapping.long()
prompt_bin_mask.index_fill_(0, idx_long, 0)
output_bin_counts.index_fill_(0, idx_long, 0)
num_tokens = expanded_idx_mapping.shape[0]
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(max_prefill_len, BLOCK_SIZE)