[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-11 19:30:34 -07:00
committed by GitHub
parent 23f25aac14
commit f97e502969
15 changed files with 116 additions and 56 deletions
+8 -2
View File
@@ -539,8 +539,14 @@ def make_kv_sharing_fast_prefill_common_attn_metadata(
decode_query_start_loc[:1].fill_(0) # Avoid sync from scalar assignment.
decode_query_start_loc[1:] = torch.cumsum(num_decode_tokens, dim=0)
decode_max_query_len = int(num_decode_tokens.max().item())
total_num_decode_tokens = int(num_decode_tokens.sum().item())
# `num_decode_tokens` is a histogram over `logits_indices`, so its total is
# just how many there were -- already known as a Python int.
total_num_decode_tokens = num_logits_indices
# Largest per-request logits count.
decode_max_query_len = common_attn_metadata.max_logits_per_req
assert decode_max_query_len is not None
common_attn_metadata = CommonAttentionMetadata(
query_start_loc=decode_query_start_loc,