[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-05-11 20:36:03 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 53181384e0
commit bbee532988
7 changed files with 49 additions and 33 deletions
+15 -9
View File
@@ -995,16 +995,23 @@ class GPUModelRunner(
if len(token_type_id_requests) == 0:
return model_kwargs
seq_lens = self.seq_lens[:num_reqs]
# Build ids on CPU using the CPU-resident upper bound for seq_lens;
# `torch.arange(seq_lens[i])` with a GPU scalar would force a sync.
seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs].tolist()
token_type_ids = []
for i in range(num_reqs):
pos = token_type_id_requests.get(i, seq_lens[i])
ids = (torch.arange(seq_lens[i]) >= pos).int()
seq_len_i = seq_lens_cpu[i]
pos = token_type_id_requests.get(i, seq_len_i)
ids = (torch.arange(seq_len_i) >= pos).int()
token_type_ids.append(ids)
model_kwargs["token_type_ids"] = torch.concat(token_type_ids).to(
device=self.device
token_type_ids_cpu = torch.empty(
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory
)
torch.cat(token_type_ids, out=token_type_ids_cpu)
model_kwargs["token_type_ids"] = token_type_ids_cpu.to(
device=self.device, non_blocking=True
)
return model_kwargs
@@ -2715,10 +2722,9 @@ class GPUModelRunner(
# There might have leftover indices in logits_indices[num_logits:]
# from previous iterations, whose values may be greater than the
# batch size in the current iteration. To ensure indices are always
# valid, we fill the padded indices with the last index.
self.kv_sharing_fast_prefill_logits_indices[num_logits:].fill_(
logits_indices[-1].item()
)
# valid, fill the padded indices with the last index. Broadcast the
# scalar GPU-side to avoid a D2H sync on `.item()`.
self.kv_sharing_fast_prefill_logits_indices[num_logits:] = logits_indices[-1]
# Dispatch for the decoder portion of the model.
_, batch_desc = self.cudagraph_dispatcher.dispatch(
num_logits, invalid_modes={CUDAGraphMode.FULL}