[Perf][1/n] Eliminate various GPU<->CPU syncs (#41429)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
53181384e0
commit
bbee532988
@@ -995,16 +995,23 @@ class GPUModelRunner(
|
||||
if len(token_type_id_requests) == 0:
|
||||
return model_kwargs
|
||||
|
||||
seq_lens = self.seq_lens[:num_reqs]
|
||||
# Build ids on CPU using the CPU-resident upper bound for seq_lens;
|
||||
# `torch.arange(seq_lens[i])` with a GPU scalar would force a sync.
|
||||
seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs].tolist()
|
||||
token_type_ids = []
|
||||
|
||||
for i in range(num_reqs):
|
||||
pos = token_type_id_requests.get(i, seq_lens[i])
|
||||
ids = (torch.arange(seq_lens[i]) >= pos).int()
|
||||
seq_len_i = seq_lens_cpu[i]
|
||||
pos = token_type_id_requests.get(i, seq_len_i)
|
||||
ids = (torch.arange(seq_len_i) >= pos).int()
|
||||
token_type_ids.append(ids)
|
||||
|
||||
model_kwargs["token_type_ids"] = torch.concat(token_type_ids).to(
|
||||
device=self.device
|
||||
token_type_ids_cpu = torch.empty(
|
||||
sum(seq_lens_cpu), dtype=torch.int32, pin_memory=self.pin_memory
|
||||
)
|
||||
torch.cat(token_type_ids, out=token_type_ids_cpu)
|
||||
model_kwargs["token_type_ids"] = token_type_ids_cpu.to(
|
||||
device=self.device, non_blocking=True
|
||||
)
|
||||
return model_kwargs
|
||||
|
||||
@@ -2715,10 +2722,9 @@ class GPUModelRunner(
|
||||
# There might have leftover indices in logits_indices[num_logits:]
|
||||
# from previous iterations, whose values may be greater than the
|
||||
# batch size in the current iteration. To ensure indices are always
|
||||
# valid, we fill the padded indices with the last index.
|
||||
self.kv_sharing_fast_prefill_logits_indices[num_logits:].fill_(
|
||||
logits_indices[-1].item()
|
||||
)
|
||||
# valid, fill the padded indices with the last index. Broadcast the
|
||||
# scalar GPU-side to avoid a D2H sync on `.item()`.
|
||||
self.kv_sharing_fast_prefill_logits_indices[num_logits:] = logits_indices[-1]
|
||||
# Dispatch for the decoder portion of the model.
|
||||
_, batch_desc = self.cudagraph_dispatcher.dispatch(
|
||||
num_logits, invalid_modes={CUDAGraphMode.FULL}
|
||||
|
||||
Reference in New Issue
Block a user