[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-19 10:35:54 -04:00
committed by GitHub
parent a78b842d0e
commit b82e908b4c
23 changed files with 129 additions and 108 deletions
+8 -8
View File
@@ -1765,8 +1765,6 @@ class GPUModelRunner(
self.input_batch.prev_sampled_token_ids[:num_common_tokens, 0],
non_blocking=True,
)
if self.enable_prompt_embeds:
self.is_token_ids.gpu[:num_common_tokens] = True
return
# Upload the index tensors asynchronously so the scatter can be non-blocking.
sampled_tokens_index_tensor = torch.tensor(
@@ -3408,13 +3406,12 @@ class GPUModelRunner(
# If a batch only has token ids, then including the embedding layer
# in the CUDA graph will be more performant (like in the else case
# below).
token_ids_idx = (
self.is_token_ids.gpu[:num_scheduled_tokens]
.nonzero(as_tuple=False)
.squeeze(1)
)
is_token_ids = self.is_token_ids.np[:num_scheduled_tokens]
token_ids_idx_np = np.nonzero(is_token_ids)[0]
# Some tokens ids may need to become embeds
if token_ids_idx.numel() > 0:
if token_ids_idx_np.size > 0:
token_ids_idx = torch.from_numpy(token_ids_idx_np)
token_ids_idx = token_ids_idx.to(self.device, non_blocking=True)
token_ids = self.input_ids.gpu[token_ids_idx]
tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids)
self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds
@@ -4601,6 +4598,9 @@ class GPUModelRunner(
# appending a placeholder (-1) token id.
if (req_state := self.requests.get(req_id)) is not None:
req_state.output_token_ids.append(-1)
pos = self.input_batch.num_tokens_no_spec[i]
self.input_batch.is_token_ids[i, pos] = True
self.input_batch.num_tokens_no_spec[i] = pos + 1
self.input_batch.prev_req_id_to_index = prev_req_id_to_index
def take_draft_token_ids(self) -> DraftTokenIds | None: