[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1765,8 +1765,6 @@ class GPUModelRunner(
|
||||
self.input_batch.prev_sampled_token_ids[:num_common_tokens, 0],
|
||||
non_blocking=True,
|
||||
)
|
||||
if self.enable_prompt_embeds:
|
||||
self.is_token_ids.gpu[:num_common_tokens] = True
|
||||
return
|
||||
# Upload the index tensors asynchronously so the scatter can be non-blocking.
|
||||
sampled_tokens_index_tensor = torch.tensor(
|
||||
@@ -3408,13 +3406,12 @@ class GPUModelRunner(
|
||||
# If a batch only has token ids, then including the embedding layer
|
||||
# in the CUDA graph will be more performant (like in the else case
|
||||
# below).
|
||||
token_ids_idx = (
|
||||
self.is_token_ids.gpu[:num_scheduled_tokens]
|
||||
.nonzero(as_tuple=False)
|
||||
.squeeze(1)
|
||||
)
|
||||
is_token_ids = self.is_token_ids.np[:num_scheduled_tokens]
|
||||
token_ids_idx_np = np.nonzero(is_token_ids)[0]
|
||||
# Some tokens ids may need to become embeds
|
||||
if token_ids_idx.numel() > 0:
|
||||
if token_ids_idx_np.size > 0:
|
||||
token_ids_idx = torch.from_numpy(token_ids_idx_np)
|
||||
token_ids_idx = token_ids_idx.to(self.device, non_blocking=True)
|
||||
token_ids = self.input_ids.gpu[token_ids_idx]
|
||||
tokens_to_embeds = self.model.embed_input_ids(input_ids=token_ids)
|
||||
self.inputs_embeds.gpu[token_ids_idx] = tokens_to_embeds
|
||||
@@ -4601,6 +4598,9 @@ class GPUModelRunner(
|
||||
# appending a placeholder (-1) token id.
|
||||
if (req_state := self.requests.get(req_id)) is not None:
|
||||
req_state.output_token_ids.append(-1)
|
||||
pos = self.input_batch.num_tokens_no_spec[i]
|
||||
self.input_batch.is_token_ids[i, pos] = True
|
||||
self.input_batch.num_tokens_no_spec[i] = pos + 1
|
||||
self.input_batch.prev_req_id_to_index = prev_req_id_to_index
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
|
||||
Reference in New Issue
Block a user