[Perf] Optimize clamp to clamp_ (#48143)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
2cab53ddee
commit
f17be06fbe
@@ -91,7 +91,8 @@ class NgramGPUKernel(nn.Module):
|
||||
suffix_indices = suffix_starts.unsqueeze(1) + torch.arange(
|
||||
ngram_len, device=device
|
||||
)
|
||||
suffix = torch.gather(token_ids, 1, suffix_indices.clamp(min=0))
|
||||
suffix_indices.clamp_(min=0)
|
||||
suffix = torch.gather(token_ids, 1, suffix_indices)
|
||||
|
||||
# Window matches for each sequence.
|
||||
matches = (search_windows == suffix.unsqueeze(1)).all(dim=-1)
|
||||
@@ -134,7 +135,7 @@ class NgramGPUKernel(nn.Module):
|
||||
draft_indices = draft_start.unsqueeze(1) + torch.arange(
|
||||
num_draft_tokens, device=device
|
||||
)
|
||||
draft_indices = draft_indices.clamp(min=0, max=max_seq_len - 1)
|
||||
draft_indices.clamp_(min=0, max=max_seq_len - 1)
|
||||
|
||||
# Extract draft tokens; gather always runs.
|
||||
draft_tokens = torch.gather(token_ids, 1, draft_indices)
|
||||
@@ -357,7 +358,8 @@ class NgramProposerGPU:
|
||||
valid_write_mask & (valid_sampled_token_ids_gpu != -1) & in_bounds
|
||||
)
|
||||
|
||||
write_positions_long = write_positions.clamp(max=max_seq_len - 1).long()
|
||||
write_positions.clamp_(max=max_seq_len - 1)
|
||||
write_positions_long = write_positions.long()
|
||||
existing_values = token_ids_gpu.gather(1, write_positions_long)
|
||||
|
||||
tokens_cast = valid_sampled_token_ids_gpu.to(token_ids_gpu.dtype)
|
||||
@@ -427,7 +429,9 @@ class NgramProposerGPU:
|
||||
)
|
||||
|
||||
# Backup last valid token before speculative tokens.
|
||||
backup_indices = (num_tokens_no_spec[:num_reqs] - 1).clamp(min=0).long()
|
||||
backup_indices = num_tokens_no_spec[:num_reqs] - 1
|
||||
backup_indices.clamp_(min=0)
|
||||
backup_indices = backup_indices.long()
|
||||
backup_next_token_ids = torch.gather(
|
||||
token_ids_gpu[:num_reqs], dim=1, index=backup_indices.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
@@ -447,16 +451,17 @@ class NgramProposerGPU:
|
||||
|
||||
# Rightmost valid index per row.
|
||||
last_valid_indices = valid_sampled_tokens_count - 1
|
||||
last_valid_indices_safe = torch.clamp(last_valid_indices, min=0)
|
||||
has_valid_sample = last_valid_indices >= 0
|
||||
last_valid_indices.clamp_(min=0)
|
||||
|
||||
# Last valid token from each row; undefined if none.
|
||||
selected_tokens = torch.gather(
|
||||
valid_sampled_token_ids_gpu, 1, last_valid_indices_safe.unsqueeze(1)
|
||||
valid_sampled_token_ids_gpu, 1, last_valid_indices.unsqueeze(1)
|
||||
).squeeze(1)
|
||||
|
||||
# Use last token if valid; otherwise fallback to backup.
|
||||
next_token_ids = torch.where(
|
||||
last_valid_indices != -1,
|
||||
has_valid_sample,
|
||||
selected_tokens,
|
||||
backup_next_token_ids,
|
||||
)
|
||||
|
||||
@@ -107,7 +107,9 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
if block_table is None:
|
||||
continue
|
||||
n_blocks = block_table.shape[1]
|
||||
bn = (new_positions_1d // block_size).clamp(max=n_blocks - 1).to(torch.long)
|
||||
bn = new_positions_1d // block_size
|
||||
bn.clamp_(max=n_blocks - 1)
|
||||
bn = bn.to(torch.long)
|
||||
block_ids = block_table[:batch_size].gather(1, bn.unsqueeze(1)).squeeze(1)
|
||||
sm = block_ids * block_size + (new_positions_1d % block_size)
|
||||
sm.masked_fill_(exceeds, PADDING_SLOT_ID)
|
||||
|
||||
Reference in New Issue
Block a user