[Perf] Optimize clamp to clamp_ (#48143)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Wentao Ye
2026-07-16 18:41:07 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 2cab53ddee
commit f17be06fbe
7 changed files with 48 additions and 32 deletions
+12 -7
View File
@@ -91,7 +91,8 @@ class NgramGPUKernel(nn.Module):
suffix_indices = suffix_starts.unsqueeze(1) + torch.arange(
ngram_len, device=device
)
suffix = torch.gather(token_ids, 1, suffix_indices.clamp(min=0))
suffix_indices.clamp_(min=0)
suffix = torch.gather(token_ids, 1, suffix_indices)
# Window matches for each sequence.
matches = (search_windows == suffix.unsqueeze(1)).all(dim=-1)
@@ -134,7 +135,7 @@ class NgramGPUKernel(nn.Module):
draft_indices = draft_start.unsqueeze(1) + torch.arange(
num_draft_tokens, device=device
)
draft_indices = draft_indices.clamp(min=0, max=max_seq_len - 1)
draft_indices.clamp_(min=0, max=max_seq_len - 1)
# Extract draft tokens; gather always runs.
draft_tokens = torch.gather(token_ids, 1, draft_indices)
@@ -357,7 +358,8 @@ class NgramProposerGPU:
valid_write_mask & (valid_sampled_token_ids_gpu != -1) & in_bounds
)
write_positions_long = write_positions.clamp(max=max_seq_len - 1).long()
write_positions.clamp_(max=max_seq_len - 1)
write_positions_long = write_positions.long()
existing_values = token_ids_gpu.gather(1, write_positions_long)
tokens_cast = valid_sampled_token_ids_gpu.to(token_ids_gpu.dtype)
@@ -427,7 +429,9 @@ class NgramProposerGPU:
)
# Backup last valid token before speculative tokens.
backup_indices = (num_tokens_no_spec[:num_reqs] - 1).clamp(min=0).long()
backup_indices = num_tokens_no_spec[:num_reqs] - 1
backup_indices.clamp_(min=0)
backup_indices = backup_indices.long()
backup_next_token_ids = torch.gather(
token_ids_gpu[:num_reqs], dim=1, index=backup_indices.unsqueeze(1)
).squeeze(1)
@@ -447,16 +451,17 @@ class NgramProposerGPU:
# Rightmost valid index per row.
last_valid_indices = valid_sampled_tokens_count - 1
last_valid_indices_safe = torch.clamp(last_valid_indices, min=0)
has_valid_sample = last_valid_indices >= 0
last_valid_indices.clamp_(min=0)
# Last valid token from each row; undefined if none.
selected_tokens = torch.gather(
valid_sampled_token_ids_gpu, 1, last_valid_indices_safe.unsqueeze(1)
valid_sampled_token_ids_gpu, 1, last_valid_indices.unsqueeze(1)
).squeeze(1)
# Use last token if valid; otherwise fallback to backup.
next_token_ids = torch.where(
last_valid_indices != -1,
has_valid_sample,
selected_tokens,
backup_next_token_ids,
)
+3 -1
View File
@@ -107,7 +107,9 @@ class Step3p5MTPProposer(EagleProposer):
if block_table is None:
continue
n_blocks = block_table.shape[1]
bn = (new_positions_1d // block_size).clamp(max=n_blocks - 1).to(torch.long)
bn = new_positions_1d // block_size
bn.clamp_(max=n_blocks - 1)
bn = bn.to(torch.long)
block_ids = block_table[:batch_size].gather(1, bn.unsqueeze(1)).squeeze(1)
sm = block_ids * block_size + (new_positions_1d % block_size)
sm.masked_fill_(exceeds, PADDING_SLOT_ID)