[V1][Sampler] Avoid an operation during temperature application (#13587)

This commit is contained in:
Nick Hill
2025-02-20 22:05:56 -08:00
committed by GitHub
parent a30c093502
commit 31aa045c11
4 changed files with 18 additions and 10 deletions
+4 -4
View File
@@ -77,11 +77,8 @@ class Sampler(nn.Module):
logits: torch.Tensor,
temp: torch.Tensor,
) -> torch.Tensor:
# Avoid division by zero.
temp = torch.where(temp < _SAMPLING_EPS, 1.0, temp)
# Use in-place division to avoid creating a new tensor.
logits.div_(temp.unsqueeze(dim=1))
return logits
return logits.div_(temp.unsqueeze(dim=1))
def greedy_sample(self, logits: torch.Tensor) -> torch.Tensor:
return logits.argmax(dim=-1).view(-1)
@@ -100,6 +97,8 @@ class Sampler(nn.Module):
if sampling_metadata.all_greedy:
return greedy_sampled
assert sampling_metadata.temperature is not None
# Apply temperature.
logits = self.apply_temperature(logits, sampling_metadata.temperature)
@@ -122,6 +121,7 @@ class Sampler(nn.Module):
sampling_metadata.temperature < _SAMPLING_EPS,
greedy_sampled,
random_sampled,
out=greedy_sampled, # Reuse tensor
)
return sampled