[V1][Sampler] Avoid an operation during temperature application (#13587)

This commit is contained in:
Nick Hill
2025-02-20 22:05:56 -08:00
committed by GitHub
parent a30c093502
commit 31aa045c11
4 changed files with 18 additions and 10 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ import torch
@dataclass
class SamplingMetadata:
temperature: torch.Tensor
temperature: Optional[torch.Tensor]
all_greedy: bool
all_random: bool
+4 -4
View File
@@ -77,11 +77,8 @@ class Sampler(nn.Module):
logits: torch.Tensor,
temp: torch.Tensor,
) -> torch.Tensor:
# Avoid division by zero.
temp = torch.where(temp < _SAMPLING_EPS, 1.0, temp)
# Use in-place division to avoid creating a new tensor.
logits.div_(temp.unsqueeze(dim=1))
return logits
return logits.div_(temp.unsqueeze(dim=1))
def greedy_sample(self, logits: torch.Tensor) -> torch.Tensor:
return logits.argmax(dim=-1).view(-1)
@@ -100,6 +97,8 @@ class Sampler(nn.Module):
if sampling_metadata.all_greedy:
return greedy_sampled
assert sampling_metadata.temperature is not None
# Apply temperature.
logits = self.apply_temperature(logits, sampling_metadata.temperature)
@@ -122,6 +121,7 @@ class Sampler(nn.Module):
sampling_metadata.temperature < _SAMPLING_EPS,
greedy_sampled,
random_sampled,
out=greedy_sampled, # Reuse tensor
)
return sampled