[Sampler] Stop upcasting logits to fp32 in apply_sampling_params (#48641)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Michael Goin
2026-07-17 18:15:20 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill
parent 02c01f442b
commit 425c4eafb0
5 changed files with 85 additions and 25 deletions
+19 -7
View File
@@ -83,11 +83,7 @@ class Sampler:
# that num_nans is computed before applying penalties and temperature.
num_nans = get_num_nans(logits) if self.compute_nans else None
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
idx_mapping_np
)
return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0
return_logprobs = self.returns_logprobs(idx_mapping_np)
sampled, processed_logits = self.sample(
logits,
@@ -102,6 +98,10 @@ class Sampler:
if return_logprobs:
if self.logprobs_mode in ("processed_logprobs", "processed_logits"):
logits = processed_logits
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
idx_mapping_np
)
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
@@ -142,6 +142,13 @@ class Sampler:
)
return sampler_output
def returns_logprobs(self, idx_mapping_np: np.ndarray) -> bool:
"""Whether any request in the batch produces logprobs this step."""
return (
self.sampling_states.max_num_logprobs(idx_mapping_np) != NO_LOGPROBS
or self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np) > 0
)
def apply_sampling_params(
self,
logits: torch.Tensor,
@@ -152,8 +159,13 @@ class Sampler:
expanded_local_pos: torch.Tensor,
skip_top_k_top_p: bool = False,
) -> torch.Tensor:
# Copy logits to a new FP32 tensor.
logits = torch.empty_like(logits, dtype=torch.float32).copy_(logits)
# The ops below upcast to fp32 internally, so the input dtype is kept and
# mutated in place. Only raw_logprobs reads the unmodified logits
# afterward, so copy just for that case.
if self.logprobs_mode.startswith("raw_") and self.returns_logprobs(
idx_mapping_np
):
logits = logits.clone()
# Apply logit bias (e.g., allowed_token_ids, min_tokens) in place.
self.logit_bias_state.apply_logit_bias(