[Sampler] Stop upcasting logits to fp32 in apply_sampling_params (#48641)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
parent
02c01f442b
commit
425c4eafb0
@@ -83,11 +83,7 @@ class Sampler:
|
||||
# that num_nans is computed before applying penalties and temperature.
|
||||
num_nans = get_num_nans(logits) if self.compute_nans else None
|
||||
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
|
||||
idx_mapping_np
|
||||
)
|
||||
return_logprobs = max_num_logprobs != NO_LOGPROBS or max_per_req_token_ids > 0
|
||||
return_logprobs = self.returns_logprobs(idx_mapping_np)
|
||||
|
||||
sampled, processed_logits = self.sample(
|
||||
logits,
|
||||
@@ -102,6 +98,10 @@ class Sampler:
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode in ("processed_logprobs", "processed_logits"):
|
||||
logits = processed_logits
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_per_req_token_ids = self.logprob_token_ids_state.max_num_token_ids(
|
||||
idx_mapping_np
|
||||
)
|
||||
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
|
||||
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
|
||||
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
|
||||
@@ -142,6 +142,13 @@ class Sampler:
|
||||
)
|
||||
return sampler_output
|
||||
|
||||
def returns_logprobs(self, idx_mapping_np: np.ndarray) -> bool:
|
||||
"""Whether any request in the batch produces logprobs this step."""
|
||||
return (
|
||||
self.sampling_states.max_num_logprobs(idx_mapping_np) != NO_LOGPROBS
|
||||
or self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np) > 0
|
||||
)
|
||||
|
||||
def apply_sampling_params(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
@@ -152,8 +159,13 @@ class Sampler:
|
||||
expanded_local_pos: torch.Tensor,
|
||||
skip_top_k_top_p: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Copy logits to a new FP32 tensor.
|
||||
logits = torch.empty_like(logits, dtype=torch.float32).copy_(logits)
|
||||
# The ops below upcast to fp32 internally, so the input dtype is kept and
|
||||
# mutated in place. Only raw_logprobs reads the unmodified logits
|
||||
# afterward, so copy just for that case.
|
||||
if self.logprobs_mode.startswith("raw_") and self.returns_logprobs(
|
||||
idx_mapping_np
|
||||
):
|
||||
logits = logits.clone()
|
||||
|
||||
# Apply logit bias (e.g., allowed_token_ids, min_tokens) in place.
|
||||
self.logit_bias_state.apply_logit_bias(
|
||||
|
||||
Reference in New Issue
Block a user