[Model Runner V2] FP32 gumbel sampling. (#41775)

Signed-off-by: PatchouliTaisa <patchychen@tencent.com>
Co-authored-by: PatchouliTaisa <patchychen@tencent.com>
This commit is contained in:
PatchyTIS
2026-05-15 09:20:08 -07:00
committed by GitHub
co-authored by PatchouliTaisa
parent 46a95815d3
commit 0162596603
8 changed files with 50 additions and 7 deletions
+3
View File
@@ -31,11 +31,13 @@ class Sampler:
req_states: RequestState,
logprobs_mode: LogprobsMode = "raw_logprobs",
num_speculative_tokens: int = 1,
use_fp64_gumbel: bool = False,
):
if logprobs_mode not in ("processed_logprobs", "raw_logprobs"):
raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}")
self.logprobs_mode = logprobs_mode
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
self.use_fp64_gumbel = use_fp64_gumbel
self.sampling_states = SamplingStates(max_num_reqs, vocab_size)
self.penalties_state = PenaltiesState(req_states)
@@ -192,5 +194,6 @@ class Sampler:
self.sampling_states.seeds.gpu,
pos,
apply_temperature=False,
use_fp64=self.use_fp64_gumbel,
)
return sampled, processed_logits