[Model Runner V2] FP32 gumbel sampling. (#41775)

Signed-off-by: PatchouliTaisa <patchychen@tencent.com>
Co-authored-by: PatchouliTaisa <patchychen@tencent.com>
This commit is contained in:
PatchyTIS
2026-05-15 09:20:08 -07:00
committed by GitHub
co-authored by PatchouliTaisa
parent 46a95815d3
commit 0162596603
8 changed files with 50 additions and 7 deletions
+5
View File
@@ -233,6 +233,10 @@ class ModelConfig:
Processed means the values after applying all processors, including
temperature and top_k/top_p.
"""
use_fp64_gumbel: bool = False
"""Whether to use FP64 (instead of FP32) for the Gumbel noise used by the
sampler. FP64 reduces the chance of ties in Gumbel-max sampling at the cost
of significantly lower kernel throughput on most GPUs."""
disable_sliding_window: bool = False
"""Whether to disable sliding window. If True, we will disable the sliding
window functionality of the model, capping to sliding window size. If the
@@ -365,6 +369,7 @@ class ModelConfig:
"spec_target_max_model_len",
"enforce_eager",
"logprobs_mode",
"use_fp64_gumbel",
"disable_cascade_attn",
"skip_tokenizer_init",
"served_model_name",
+3
View File
@@ -519,6 +519,7 @@ class EngineArgs:
max_num_seqs: int | None = None
max_logprobs: int = ModelConfig.max_logprobs
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
disable_log_stats: bool = False
aggregate_engine_logging: bool = False
revision: str | None = ModelConfig.revision
@@ -804,6 +805,7 @@ class EngineArgs:
)
model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"])
model_group.add_argument(
"--disable-sliding-window", **model_kwargs["disable_sliding_window"]
)
@@ -1542,6 +1544,7 @@ class EngineArgs:
enable_return_routed_experts=self.enable_return_routed_experts,
max_logprobs=self.max_logprobs,
logprobs_mode=self.logprobs_mode,
use_fp64_gumbel=self.use_fp64_gumbel,
disable_sliding_window=self.disable_sliding_window,
disable_cascade_attn=self.disable_cascade_attn,
skip_tokenizer_init=self.skip_tokenizer_init,
+1
View File
@@ -218,6 +218,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_states=self.req_states,
logprobs_mode=self.model_config.logprobs_mode,
num_speculative_tokens=self.num_speculative_steps + 1,
use_fp64_gumbel=self.model_config.use_fp64_gumbel,
)
if self.speculative_config is not None:
self.rejection_sampler = RejectionSampler(
+28 -6
View File
@@ -2,7 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.triton_utils import HAS_TRITON, tl, triton
# Smallest positive normal fp32 value. Used to clamp the uniform draw so that
# `log(u)` cannot produce -inf (and thus `-log(-log(u))` stays finite).
#
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
# in `tl.constexpr(...)`. We can only do that when Triton is actually
# available — on the CPU worker path `tl` is a placeholder whose `constexpr`
# attribute is `None`, and `tl.constexpr(...)` would crash at import time.
_FP32_TINY = (
tl.constexpr(float.fromhex("0x1p-126")) if HAS_TRITON else float.fromhex("0x1p-126")
)
@triton.jit
@@ -79,6 +90,7 @@ def gumbel_block_argmax(
processed_logits_col_ptr,
vocab_size,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
):
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
@@ -103,16 +115,21 @@ def gumbel_block_argmax(
mask=mask,
)
logits = logits.to(tl.float64)
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
if USE_FP64:
logits = logits.to(tl.float64)
if temp != 0.0:
# Calculate the seed for gumbel noise.
seed = tl.load(seeds_ptr + req_state_idx)
pos = tl.load(pos_ptr + token_idx)
gumbel_seed = tl.randint(seed, pos)
# tl.rand returns fp32, so build a true fp64 uniform from 64 random
# bits before applying the double-log transform.
u = tl_rand64(gumbel_seed, block, includes_zero=False)
if USE_FP64:
u = tl_rand64(gumbel_seed, block, includes_zero=False)
else:
u = tl.rand(gumbel_seed, block)
u = tl.maximum(u, _FP32_TINY)
gumbel_noise = -tl.log(-tl.log(u))
# Apply gumbel noise.
@@ -140,6 +157,7 @@ def _gumbel_sample_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
):
token_idx = tl.program_id(0)
block_idx = tl.program_id(1)
@@ -166,6 +184,7 @@ def _gumbel_sample_kernel(
processed_logits_col_ptr,
vocab_size,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
USE_FP64=USE_FP64,
)
token_id = block_idx * BLOCK_SIZE + idx
tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id)
@@ -181,12 +200,14 @@ def gumbel_sample(
apply_temperature: bool,
output_processed_logits: torch.Tensor | None = None,
output_processed_logits_col: torch.Tensor | None = None,
use_fp64: bool = False,
) -> torch.Tensor:
num_tokens, vocab_size = logits.shape
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
local_max = logits.new_empty(num_tokens, num_blocks, dtype=torch.float64)
local_max_dtype = torch.float64 if use_fp64 else torch.float32
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
_gumbel_sample_kernel[(num_tokens, num_blocks)](
local_argmax,
local_argmax.stride(0),
@@ -204,6 +225,7 @@ def gumbel_sample(
vocab_size,
BLOCK_SIZE=BLOCK_SIZE,
APPLY_TEMPERATURE=apply_temperature,
USE_FP64=use_fp64,
)
# NOTE(woosuk): Use int64 for later indexing.
max_block_idx = local_max.argmax(dim=-1, keepdim=True)
+3
View File
@@ -31,11 +31,13 @@ class Sampler:
req_states: RequestState,
logprobs_mode: LogprobsMode = "raw_logprobs",
num_speculative_tokens: int = 1,
use_fp64_gumbel: bool = False,
):
if logprobs_mode not in ("processed_logprobs", "raw_logprobs"):
raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}")
self.logprobs_mode = logprobs_mode
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
self.use_fp64_gumbel = use_fp64_gumbel
self.sampling_states = SamplingStates(max_num_reqs, vocab_size)
self.penalties_state = PenaltiesState(req_states)
@@ -192,5 +194,6 @@ class Sampler:
self.sampling_states.seeds.gpu,
pos,
apply_temperature=False,
use_fp64=self.use_fp64_gumbel,
)
return sampled, processed_logits
@@ -63,6 +63,7 @@ class EagleSpeculator:
self.hidden_size = self.hidden_size * hc_mult
self.vocab_size = self.draft_model_config.get_vocab_size()
self.dtype = vllm_config.model_config.dtype
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
# DP configuration
self.dp_size = vllm_config.parallel_config.data_parallel_size
@@ -247,6 +248,7 @@ class EagleSpeculator:
apply_temperature=True,
output_processed_logits=draft_logits,
output_processed_logits_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
else:
return logits.argmax(dim=-1)
@@ -125,6 +125,7 @@ class RejectionSampler:
self.sampler.sampling_states.seeds.gpu,
self.num_speculative_steps,
self.synthetic_conditional_rates,
use_fp64=self.sampler.use_fp64_gumbel,
)
logprobs_tensors = self._get_logprobs_tensors(
input_batch,
@@ -339,6 +339,7 @@ def _resample_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
USE_FP64: tl.constexpr,
):
req_idx = tl.program_id(0)
resample_idx = tl.load(rejected_step_ptr + req_idx)
@@ -415,6 +416,7 @@ def _resample_kernel(
None, # processed_logits_col_ptr
vocab_size,
APPLY_TEMPERATURE=False,
USE_FP64=USE_FP64,
)
token_id = block_idx * BLOCK_SIZE + idx
tl.store(
@@ -512,6 +514,7 @@ def rejection_sample(
num_speculative_steps: int,
# [num_speculative_steps]
synthetic_conditional_rates: torch.Tensor | None = None,
use_fp64: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
num_reqs = cu_num_logits.shape[0] - 1
num_logits, vocab_size = target_logits.shape
@@ -620,7 +623,9 @@ def rejection_sample(
num_reqs, resample_num_blocks, dtype=torch.int64
)
resampled_local_max = target_logits.new_empty(
num_reqs, resample_num_blocks, dtype=torch.float64
num_reqs,
resample_num_blocks,
dtype=torch.float64 if use_fp64 else torch.float32,
)
_resample_kernel[(num_reqs, resample_num_blocks)](
resampled_local_argmax,
@@ -644,6 +649,7 @@ def rejection_sample(
vocab_size,
BLOCK_SIZE=RESAMPLE_BLOCK_SIZE,
HAS_DRAFT_LOGITS=has_draft_logits,
USE_FP64=use_fp64,
)
# Insert the resampled tokens into the output sampled.