diff --git a/vllm/config/model.py b/vllm/config/model.py index 3f4dd471ec..e09364a5de 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -233,6 +233,10 @@ class ModelConfig: Processed means the values after applying all processors, including temperature and top_k/top_p. """ + use_fp64_gumbel: bool = False + """Whether to use FP64 (instead of FP32) for the Gumbel noise used by the + sampler. FP64 reduces the chance of ties in Gumbel-max sampling at the cost + of significantly lower kernel throughput on most GPUs.""" disable_sliding_window: bool = False """Whether to disable sliding window. If True, we will disable the sliding window functionality of the model, capping to sliding window size. If the @@ -365,6 +369,7 @@ class ModelConfig: "spec_target_max_model_len", "enforce_eager", "logprobs_mode", + "use_fp64_gumbel", "disable_cascade_attn", "skip_tokenizer_init", "served_model_name", diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index ad492fde4e..2b5e242bb2 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -519,6 +519,7 @@ class EngineArgs: max_num_seqs: int | None = None max_logprobs: int = ModelConfig.max_logprobs logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode + use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel disable_log_stats: bool = False aggregate_engine_logging: bool = False revision: str | None = ModelConfig.revision @@ -804,6 +805,7 @@ class EngineArgs: ) model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"]) model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"]) + model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"]) model_group.add_argument( "--disable-sliding-window", **model_kwargs["disable_sliding_window"] ) @@ -1542,6 +1544,7 @@ class EngineArgs: enable_return_routed_experts=self.enable_return_routed_experts, max_logprobs=self.max_logprobs, logprobs_mode=self.logprobs_mode, + use_fp64_gumbel=self.use_fp64_gumbel, disable_sliding_window=self.disable_sliding_window, disable_cascade_attn=self.disable_cascade_attn, skip_tokenizer_init=self.skip_tokenizer_init, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 6485444e6a..4fc5a754db 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -218,6 +218,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_states=self.req_states, logprobs_mode=self.model_config.logprobs_mode, num_speculative_tokens=self.num_speculative_steps + 1, + use_fp64_gumbel=self.model_config.use_fp64_gumbel, ) if self.speculative_config is not None: self.rejection_sampler = RejectionSampler( diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index a02dd62026..aaa49283d3 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -2,7 +2,18 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from vllm.triton_utils import tl, triton +from vllm.triton_utils import HAS_TRITON, tl, triton + +# Smallest positive normal fp32 value. Used to clamp the uniform draw so that +# `log(u)` cannot produce -inf (and thus `-log(-log(u))` stays finite). +# +# Triton requires globals accessed from `@triton.jit` functions to be wrapped +# in `tl.constexpr(...)`. We can only do that when Triton is actually +# available — on the CPU worker path `tl` is a placeholder whose `constexpr` +# attribute is `None`, and `tl.constexpr(...)` would crash at import time. +_FP32_TINY = ( + tl.constexpr(float.fromhex("0x1p-126")) if HAS_TRITON else float.fromhex("0x1p-126") +) @triton.jit @@ -79,6 +90,7 @@ def gumbel_block_argmax( processed_logits_col_ptr, vocab_size, APPLY_TEMPERATURE: tl.constexpr, + USE_FP64: tl.constexpr, ): req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx) temp = tl.load(temp_ptr + req_state_idx).to(tl.float32) @@ -103,16 +115,21 @@ def gumbel_block_argmax( mask=mask, ) - logits = logits.to(tl.float64) + # fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput + # on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max. + if USE_FP64: + logits = logits.to(tl.float64) if temp != 0.0: # Calculate the seed for gumbel noise. seed = tl.load(seeds_ptr + req_state_idx) pos = tl.load(pos_ptr + token_idx) gumbel_seed = tl.randint(seed, pos) - # tl.rand returns fp32, so build a true fp64 uniform from 64 random - # bits before applying the double-log transform. - u = tl_rand64(gumbel_seed, block, includes_zero=False) + if USE_FP64: + u = tl_rand64(gumbel_seed, block, includes_zero=False) + else: + u = tl.rand(gumbel_seed, block) + u = tl.maximum(u, _FP32_TINY) gumbel_noise = -tl.log(-tl.log(u)) # Apply gumbel noise. @@ -140,6 +157,7 @@ def _gumbel_sample_kernel( vocab_size, BLOCK_SIZE: tl.constexpr, APPLY_TEMPERATURE: tl.constexpr, + USE_FP64: tl.constexpr, ): token_idx = tl.program_id(0) block_idx = tl.program_id(1) @@ -166,6 +184,7 @@ def _gumbel_sample_kernel( processed_logits_col_ptr, vocab_size, APPLY_TEMPERATURE=APPLY_TEMPERATURE, + USE_FP64=USE_FP64, ) token_id = block_idx * BLOCK_SIZE + idx tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id) @@ -181,12 +200,14 @@ def gumbel_sample( apply_temperature: bool, output_processed_logits: torch.Tensor | None = None, output_processed_logits_col: torch.Tensor | None = None, + use_fp64: bool = False, ) -> torch.Tensor: num_tokens, vocab_size = logits.shape BLOCK_SIZE = 1024 num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE) local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64) - local_max = logits.new_empty(num_tokens, num_blocks, dtype=torch.float64) + local_max_dtype = torch.float64 if use_fp64 else torch.float32 + local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype) _gumbel_sample_kernel[(num_tokens, num_blocks)]( local_argmax, local_argmax.stride(0), @@ -204,6 +225,7 @@ def gumbel_sample( vocab_size, BLOCK_SIZE=BLOCK_SIZE, APPLY_TEMPERATURE=apply_temperature, + USE_FP64=use_fp64, ) # NOTE(woosuk): Use int64 for later indexing. max_block_idx = local_max.argmax(dim=-1, keepdim=True) diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 7b4b01666f..8bf884fd9b 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -31,11 +31,13 @@ class Sampler: req_states: RequestState, logprobs_mode: LogprobsMode = "raw_logprobs", num_speculative_tokens: int = 1, + use_fp64_gumbel: bool = False, ): if logprobs_mode not in ("processed_logprobs", "raw_logprobs"): raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}") self.logprobs_mode = logprobs_mode self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default. + self.use_fp64_gumbel = use_fp64_gumbel self.sampling_states = SamplingStates(max_num_reqs, vocab_size) self.penalties_state = PenaltiesState(req_states) @@ -192,5 +194,6 @@ class Sampler: self.sampling_states.seeds.gpu, pos, apply_temperature=False, + use_fp64=self.use_fp64_gumbel, ) return sampled, processed_logits diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index 1e3b312e3c..00777cbd81 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -63,6 +63,7 @@ class EagleSpeculator: self.hidden_size = self.hidden_size * hc_mult self.vocab_size = self.draft_model_config.get_vocab_size() self.dtype = vllm_config.model_config.dtype + self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel # DP configuration self.dp_size = vllm_config.parallel_config.data_parallel_size @@ -247,6 +248,7 @@ class EagleSpeculator: apply_temperature=True, output_processed_logits=draft_logits, output_processed_logits_col=draft_step, + use_fp64=self.use_fp64_gumbel, ) else: return logits.argmax(dim=-1) diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 9ead3b7d52..1fe079a43e 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -125,6 +125,7 @@ class RejectionSampler: self.sampler.sampling_states.seeds.gpu, self.num_speculative_steps, self.synthetic_conditional_rates, + use_fp64=self.sampler.use_fp64_gumbel, ) logprobs_tensors = self._get_logprobs_tensors( input_batch, diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index 3c939c9bff..0cfbdf4182 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -339,6 +339,7 @@ def _resample_kernel( vocab_size, BLOCK_SIZE: tl.constexpr, HAS_DRAFT_LOGITS: tl.constexpr, + USE_FP64: tl.constexpr, ): req_idx = tl.program_id(0) resample_idx = tl.load(rejected_step_ptr + req_idx) @@ -415,6 +416,7 @@ def _resample_kernel( None, # processed_logits_col_ptr vocab_size, APPLY_TEMPERATURE=False, + USE_FP64=USE_FP64, ) token_id = block_idx * BLOCK_SIZE + idx tl.store( @@ -512,6 +514,7 @@ def rejection_sample( num_speculative_steps: int, # [num_speculative_steps] synthetic_conditional_rates: torch.Tensor | None = None, + use_fp64: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: num_reqs = cu_num_logits.shape[0] - 1 num_logits, vocab_size = target_logits.shape @@ -620,7 +623,9 @@ def rejection_sample( num_reqs, resample_num_blocks, dtype=torch.int64 ) resampled_local_max = target_logits.new_empty( - num_reqs, resample_num_blocks, dtype=torch.float64 + num_reqs, + resample_num_blocks, + dtype=torch.float64 if use_fp64 else torch.float32, ) _resample_kernel[(num_reqs, resample_num_blocks)]( resampled_local_argmax, @@ -644,6 +649,7 @@ def rejection_sample( vocab_size, BLOCK_SIZE=RESAMPLE_BLOCK_SIZE, HAS_DRAFT_LOGITS=has_draft_logits, + USE_FP64=use_fp64, ) # Insert the resampled tokens into the output sampled.