[Model Runner V2] FP32 gumbel sampling. (#41775)

Signed-off-by: PatchouliTaisa <patchychen@tencent.com>
Co-authored-by: PatchouliTaisa <patchychen@tencent.com>
This commit is contained in:
PatchyTIS
2026-05-15 09:20:08 -07:00
committed by GitHub
co-authored by PatchouliTaisa
parent 46a95815d3
commit 0162596603
8 changed files with 50 additions and 7 deletions
+28 -6
View File
@@ -2,7 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.triton_utils import HAS_TRITON, tl, triton
# Smallest positive normal fp32 value. Used to clamp the uniform draw so that
# `log(u)` cannot produce -inf (and thus `-log(-log(u))` stays finite).
#
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
# in `tl.constexpr(...)`. We can only do that when Triton is actually
# available — on the CPU worker path `tl` is a placeholder whose `constexpr`
# attribute is `None`, and `tl.constexpr(...)` would crash at import time.
_FP32_TINY = (
tl.constexpr(float.fromhex("0x1p-126")) if HAS_TRITON else float.fromhex("0x1p-126")
)
@triton.jit
@@ -79,6 +90,7 @@ def gumbel_block_argmax(
processed_logits_col_ptr,
vocab_size,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
):
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
@@ -103,16 +115,21 @@ def gumbel_block_argmax(
mask=mask,
)
logits = logits.to(tl.float64)
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
if USE_FP64:
logits = logits.to(tl.float64)
if temp != 0.0:
# Calculate the seed for gumbel noise.
seed = tl.load(seeds_ptr + req_state_idx)
pos = tl.load(pos_ptr + token_idx)
gumbel_seed = tl.randint(seed, pos)
# tl.rand returns fp32, so build a true fp64 uniform from 64 random
# bits before applying the double-log transform.
u = tl_rand64(gumbel_seed, block, includes_zero=False)
if USE_FP64:
u = tl_rand64(gumbel_seed, block, includes_zero=False)
else:
u = tl.rand(gumbel_seed, block)
u = tl.maximum(u, _FP32_TINY)
gumbel_noise = -tl.log(-tl.log(u))
# Apply gumbel noise.
@@ -140,6 +157,7 @@ def _gumbel_sample_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
):
token_idx = tl.program_id(0)
block_idx = tl.program_id(1)
@@ -166,6 +184,7 @@ def _gumbel_sample_kernel(
processed_logits_col_ptr,
vocab_size,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
USE_FP64=USE_FP64,
)
token_id = block_idx * BLOCK_SIZE + idx
tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id)
@@ -181,12 +200,14 @@ def gumbel_sample(
apply_temperature: bool,
output_processed_logits: torch.Tensor | None = None,
output_processed_logits_col: torch.Tensor | None = None,
use_fp64: bool = False,
) -> torch.Tensor:
num_tokens, vocab_size = logits.shape
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
local_max = logits.new_empty(num_tokens, num_blocks, dtype=torch.float64)
local_max_dtype = torch.float64 if use_fp64 else torch.float32
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
_gumbel_sample_kernel[(num_tokens, num_blocks)](
local_argmax,
local_argmax.stride(0),
@@ -204,6 +225,7 @@ def gumbel_sample(
vocab_size,
BLOCK_SIZE=BLOCK_SIZE,
APPLY_TEMPERATURE=apply_temperature,
USE_FP64=use_fp64,
)
# NOTE(woosuk): Use int64 for later indexing.
max_block_idx = local_max.argmax(dim=-1, keepdim=True)
+3
View File
@@ -31,11 +31,13 @@ class Sampler:
req_states: RequestState,
logprobs_mode: LogprobsMode = "raw_logprobs",
num_speculative_tokens: int = 1,
use_fp64_gumbel: bool = False,
):
if logprobs_mode not in ("processed_logprobs", "raw_logprobs"):
raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}")
self.logprobs_mode = logprobs_mode
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
self.use_fp64_gumbel = use_fp64_gumbel
self.sampling_states = SamplingStates(max_num_reqs, vocab_size)
self.penalties_state = PenaltiesState(req_states)
@@ -192,5 +194,6 @@ class Sampler:
self.sampling_states.seeds.gpu,
pos,
apply_temperature=False,
use_fp64=self.use_fp64_gumbel,
)
return sampled, processed_logits