[Model Runner V2] FP32 gumbel sampling. (#41775)
Signed-off-by: PatchouliTaisa <patchychen@tencent.com> Co-authored-by: PatchouliTaisa <patchychen@tencent.com>
This commit is contained in:
co-authored by
PatchouliTaisa
parent
46a95815d3
commit
0162596603
@@ -233,6 +233,10 @@ class ModelConfig:
|
||||
Processed means the values after applying all processors, including
|
||||
temperature and top_k/top_p.
|
||||
"""
|
||||
use_fp64_gumbel: bool = False
|
||||
"""Whether to use FP64 (instead of FP32) for the Gumbel noise used by the
|
||||
sampler. FP64 reduces the chance of ties in Gumbel-max sampling at the cost
|
||||
of significantly lower kernel throughput on most GPUs."""
|
||||
disable_sliding_window: bool = False
|
||||
"""Whether to disable sliding window. If True, we will disable the sliding
|
||||
window functionality of the model, capping to sliding window size. If the
|
||||
@@ -365,6 +369,7 @@ class ModelConfig:
|
||||
"spec_target_max_model_len",
|
||||
"enforce_eager",
|
||||
"logprobs_mode",
|
||||
"use_fp64_gumbel",
|
||||
"disable_cascade_attn",
|
||||
"skip_tokenizer_init",
|
||||
"served_model_name",
|
||||
|
||||
@@ -519,6 +519,7 @@ class EngineArgs:
|
||||
max_num_seqs: int | None = None
|
||||
max_logprobs: int = ModelConfig.max_logprobs
|
||||
logprobs_mode: LogprobsMode = ModelConfig.logprobs_mode
|
||||
use_fp64_gumbel: bool = ModelConfig.use_fp64_gumbel
|
||||
disable_log_stats: bool = False
|
||||
aggregate_engine_logging: bool = False
|
||||
revision: str | None = ModelConfig.revision
|
||||
@@ -804,6 +805,7 @@ class EngineArgs:
|
||||
)
|
||||
model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"])
|
||||
model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"])
|
||||
model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"])
|
||||
model_group.add_argument(
|
||||
"--disable-sliding-window", **model_kwargs["disable_sliding_window"]
|
||||
)
|
||||
@@ -1542,6 +1544,7 @@ class EngineArgs:
|
||||
enable_return_routed_experts=self.enable_return_routed_experts,
|
||||
max_logprobs=self.max_logprobs,
|
||||
logprobs_mode=self.logprobs_mode,
|
||||
use_fp64_gumbel=self.use_fp64_gumbel,
|
||||
disable_sliding_window=self.disable_sliding_window,
|
||||
disable_cascade_attn=self.disable_cascade_attn,
|
||||
skip_tokenizer_init=self.skip_tokenizer_init,
|
||||
|
||||
@@ -218,6 +218,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_states=self.req_states,
|
||||
logprobs_mode=self.model_config.logprobs_mode,
|
||||
num_speculative_tokens=self.num_speculative_steps + 1,
|
||||
use_fp64_gumbel=self.model_config.use_fp64_gumbel,
|
||||
)
|
||||
if self.speculative_config is not None:
|
||||
self.rejection_sampler = RejectionSampler(
|
||||
|
||||
@@ -2,7 +2,18 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
# Smallest positive normal fp32 value. Used to clamp the uniform draw so that
|
||||
# `log(u)` cannot produce -inf (and thus `-log(-log(u))` stays finite).
|
||||
#
|
||||
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
|
||||
# in `tl.constexpr(...)`. We can only do that when Triton is actually
|
||||
# available — on the CPU worker path `tl` is a placeholder whose `constexpr`
|
||||
# attribute is `None`, and `tl.constexpr(...)` would crash at import time.
|
||||
_FP32_TINY = (
|
||||
tl.constexpr(float.fromhex("0x1p-126")) if HAS_TRITON else float.fromhex("0x1p-126")
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -79,6 +90,7 @@ def gumbel_block_argmax(
|
||||
processed_logits_col_ptr,
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
):
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
@@ -103,16 +115,21 @@ def gumbel_block_argmax(
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
logits = logits.to(tl.float64)
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
|
||||
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
|
||||
if USE_FP64:
|
||||
logits = logits.to(tl.float64)
|
||||
if temp != 0.0:
|
||||
# Calculate the seed for gumbel noise.
|
||||
seed = tl.load(seeds_ptr + req_state_idx)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
|
||||
# tl.rand returns fp32, so build a true fp64 uniform from 64 random
|
||||
# bits before applying the double-log transform.
|
||||
u = tl_rand64(gumbel_seed, block, includes_zero=False)
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, block, includes_zero=False)
|
||||
else:
|
||||
u = tl.rand(gumbel_seed, block)
|
||||
u = tl.maximum(u, _FP32_TINY)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
|
||||
# Apply gumbel noise.
|
||||
@@ -140,6 +157,7 @@ def _gumbel_sample_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
@@ -166,6 +184,7 @@ def _gumbel_sample_kernel(
|
||||
processed_logits_col_ptr,
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
token_id = block_idx * BLOCK_SIZE + idx
|
||||
tl.store(local_argmax_ptr + token_idx * local_argmax_stride + block_idx, token_id)
|
||||
@@ -181,12 +200,14 @@ def gumbel_sample(
|
||||
apply_temperature: bool,
|
||||
output_processed_logits: torch.Tensor | None = None,
|
||||
output_processed_logits_col: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, vocab_size = logits.shape
|
||||
BLOCK_SIZE = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
|
||||
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
|
||||
local_max = logits.new_empty(num_tokens, num_blocks, dtype=torch.float64)
|
||||
local_max_dtype = torch.float64 if use_fp64 else torch.float32
|
||||
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
|
||||
_gumbel_sample_kernel[(num_tokens, num_blocks)](
|
||||
local_argmax,
|
||||
local_argmax.stride(0),
|
||||
@@ -204,6 +225,7 @@ def gumbel_sample(
|
||||
vocab_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
APPLY_TEMPERATURE=apply_temperature,
|
||||
USE_FP64=use_fp64,
|
||||
)
|
||||
# NOTE(woosuk): Use int64 for later indexing.
|
||||
max_block_idx = local_max.argmax(dim=-1, keepdim=True)
|
||||
|
||||
@@ -31,11 +31,13 @@ class Sampler:
|
||||
req_states: RequestState,
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
num_speculative_tokens: int = 1,
|
||||
use_fp64_gumbel: bool = False,
|
||||
):
|
||||
if logprobs_mode not in ("processed_logprobs", "raw_logprobs"):
|
||||
raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}")
|
||||
self.logprobs_mode = logprobs_mode
|
||||
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
|
||||
self.use_fp64_gumbel = use_fp64_gumbel
|
||||
|
||||
self.sampling_states = SamplingStates(max_num_reqs, vocab_size)
|
||||
self.penalties_state = PenaltiesState(req_states)
|
||||
@@ -192,5 +194,6 @@ class Sampler:
|
||||
self.sampling_states.seeds.gpu,
|
||||
pos,
|
||||
apply_temperature=False,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
return sampled, processed_logits
|
||||
|
||||
@@ -63,6 +63,7 @@ class EagleSpeculator:
|
||||
self.hidden_size = self.hidden_size * hc_mult
|
||||
self.vocab_size = self.draft_model_config.get_vocab_size()
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
|
||||
|
||||
# DP configuration
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
@@ -247,6 +248,7 @@ class EagleSpeculator:
|
||||
apply_temperature=True,
|
||||
output_processed_logits=draft_logits,
|
||||
output_processed_logits_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
else:
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
@@ -125,6 +125,7 @@ class RejectionSampler:
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
self.num_speculative_steps,
|
||||
self.synthetic_conditional_rates,
|
||||
use_fp64=self.sampler.use_fp64_gumbel,
|
||||
)
|
||||
logprobs_tensors = self._get_logprobs_tensors(
|
||||
input_batch,
|
||||
|
||||
@@ -339,6 +339,7 @@ def _resample_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_DRAFT_LOGITS: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
resample_idx = tl.load(rejected_step_ptr + req_idx)
|
||||
@@ -415,6 +416,7 @@ def _resample_kernel(
|
||||
None, # processed_logits_col_ptr
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE=False,
|
||||
USE_FP64=USE_FP64,
|
||||
)
|
||||
token_id = block_idx * BLOCK_SIZE + idx
|
||||
tl.store(
|
||||
@@ -512,6 +514,7 @@ def rejection_sample(
|
||||
num_speculative_steps: int,
|
||||
# [num_speculative_steps]
|
||||
synthetic_conditional_rates: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_reqs = cu_num_logits.shape[0] - 1
|
||||
num_logits, vocab_size = target_logits.shape
|
||||
@@ -620,7 +623,9 @@ def rejection_sample(
|
||||
num_reqs, resample_num_blocks, dtype=torch.int64
|
||||
)
|
||||
resampled_local_max = target_logits.new_empty(
|
||||
num_reqs, resample_num_blocks, dtype=torch.float64
|
||||
num_reqs,
|
||||
resample_num_blocks,
|
||||
dtype=torch.float64 if use_fp64 else torch.float32,
|
||||
)
|
||||
_resample_kernel[(num_reqs, resample_num_blocks)](
|
||||
resampled_local_argmax,
|
||||
@@ -644,6 +649,7 @@ def rejection_sample(
|
||||
vocab_size,
|
||||
BLOCK_SIZE=RESAMPLE_BLOCK_SIZE,
|
||||
HAS_DRAFT_LOGITS=has_draft_logits,
|
||||
USE_FP64=use_fp64,
|
||||
)
|
||||
|
||||
# Insert the resampled tokens into the output sampled.
|
||||
|
||||
Reference in New Issue
Block a user