diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index 6dbb04cd93..4b0a1694f7 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -73,6 +73,14 @@ def tl_rand64(seed, offset, includes_zero: tl.constexpr): return u +@triton.jit +def tl_rand32(seed, offset, includes_zero: tl.constexpr): + u = tl.rand(seed, offset) + if not includes_zero: + u = tl.maximum(u, _TL_RAND_MIN) + return u + + @triton.jit def gumbel_block_argmax( logits, @@ -131,8 +139,7 @@ def gumbel_block_argmax( u = tl_rand64(gumbel_seed, block, includes_zero=False) gumbel_noise = -tl.log(-tl.log(u)) else: - u = tl.rand(gumbel_seed, block) - u = tl.maximum(u, _TL_RAND_MIN) + u = tl_rand32(gumbel_seed, block, includes_zero=False) # Draw the large-noise tail (which decides the argmax winner) from u -> 0, # where fp32 has fine resolution, instead of u -> 1, where fp32 spacing is # ~2**-24. The naive `-log(-log(u))` puts the winning tail at u -> 1, diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index 7020f22804..070270c832 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -3,7 +3,7 @@ import torch from vllm.triton_utils import tl, tldevice, triton -from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand64 +from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32 @triton.jit @@ -243,7 +243,7 @@ def _rejection_kernel( if SYNTHETIC_MODE: pos = tl.load(pos_ptr + logit_idx) - u = tl_rand64(seed, pos, includes_zero=False) + u = tl_rand32(seed, pos, includes_zero=False) rate = tl.load(synthetic_conditional_rates_ptr + i) # -1 is used for padded draft token ids that should be rejected. accepted &= (u < rate) & (draft_sampled >= 0) @@ -272,7 +272,7 @@ def _rejection_kernel( ) target_log_prob = target_logit - target_lse pos = tl.load(pos_ptr + logit_idx) - u = tl_rand64(seed, pos, includes_zero=False) + u = tl_rand32(seed, pos, includes_zero=False) if HAS_DRAFT_LOGITS: draft_logit = tl.load( draft_logits_ptr