[Model Runner V2][Spec Decode] Use fp32 uniform threshold for acceptance (#46878)
This commit is contained in:
@@ -73,6 +73,14 @@ def tl_rand64(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
u = tl.rand(seed, offset)
|
||||
if not includes_zero:
|
||||
u = tl.maximum(u, _TL_RAND_MIN)
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_block_argmax(
|
||||
logits,
|
||||
@@ -131,8 +139,7 @@ def gumbel_block_argmax(
|
||||
u = tl_rand64(gumbel_seed, block, includes_zero=False)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl.rand(gumbel_seed, block)
|
||||
u = tl.maximum(u, _TL_RAND_MIN)
|
||||
u = tl_rand32(gumbel_seed, block, includes_zero=False)
|
||||
# Draw the large-noise tail (which decides the argmax winner) from u -> 0,
|
||||
# where fp32 has fine resolution, instead of u -> 1, where fp32 spacing is
|
||||
# ~2**-24. The naive `-log(-log(u))` puts the winning tail at u -> 1,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, tldevice, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand64
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -243,7 +243,7 @@ def _rejection_kernel(
|
||||
|
||||
if SYNTHETIC_MODE:
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand64(seed, pos, includes_zero=False)
|
||||
u = tl_rand32(seed, pos, includes_zero=False)
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + i)
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted &= (u < rate) & (draft_sampled >= 0)
|
||||
@@ -272,7 +272,7 @@ def _rejection_kernel(
|
||||
)
|
||||
target_log_prob = target_logit - target_lse
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand64(seed, pos, includes_zero=False)
|
||||
u = tl_rand32(seed, pos, includes_zero=False)
|
||||
if HAS_DRAFT_LOGITS:
|
||||
draft_logit = tl.load(
|
||||
draft_logits_ptr
|
||||
|
||||
Reference in New Issue
Block a user