[Spec Decode] DFlash2: local convolution + candidate selector (#52816)
This commit is contained in:
@@ -81,6 +81,46 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_noised_argmax(
|
||||
logits,
|
||||
keys,
|
||||
mask,
|
||||
seed,
|
||||
pos,
|
||||
temp,
|
||||
USE_FP64: tl.constexpr,
|
||||
APPLY_TEMPERATURE: tl.constexpr = True,
|
||||
):
|
||||
"""Argmax of logits under Gumbel-max sampling, or plain argmax at temp 0.
|
||||
|
||||
`keys` indexes the noise, so the same token draws the same noise wherever it
|
||||
appears; `pos` and `seed` place the draw in the request's stream, which is
|
||||
what lets a draft and its verification agree.
|
||||
"""
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Match the behavior of _temperature_kernel: if that kernel uses
|
||||
# tl.div_rn, this must too.
|
||||
logits = logits / temp
|
||||
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32-1/64x the throughput
|
||||
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
|
||||
if USE_FP64:
|
||||
logits = logits.to(tl.float64)
|
||||
if temp != 0.0:
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
|
||||
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
|
||||
gumbel_noise = -tl.log(-tldevice.log1p(-u))
|
||||
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
|
||||
|
||||
return tl.max(logits, axis=0, return_indices=True)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_block_argmax(
|
||||
logits,
|
||||
@@ -124,40 +164,18 @@ def gumbel_block_argmax(
|
||||
mask=mask & is_valid_req,
|
||||
)
|
||||
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Apply temperature.
|
||||
# NOTE(woosuk): Match the behavior of _temperature_kernel.
|
||||
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
|
||||
logits = logits / temp
|
||||
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
|
||||
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
|
||||
if USE_FP64:
|
||||
logits = logits.to(tl.float64)
|
||||
if temp != 0.0:
|
||||
# Calculate the seed for gumbel noise.
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, block, includes_zero=False)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl_rand32(gumbel_seed, block, includes_zero=False)
|
||||
# Draw the large-noise tail (which decides the argmax winner) from u -> 0,
|
||||
# where fp32 has fine resolution, instead of u -> 1, where fp32 spacing is
|
||||
# ~2**-24. The naive `-log(-log(u))` puts the winning tail at u -> 1,
|
||||
# hard-capping the noise at ~16.6 and coarsely quantizing it; using
|
||||
# `log1p(-u)` == `log(1 - u)` keeps the tail in the well-resolved region.
|
||||
# Note `1 - u` would lose precision for small u, so `log1p` is required.
|
||||
gumbel_noise = -tl.log(-tldevice.log1p(-u))
|
||||
|
||||
# Apply gumbel noise.
|
||||
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
|
||||
|
||||
value, idx = tl.max(logits, axis=0, return_indices=True)
|
||||
return value, idx
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
return gumbel_noised_argmax(
|
||||
logits,
|
||||
block,
|
||||
mask,
|
||||
seed,
|
||||
pos,
|
||||
temp,
|
||||
USE_FP64=USE_FP64,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
Reference in New Issue
Block a user