[Spec Decode] DFlash2: local convolution + candidate selector (#52816)

This commit is contained in:
Zihan Zhang
2026-08-20 22:27:22 -07:00
committed by GitHub
parent df344bf36f
commit b389ac2946
14 changed files with 866 additions and 44 deletions
+52 -34
View File
@@ -81,6 +81,46 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
return u
@triton.jit
def gumbel_noised_argmax(
logits,
keys,
mask,
seed,
pos,
temp,
USE_FP64: tl.constexpr,
APPLY_TEMPERATURE: tl.constexpr = True,
):
"""Argmax of logits under Gumbel-max sampling, or plain argmax at temp 0.
`keys` indexes the noise, so the same token draws the same noise wherever it
appears; `pos` and `seed` place the draw in the request's stream, which is
what lets a draft and its verification agree.
"""
if temp != 0.0 and APPLY_TEMPERATURE:
# Match the behavior of _temperature_kernel: if that kernel uses
# tl.div_rn, this must too.
logits = logits / temp
# fp32 is the default reduction dtype; fp64 is ~1/32-1/64x the throughput
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
if USE_FP64:
logits = logits.to(tl.float64)
if temp != 0.0:
gumbel_seed = tl.randint(seed, pos)
if USE_FP64:
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
gumbel_noise = -tl.log(-tl.log(u))
else:
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
gumbel_noise = -tl.log(-tldevice.log1p(-u))
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
return tl.max(logits, axis=0, return_indices=True)
@triton.jit
def gumbel_block_argmax(
logits,
@@ -124,40 +164,18 @@ def gumbel_block_argmax(
mask=mask & is_valid_req,
)
if temp != 0.0 and APPLY_TEMPERATURE:
# Apply temperature.
# NOTE(woosuk): Match the behavior of _temperature_kernel.
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
logits = logits / temp
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
if USE_FP64:
logits = logits.to(tl.float64)
if temp != 0.0:
# Calculate the seed for gumbel noise.
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
pos = tl.load(pos_ptr + token_idx)
gumbel_seed = tl.randint(seed, pos)
if USE_FP64:
u = tl_rand64(gumbel_seed, block, includes_zero=False)
gumbel_noise = -tl.log(-tl.log(u))
else:
u = tl_rand32(gumbel_seed, block, includes_zero=False)
# Draw the large-noise tail (which decides the argmax winner) from u -> 0,
# where fp32 has fine resolution, instead of u -> 1, where fp32 spacing is
# ~2**-24. The naive `-log(-log(u))` puts the winning tail at u -> 1,
# hard-capping the noise at ~16.6 and coarsely quantizing it; using
# `log1p(-u)` == `log(1 - u)` keeps the tail in the well-resolved region.
# Note `1 - u` would lose precision for small u, so `log1p` is required.
gumbel_noise = -tl.log(-tldevice.log1p(-u))
# Apply gumbel noise.
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
value, idx = tl.max(logits, axis=0, return_indices=True)
return value, idx
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
pos = tl.load(pos_ptr + token_idx)
return gumbel_noised_argmax(
logits,
block,
mask,
seed,
pos,
temp,
USE_FP64=USE_FP64,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
)
@triton.jit