[Kernel] Use Murmur3 RNG for Gumbel sampling (#51367)

Signed-off-by: freyfwt <110148731+freyfwt@users.noreply.github.com>
Signed-off-by: freyfwt <freytian1996@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
freyfwt
2026-09-15 23:52:26 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jiangyun Zhu
parent 0f8fa53acf
commit 18553c5c9f
5 changed files with 216 additions and 26 deletions
+100 -8
View File
@@ -2,10 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import HAS_TRITON, tl, tldevice, triton
from vllm.triton_utils import HAS_TRITON, tl, triton
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used to clamp
# zero draws before the flipped Gumbel transform below.
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used by the
# retained Philox helper for rejection sampling.
#
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
# in `tl.constexpr(...)`. We can only do that when Triton is actually
@@ -87,6 +87,97 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
return u
@triton.jit
def _murmur3_rotl32(value, shift: tl.constexpr):
return (value << shift) | (value >> (32 - shift))
@triton.jit
def _murmur3_mix(h, key):
key *= 0xCC9E2D51
key = _murmur3_rotl32(key, 15)
key *= 0x1B873593
h ^= key
h = _murmur3_rotl32(h, 13)
return h * 5 + 0xE6546B64
@triton.jit
def _murmur3_fmix32(h):
h ^= h >> 16
h *= 0x85EBCA6B
h ^= h >> 13
h *= 0xC2B2AE35
return h ^ (h >> 16)
@triton.jit
def murmur3_hash32(seed, pos, offset, domain: tl.constexpr = 0):
seed = seed.to(tl.int64)
pos = pos.to(tl.int64)
offset = offset.to(tl.uint32)
# Keep the request-wide prefix scalar until the token offset is mixed in.
h = (seed ^ seed).to(tl.uint32)
h ^= domain
h = _murmur3_mix(h, (seed & 0xFFFFFFFF).to(tl.uint32))
h = _murmur3_mix(h, ((seed >> 32) & 0xFFFFFFFF).to(tl.uint32))
h = _murmur3_mix(h, (pos & 0xFFFFFFFF).to(tl.uint32))
h = _murmur3_mix(h, offset)
return _murmur3_fmix32(h ^ 16)
@triton.jit
def murmur3_uniform32(seed, pos, offset):
random32 = murmur3_hash32(seed, pos, offset)
# Split the uint32 before converting to fp32 so backends without a native
# uint32-to-float conversion can still use all 32 source bits. Both 16-bit
# halves convert exactly; their sum is the correctly rounded fp32 value of
# (random32 + 0.5) * 2**-32. In particular, the u -> 0 winning tail keeps
# the full 32-bit source resolution instead of being truncated to 24 bits.
hi16 = (random32 >> 16).to(tl.int32)
lo16 = (random32 & 0xFFFF).to(tl.int32)
return (
hi16.to(tl.float32) * 1.52587890625e-05
+ (lo16.to(tl.float32) + 0.5) * 2.3283064365386963e-10
)
@triton.jit
def _uniform64_from_random53(random53):
uniform = (random53.to(tl.float64) + 0.5) * 1.1102230246251565e-16
# The largest midpoint rounds to 1.0 in fp64; keep the uniform open without
# relying on a near-one literal that Triton may materialize in fp32.
return tl.where(uniform == 1.0, uniform - 1.1102230246251565e-16, uniform)
@triton.jit
def murmur3_uniform64(seed, pos, offset):
lo = murmur3_hash32(seed, pos, offset).to(tl.uint64)
hi = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.uint64)
random53 = ((hi << 32) | lo) >> 11
return _uniform64_from_random53(random53)
@triton.jit
def _log1p_neg_stable(value):
# Preserve precision for the positive Gumbel tail without relying on a
# backend-specific libdevice log1p. The degree-8 series has absolute error
# below 6e-7 on [0, 0.25]; subtraction is well-conditioned elsewhere for
# the part of the distribution that can win the argmax.
polynomial = 1.0 / 8.0
polynomial = 1.0 / 7.0 + value * polynomial
polynomial = 1.0 / 6.0 + value * polynomial
polynomial = 1.0 / 5.0 + value * polynomial
polynomial = 1.0 / 4.0 + value * polynomial
polynomial = 1.0 / 3.0 + value * polynomial
polynomial = 1.0 / 2.0 + value * polynomial
polynomial = 1.0 + value * polynomial
series = -value * polynomial
direct = tl.log(tl.maximum(1.0 - value, 5.960464477539063e-08))
return tl.where(value < 0.25, series, direct)
@triton.jit
def gumbel_noised_argmax(
logits,
@@ -117,14 +208,15 @@ def gumbel_noised_argmax(
if temp != 0.0:
if IS_DRAFTING:
pos = pos + _DRAFT_NOISE_SALT
gumbel_seed = tl.randint(seed, pos)
if USE_FP64:
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
u = murmur3_uniform64(seed, pos, keys)
gumbel_noise = -tl.log(-tl.log(u))
else:
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
gumbel_noise = -tl.log(-tldevice.log1p(-u))
u = murmur3_uniform32(seed, pos, keys)
# Draw the large-noise tail (which decides the argmax winner) from
# u -> 0, where fp32 has fine resolution. Avoid backend-specific
# log1p while preserving precision in the winning tail.
gumbel_noise = -tl.log(-_log1p_neg_stable(u))
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
return tl.max(logits, axis=0, return_indices=True)