[Kernel] Use Murmur3 RNG for Gumbel sampling (#51367)
Signed-off-by: freyfwt <110148731+freyfwt@users.noreply.github.com> Signed-off-by: freyfwt <freytian1996@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
0f8fa53acf
commit
18553c5c9f
@@ -2,10 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import HAS_TRITON, tl, tldevice, triton
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used to clamp
|
||||
# zero draws before the flipped Gumbel transform below.
|
||||
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used by the
|
||||
# retained Philox helper for rejection sampling.
|
||||
#
|
||||
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
|
||||
# in `tl.constexpr(...)`. We can only do that when Triton is actually
|
||||
@@ -87,6 +87,97 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_rotl32(value, shift: tl.constexpr):
|
||||
return (value << shift) | (value >> (32 - shift))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_mix(h, key):
|
||||
key *= 0xCC9E2D51
|
||||
key = _murmur3_rotl32(key, 15)
|
||||
key *= 0x1B873593
|
||||
h ^= key
|
||||
h = _murmur3_rotl32(h, 13)
|
||||
return h * 5 + 0xE6546B64
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_fmix32(h):
|
||||
h ^= h >> 16
|
||||
h *= 0x85EBCA6B
|
||||
h ^= h >> 13
|
||||
h *= 0xC2B2AE35
|
||||
return h ^ (h >> 16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def murmur3_hash32(seed, pos, offset, domain: tl.constexpr = 0):
|
||||
seed = seed.to(tl.int64)
|
||||
pos = pos.to(tl.int64)
|
||||
offset = offset.to(tl.uint32)
|
||||
# Keep the request-wide prefix scalar until the token offset is mixed in.
|
||||
h = (seed ^ seed).to(tl.uint32)
|
||||
h ^= domain
|
||||
h = _murmur3_mix(h, (seed & 0xFFFFFFFF).to(tl.uint32))
|
||||
h = _murmur3_mix(h, ((seed >> 32) & 0xFFFFFFFF).to(tl.uint32))
|
||||
h = _murmur3_mix(h, (pos & 0xFFFFFFFF).to(tl.uint32))
|
||||
h = _murmur3_mix(h, offset)
|
||||
return _murmur3_fmix32(h ^ 16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def murmur3_uniform32(seed, pos, offset):
|
||||
random32 = murmur3_hash32(seed, pos, offset)
|
||||
# Split the uint32 before converting to fp32 so backends without a native
|
||||
# uint32-to-float conversion can still use all 32 source bits. Both 16-bit
|
||||
# halves convert exactly; their sum is the correctly rounded fp32 value of
|
||||
# (random32 + 0.5) * 2**-32. In particular, the u -> 0 winning tail keeps
|
||||
# the full 32-bit source resolution instead of being truncated to 24 bits.
|
||||
hi16 = (random32 >> 16).to(tl.int32)
|
||||
lo16 = (random32 & 0xFFFF).to(tl.int32)
|
||||
return (
|
||||
hi16.to(tl.float32) * 1.52587890625e-05
|
||||
+ (lo16.to(tl.float32) + 0.5) * 2.3283064365386963e-10
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _uniform64_from_random53(random53):
|
||||
uniform = (random53.to(tl.float64) + 0.5) * 1.1102230246251565e-16
|
||||
# The largest midpoint rounds to 1.0 in fp64; keep the uniform open without
|
||||
# relying on a near-one literal that Triton may materialize in fp32.
|
||||
return tl.where(uniform == 1.0, uniform - 1.1102230246251565e-16, uniform)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def murmur3_uniform64(seed, pos, offset):
|
||||
lo = murmur3_hash32(seed, pos, offset).to(tl.uint64)
|
||||
hi = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.uint64)
|
||||
random53 = ((hi << 32) | lo) >> 11
|
||||
return _uniform64_from_random53(random53)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _log1p_neg_stable(value):
|
||||
# Preserve precision for the positive Gumbel tail without relying on a
|
||||
# backend-specific libdevice log1p. The degree-8 series has absolute error
|
||||
# below 6e-7 on [0, 0.25]; subtraction is well-conditioned elsewhere for
|
||||
# the part of the distribution that can win the argmax.
|
||||
polynomial = 1.0 / 8.0
|
||||
polynomial = 1.0 / 7.0 + value * polynomial
|
||||
polynomial = 1.0 / 6.0 + value * polynomial
|
||||
polynomial = 1.0 / 5.0 + value * polynomial
|
||||
polynomial = 1.0 / 4.0 + value * polynomial
|
||||
polynomial = 1.0 / 3.0 + value * polynomial
|
||||
polynomial = 1.0 / 2.0 + value * polynomial
|
||||
polynomial = 1.0 + value * polynomial
|
||||
series = -value * polynomial
|
||||
|
||||
direct = tl.log(tl.maximum(1.0 - value, 5.960464477539063e-08))
|
||||
return tl.where(value < 0.25, series, direct)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_noised_argmax(
|
||||
logits,
|
||||
@@ -117,14 +208,15 @@ def gumbel_noised_argmax(
|
||||
if temp != 0.0:
|
||||
if IS_DRAFTING:
|
||||
pos = pos + _DRAFT_NOISE_SALT
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
|
||||
u = murmur3_uniform64(seed, pos, keys)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
|
||||
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
|
||||
gumbel_noise = -tl.log(-tldevice.log1p(-u))
|
||||
u = murmur3_uniform32(seed, pos, keys)
|
||||
# Draw the large-noise tail (which decides the argmax winner) from
|
||||
# u -> 0, where fp32 has fine resolution. Avoid backend-specific
|
||||
# log1p while preserving precision in the winning tail.
|
||||
gumbel_noise = -tl.log(-_log1p_neg_stable(u))
|
||||
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
|
||||
|
||||
return tl.max(logits, axis=0, return_indices=True)
|
||||
|
||||
Reference in New Issue
Block a user