[Kernel] Use Murmur3 RNG for Gumbel sampling (#51367)

Signed-off-by: freyfwt <110148731+freyfwt@users.noreply.github.com>
Signed-off-by: freyfwt <freytian1996@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
freyfwt
2026-09-15 23:52:26 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jiangyun Zhu
parent 0f8fa53acf
commit 18553c5c9f
5 changed files with 216 additions and 26 deletions
@@ -948,7 +948,7 @@ def test_structured_output_batched_with_non_structured_outputs_requests(
prompts = [structured_outputs_prompt, non_structured_outputs_prompt]
sampling_params = [
SamplingParams(
temperature=1.0,
temperature=0,
max_tokens=400,
structured_outputs=StructuredOutputsParams(json=sample_json_schema),
),
@@ -45,7 +45,8 @@ async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
extra_body={"include_reasoning": True},
)
@@ -63,7 +64,8 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI)
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
extra_body={"include_reasoning": False},
)
@@ -99,7 +101,8 @@ async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
stream=True,
extra_body={"include_reasoning": True},
)
@@ -130,7 +133,8 @@ async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
stream=True,
extra_body={"include_reasoning": False},
)
@@ -353,7 +353,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
prompt = "What is an LLM?"
n = 3
max_tokens = 50 # we want some to finish earlier than others
max_tokens = 50
# High temperature to maximize chance of unique completions.
completion = await client.completions.create(
@@ -371,16 +371,12 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
num_completions = len(completion.choices)
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
completion_repeats: dict[str, int] = {}
output_token_lengths = set()
for idx, choice in enumerate(completion.choices):
# Assert correct completion index & some finish reason.
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
assert choice.finish_reason is not None, "None finish_reason is invalid."
text = choice.text
completion_repeats[text] = completion_repeats.get(text, 0) + 1
output_token_lengths.add(len(choice.logprobs.tokens))
# Assert subrequests finished at different times
assert len(output_token_lengths) > 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
@@ -403,7 +399,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
prompt = "What is an LLM?"
n = 3
max_tokens = 50 # we want some to finish earlier than others
max_tokens = 50
stream = await client.completions.create(
model=model_name,
@@ -427,19 +423,15 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
f"Expected {n} completions with valid indices and finish_reason."
)
completion_repeats: dict[str, int] = {}
chunk_lengths = set()
for chunk in chunks:
chunk_len = len(chunk)
# Assert correct number of completion tokens
chunk_lengths.add(chunk_len)
assert chunk_len <= max_tokens, (
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
)
text = "".join(chunk)
completion_repeats[text] = completion_repeats.get(text, 0) + 1
print(text)
# Assert subrequests finished at different times
assert len(chunk_lengths) > 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
+105 -3
View File
@@ -21,7 +21,12 @@ pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True)
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.sample.gumbel import (
_uniform64_from_random53,
gumbel_sample,
murmur3_hash32,
)
DEVICE = "cuda"
VOCAB_SIZE = 200_000
@@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0
Z_TOLERANCE = 10.0
@triton.jit
def _murmur3_known_answer_kernel(
seeds_ptr,
positions_ptr,
offsets_ptr,
hashes_ptr,
domain_hashes_ptr,
numel: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
idx = tl.arange(0, BLOCK_SIZE)
mask = idx < numel
seed = tl.load(seeds_ptr + idx, mask=mask)
pos = tl.load(positions_ptr + idx, mask=mask)
offset = tl.load(offsets_ptr + idx, mask=mask)
hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64)
domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64)
tl.store(hashes_ptr + idx, hash32, mask=mask)
tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask)
@triton.jit
def _uniform64_endpoint_kernel(
random53_ptr,
uniform_ptr,
numel: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
idx = tl.arange(0, BLOCK_SIZE)
mask = idx < numel
random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64)
tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask)
def test_murmur3_known_answers():
seeds = torch.tensor(
[0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1],
dtype=torch.int64,
device=DEVICE,
)
positions = torch.tensor(
[0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7],
dtype=torch.int64,
device=DEVICE,
)
offsets = torch.tensor(
[0, 3, 1023, 199999, 4096, 0xFFFFFFFF],
dtype=torch.int64,
device=DEVICE,
)
hashes = torch.empty_like(seeds)
domain_hashes = torch.empty_like(seeds)
_murmur3_known_answer_kernel[(1,)](
seeds,
positions,
offsets,
hashes,
domain_hashes,
numel=seeds.numel(),
BLOCK_SIZE=8,
)
assert hashes.tolist() == [
0x8134CDF8,
0x23B53F91,
0xC8C13C14,
0x5DDE974F,
0xF1A9355B,
0xD952F9AA,
]
assert domain_hashes.tolist() == [
0xD28AD962,
0xE3AAF818,
0xD223C51C,
0x0547D2FE,
0x3AED160F,
0x49E8EA7F,
]
def test_uniform64_excludes_endpoints():
random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE)
uniform = torch.empty(2, dtype=torch.float64, device=DEVICE)
_uniform64_endpoint_kernel[(1,)](
random53,
uniform,
numel=random53.numel(),
BLOCK_SIZE=2,
)
assert torch.isfinite(uniform).all()
assert ((uniform > 0.0) & (uniform < 1.0)).all()
assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53]
def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor:
"""Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N."""
gen = torch.Generator(device=DEVICE).manual_seed(seed)
@@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity():
def test_drafting_uses_a_separate_noise_stream():
"""is_drafting salts the Philox offset: same inputs, different draws.
"""is_drafting salts the RNG position: same inputs, different draws.
The draft proposal and the residual resample after a rejection must be
independent. They key noise by the same (seed, pos), so only the salt keeps
@@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream():
tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional
consequence.
Relocating the offset must not distort the draw either, so both streams are
Salting the position must not distort the draw either, so both streams are
checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits
below the head -- the regime where fp32 Gumbel precision matters.
"""
+100 -8
View File
@@ -2,10 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import HAS_TRITON, tl, tldevice, triton
from vllm.triton_utils import HAS_TRITON, tl, triton
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used to clamp
# zero draws before the flipped Gumbel transform below.
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used by the
# retained Philox helper for rejection sampling.
#
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
# in `tl.constexpr(...)`. We can only do that when Triton is actually
@@ -87,6 +87,97 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
return u
@triton.jit
def _murmur3_rotl32(value, shift: tl.constexpr):
return (value << shift) | (value >> (32 - shift))
@triton.jit
def _murmur3_mix(h, key):
key *= 0xCC9E2D51
key = _murmur3_rotl32(key, 15)
key *= 0x1B873593
h ^= key
h = _murmur3_rotl32(h, 13)
return h * 5 + 0xE6546B64
@triton.jit
def _murmur3_fmix32(h):
h ^= h >> 16
h *= 0x85EBCA6B
h ^= h >> 13
h *= 0xC2B2AE35
return h ^ (h >> 16)
@triton.jit
def murmur3_hash32(seed, pos, offset, domain: tl.constexpr = 0):
seed = seed.to(tl.int64)
pos = pos.to(tl.int64)
offset = offset.to(tl.uint32)
# Keep the request-wide prefix scalar until the token offset is mixed in.
h = (seed ^ seed).to(tl.uint32)
h ^= domain
h = _murmur3_mix(h, (seed & 0xFFFFFFFF).to(tl.uint32))
h = _murmur3_mix(h, ((seed >> 32) & 0xFFFFFFFF).to(tl.uint32))
h = _murmur3_mix(h, (pos & 0xFFFFFFFF).to(tl.uint32))
h = _murmur3_mix(h, offset)
return _murmur3_fmix32(h ^ 16)
@triton.jit
def murmur3_uniform32(seed, pos, offset):
random32 = murmur3_hash32(seed, pos, offset)
# Split the uint32 before converting to fp32 so backends without a native
# uint32-to-float conversion can still use all 32 source bits. Both 16-bit
# halves convert exactly; their sum is the correctly rounded fp32 value of
# (random32 + 0.5) * 2**-32. In particular, the u -> 0 winning tail keeps
# the full 32-bit source resolution instead of being truncated to 24 bits.
hi16 = (random32 >> 16).to(tl.int32)
lo16 = (random32 & 0xFFFF).to(tl.int32)
return (
hi16.to(tl.float32) * 1.52587890625e-05
+ (lo16.to(tl.float32) + 0.5) * 2.3283064365386963e-10
)
@triton.jit
def _uniform64_from_random53(random53):
uniform = (random53.to(tl.float64) + 0.5) * 1.1102230246251565e-16
# The largest midpoint rounds to 1.0 in fp64; keep the uniform open without
# relying on a near-one literal that Triton may materialize in fp32.
return tl.where(uniform == 1.0, uniform - 1.1102230246251565e-16, uniform)
@triton.jit
def murmur3_uniform64(seed, pos, offset):
lo = murmur3_hash32(seed, pos, offset).to(tl.uint64)
hi = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.uint64)
random53 = ((hi << 32) | lo) >> 11
return _uniform64_from_random53(random53)
@triton.jit
def _log1p_neg_stable(value):
# Preserve precision for the positive Gumbel tail without relying on a
# backend-specific libdevice log1p. The degree-8 series has absolute error
# below 6e-7 on [0, 0.25]; subtraction is well-conditioned elsewhere for
# the part of the distribution that can win the argmax.
polynomial = 1.0 / 8.0
polynomial = 1.0 / 7.0 + value * polynomial
polynomial = 1.0 / 6.0 + value * polynomial
polynomial = 1.0 / 5.0 + value * polynomial
polynomial = 1.0 / 4.0 + value * polynomial
polynomial = 1.0 / 3.0 + value * polynomial
polynomial = 1.0 / 2.0 + value * polynomial
polynomial = 1.0 + value * polynomial
series = -value * polynomial
direct = tl.log(tl.maximum(1.0 - value, 5.960464477539063e-08))
return tl.where(value < 0.25, series, direct)
@triton.jit
def gumbel_noised_argmax(
logits,
@@ -117,14 +208,15 @@ def gumbel_noised_argmax(
if temp != 0.0:
if IS_DRAFTING:
pos = pos + _DRAFT_NOISE_SALT
gumbel_seed = tl.randint(seed, pos)
if USE_FP64:
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
u = murmur3_uniform64(seed, pos, keys)
gumbel_noise = -tl.log(-tl.log(u))
else:
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
gumbel_noise = -tl.log(-tldevice.log1p(-u))
u = murmur3_uniform32(seed, pos, keys)
# Draw the large-noise tail (which decides the argmax winner) from
# u -> 0, where fp32 has fine resolution. Avoid backend-specific
# log1p while preserving precision in the winning tail.
gumbel_noise = -tl.log(-_log1p_neg_stable(u))
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
return tl.max(logits, axis=0, return_indices=True)