[Kernel] Use Murmur3 RNG for Gumbel sampling (#51367)
Signed-off-by: freyfwt <110148731+freyfwt@users.noreply.github.com> Signed-off-by: freyfwt <freytian1996@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
0f8fa53acf
commit
18553c5c9f
@@ -948,7 +948,7 @@ def test_structured_output_batched_with_non_structured_outputs_requests(
|
||||
prompts = [structured_outputs_prompt, non_structured_outputs_prompt]
|
||||
sampling_params = [
|
||||
SamplingParams(
|
||||
temperature=1.0,
|
||||
temperature=0,
|
||||
max_tokens=400,
|
||||
structured_outputs=StructuredOutputsParams(json=sample_json_schema),
|
||||
),
|
||||
|
||||
@@ -45,7 +45,8 @@ async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
extra_body={"include_reasoning": True},
|
||||
)
|
||||
|
||||
@@ -63,7 +64,8 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI)
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
@@ -99,7 +101,8 @@ async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": True},
|
||||
)
|
||||
@@ -130,7 +133,8 @@ async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
max_tokens=512,
|
||||
temperature=0,
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
@@ -353,7 +353,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
|
||||
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50 # we want some to finish earlier than others
|
||||
max_tokens = 50
|
||||
|
||||
# High temperature to maximize chance of unique completions.
|
||||
completion = await client.completions.create(
|
||||
@@ -371,16 +371,12 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
|
||||
num_completions = len(completion.choices)
|
||||
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
|
||||
completion_repeats: dict[str, int] = {}
|
||||
output_token_lengths = set()
|
||||
for idx, choice in enumerate(completion.choices):
|
||||
# Assert correct completion index & some finish reason.
|
||||
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
|
||||
assert choice.finish_reason is not None, "None finish_reason is invalid."
|
||||
text = choice.text
|
||||
completion_repeats[text] = completion_repeats.get(text, 0) + 1
|
||||
output_token_lengths.add(len(choice.logprobs.tokens))
|
||||
# Assert subrequests finished at different times
|
||||
assert len(output_token_lengths) > 1
|
||||
# Assert `n` unique completions
|
||||
num_unique = len(completion_repeats)
|
||||
if num_unique != n:
|
||||
@@ -403,7 +399,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
|
||||
prompt = "What is an LLM?"
|
||||
n = 3
|
||||
max_tokens = 50 # we want some to finish earlier than others
|
||||
max_tokens = 50
|
||||
|
||||
stream = await client.completions.create(
|
||||
model=model_name,
|
||||
@@ -427,19 +423,15 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
|
||||
f"Expected {n} completions with valid indices and finish_reason."
|
||||
)
|
||||
completion_repeats: dict[str, int] = {}
|
||||
chunk_lengths = set()
|
||||
for chunk in chunks:
|
||||
chunk_len = len(chunk)
|
||||
# Assert correct number of completion tokens
|
||||
chunk_lengths.add(chunk_len)
|
||||
assert chunk_len <= max_tokens, (
|
||||
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
|
||||
)
|
||||
text = "".join(chunk)
|
||||
completion_repeats[text] = completion_repeats.get(text, 0) + 1
|
||||
print(text)
|
||||
# Assert subrequests finished at different times
|
||||
assert len(chunk_lengths) > 1
|
||||
# Assert `n` unique completions
|
||||
num_unique = len(completion_repeats)
|
||||
if num_unique != n:
|
||||
|
||||
@@ -21,7 +21,12 @@ pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True)
|
||||
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import (
|
||||
_uniform64_from_random53,
|
||||
gumbel_sample,
|
||||
murmur3_hash32,
|
||||
)
|
||||
|
||||
DEVICE = "cuda"
|
||||
VOCAB_SIZE = 200_000
|
||||
@@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0
|
||||
Z_TOLERANCE = 10.0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_known_answer_kernel(
|
||||
seeds_ptr,
|
||||
positions_ptr,
|
||||
offsets_ptr,
|
||||
hashes_ptr,
|
||||
domain_hashes_ptr,
|
||||
numel: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
idx = tl.arange(0, BLOCK_SIZE)
|
||||
mask = idx < numel
|
||||
seed = tl.load(seeds_ptr + idx, mask=mask)
|
||||
pos = tl.load(positions_ptr + idx, mask=mask)
|
||||
offset = tl.load(offsets_ptr + idx, mask=mask)
|
||||
hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64)
|
||||
domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64)
|
||||
tl.store(hashes_ptr + idx, hash32, mask=mask)
|
||||
tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _uniform64_endpoint_kernel(
|
||||
random53_ptr,
|
||||
uniform_ptr,
|
||||
numel: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
idx = tl.arange(0, BLOCK_SIZE)
|
||||
mask = idx < numel
|
||||
random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64)
|
||||
tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask)
|
||||
|
||||
|
||||
def test_murmur3_known_answers():
|
||||
seeds = torch.tensor(
|
||||
[0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
positions = torch.tensor(
|
||||
[0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
offsets = torch.tensor(
|
||||
[0, 3, 1023, 199999, 4096, 0xFFFFFFFF],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
hashes = torch.empty_like(seeds)
|
||||
domain_hashes = torch.empty_like(seeds)
|
||||
|
||||
_murmur3_known_answer_kernel[(1,)](
|
||||
seeds,
|
||||
positions,
|
||||
offsets,
|
||||
hashes,
|
||||
domain_hashes,
|
||||
numel=seeds.numel(),
|
||||
BLOCK_SIZE=8,
|
||||
)
|
||||
|
||||
assert hashes.tolist() == [
|
||||
0x8134CDF8,
|
||||
0x23B53F91,
|
||||
0xC8C13C14,
|
||||
0x5DDE974F,
|
||||
0xF1A9355B,
|
||||
0xD952F9AA,
|
||||
]
|
||||
assert domain_hashes.tolist() == [
|
||||
0xD28AD962,
|
||||
0xE3AAF818,
|
||||
0xD223C51C,
|
||||
0x0547D2FE,
|
||||
0x3AED160F,
|
||||
0x49E8EA7F,
|
||||
]
|
||||
|
||||
|
||||
def test_uniform64_excludes_endpoints():
|
||||
random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE)
|
||||
uniform = torch.empty(2, dtype=torch.float64, device=DEVICE)
|
||||
|
||||
_uniform64_endpoint_kernel[(1,)](
|
||||
random53,
|
||||
uniform,
|
||||
numel=random53.numel(),
|
||||
BLOCK_SIZE=2,
|
||||
)
|
||||
|
||||
assert torch.isfinite(uniform).all()
|
||||
assert ((uniform > 0.0) & (uniform < 1.0)).all()
|
||||
assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53]
|
||||
|
||||
|
||||
def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor:
|
||||
"""Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N."""
|
||||
gen = torch.Generator(device=DEVICE).manual_seed(seed)
|
||||
@@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity():
|
||||
|
||||
|
||||
def test_drafting_uses_a_separate_noise_stream():
|
||||
"""is_drafting salts the Philox offset: same inputs, different draws.
|
||||
"""is_drafting salts the RNG position: same inputs, different draws.
|
||||
|
||||
The draft proposal and the residual resample after a rejection must be
|
||||
independent. They key noise by the same (seed, pos), so only the salt keeps
|
||||
@@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream():
|
||||
tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional
|
||||
consequence.
|
||||
|
||||
Relocating the offset must not distort the draw either, so both streams are
|
||||
Salting the position must not distort the draw either, so both streams are
|
||||
checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits
|
||||
below the head -- the regime where fp32 Gumbel precision matters.
|
||||
"""
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import HAS_TRITON, tl, tldevice, triton
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used to clamp
|
||||
# zero draws before the flipped Gumbel transform below.
|
||||
# Smallest positive value produced by Triton's fp32 `tl.rand`. Used by the
|
||||
# retained Philox helper for rejection sampling.
|
||||
#
|
||||
# Triton requires globals accessed from `@triton.jit` functions to be wrapped
|
||||
# in `tl.constexpr(...)`. We can only do that when Triton is actually
|
||||
@@ -87,6 +87,97 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_rotl32(value, shift: tl.constexpr):
|
||||
return (value << shift) | (value >> (32 - shift))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_mix(h, key):
|
||||
key *= 0xCC9E2D51
|
||||
key = _murmur3_rotl32(key, 15)
|
||||
key *= 0x1B873593
|
||||
h ^= key
|
||||
h = _murmur3_rotl32(h, 13)
|
||||
return h * 5 + 0xE6546B64
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_fmix32(h):
|
||||
h ^= h >> 16
|
||||
h *= 0x85EBCA6B
|
||||
h ^= h >> 13
|
||||
h *= 0xC2B2AE35
|
||||
return h ^ (h >> 16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def murmur3_hash32(seed, pos, offset, domain: tl.constexpr = 0):
|
||||
seed = seed.to(tl.int64)
|
||||
pos = pos.to(tl.int64)
|
||||
offset = offset.to(tl.uint32)
|
||||
# Keep the request-wide prefix scalar until the token offset is mixed in.
|
||||
h = (seed ^ seed).to(tl.uint32)
|
||||
h ^= domain
|
||||
h = _murmur3_mix(h, (seed & 0xFFFFFFFF).to(tl.uint32))
|
||||
h = _murmur3_mix(h, ((seed >> 32) & 0xFFFFFFFF).to(tl.uint32))
|
||||
h = _murmur3_mix(h, (pos & 0xFFFFFFFF).to(tl.uint32))
|
||||
h = _murmur3_mix(h, offset)
|
||||
return _murmur3_fmix32(h ^ 16)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def murmur3_uniform32(seed, pos, offset):
|
||||
random32 = murmur3_hash32(seed, pos, offset)
|
||||
# Split the uint32 before converting to fp32 so backends without a native
|
||||
# uint32-to-float conversion can still use all 32 source bits. Both 16-bit
|
||||
# halves convert exactly; their sum is the correctly rounded fp32 value of
|
||||
# (random32 + 0.5) * 2**-32. In particular, the u -> 0 winning tail keeps
|
||||
# the full 32-bit source resolution instead of being truncated to 24 bits.
|
||||
hi16 = (random32 >> 16).to(tl.int32)
|
||||
lo16 = (random32 & 0xFFFF).to(tl.int32)
|
||||
return (
|
||||
hi16.to(tl.float32) * 1.52587890625e-05
|
||||
+ (lo16.to(tl.float32) + 0.5) * 2.3283064365386963e-10
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _uniform64_from_random53(random53):
|
||||
uniform = (random53.to(tl.float64) + 0.5) * 1.1102230246251565e-16
|
||||
# The largest midpoint rounds to 1.0 in fp64; keep the uniform open without
|
||||
# relying on a near-one literal that Triton may materialize in fp32.
|
||||
return tl.where(uniform == 1.0, uniform - 1.1102230246251565e-16, uniform)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def murmur3_uniform64(seed, pos, offset):
|
||||
lo = murmur3_hash32(seed, pos, offset).to(tl.uint64)
|
||||
hi = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.uint64)
|
||||
random53 = ((hi << 32) | lo) >> 11
|
||||
return _uniform64_from_random53(random53)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _log1p_neg_stable(value):
|
||||
# Preserve precision for the positive Gumbel tail without relying on a
|
||||
# backend-specific libdevice log1p. The degree-8 series has absolute error
|
||||
# below 6e-7 on [0, 0.25]; subtraction is well-conditioned elsewhere for
|
||||
# the part of the distribution that can win the argmax.
|
||||
polynomial = 1.0 / 8.0
|
||||
polynomial = 1.0 / 7.0 + value * polynomial
|
||||
polynomial = 1.0 / 6.0 + value * polynomial
|
||||
polynomial = 1.0 / 5.0 + value * polynomial
|
||||
polynomial = 1.0 / 4.0 + value * polynomial
|
||||
polynomial = 1.0 / 3.0 + value * polynomial
|
||||
polynomial = 1.0 / 2.0 + value * polynomial
|
||||
polynomial = 1.0 + value * polynomial
|
||||
series = -value * polynomial
|
||||
|
||||
direct = tl.log(tl.maximum(1.0 - value, 5.960464477539063e-08))
|
||||
return tl.where(value < 0.25, series, direct)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def gumbel_noised_argmax(
|
||||
logits,
|
||||
@@ -117,14 +208,15 @@ def gumbel_noised_argmax(
|
||||
if temp != 0.0:
|
||||
if IS_DRAFTING:
|
||||
pos = pos + _DRAFT_NOISE_SALT
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
if USE_FP64:
|
||||
u = tl_rand64(gumbel_seed, keys, includes_zero=False)
|
||||
u = murmur3_uniform64(seed, pos, keys)
|
||||
gumbel_noise = -tl.log(-tl.log(u))
|
||||
else:
|
||||
u = tl_rand32(gumbel_seed, keys, includes_zero=False)
|
||||
# log1p keeps the winning tail at u -> 0, where fp32 resolves it.
|
||||
gumbel_noise = -tl.log(-tldevice.log1p(-u))
|
||||
u = murmur3_uniform32(seed, pos, keys)
|
||||
# Draw the large-noise tail (which decides the argmax winner) from
|
||||
# u -> 0, where fp32 has fine resolution. Avoid backend-specific
|
||||
# log1p while preserving precision in the winning tail.
|
||||
gumbel_noise = -tl.log(-_log1p_neg_stable(u))
|
||||
logits = tl.where(mask, logits + gumbel_noise, float("-inf"))
|
||||
|
||||
return tl.max(logits, axis=0, return_indices=True)
|
||||
|
||||
Reference in New Issue
Block a user