[Kernel] Use Murmur3 RNG for Gumbel sampling (#51367)

Signed-off-by: freyfwt <110148731+freyfwt@users.noreply.github.com>
Signed-off-by: freyfwt <freytian1996@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
freyfwt
2026-09-15 23:52:26 -07:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jiangyun Zhu
parent 0f8fa53acf
commit 18553c5c9f
5 changed files with 216 additions and 26 deletions
@@ -948,7 +948,7 @@ def test_structured_output_batched_with_non_structured_outputs_requests(
prompts = [structured_outputs_prompt, non_structured_outputs_prompt]
sampling_params = [
SamplingParams(
temperature=1.0,
temperature=0,
max_tokens=400,
structured_outputs=StructuredOutputsParams(json=sample_json_schema),
),
@@ -45,7 +45,8 @@ async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
extra_body={"include_reasoning": True},
)
@@ -63,7 +64,8 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI)
response = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
extra_body={"include_reasoning": False},
)
@@ -99,7 +101,8 @@ async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
stream=True,
extra_body={"include_reasoning": True},
)
@@ -130,7 +133,8 @@ async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=200,
max_tokens=512,
temperature=0,
stream=True,
extra_body={"include_reasoning": False},
)
@@ -353,7 +353,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
prompt = "What is an LLM?"
n = 3
max_tokens = 50 # we want some to finish earlier than others
max_tokens = 50
# High temperature to maximize chance of unique completions.
completion = await client.completions.create(
@@ -371,16 +371,12 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str
num_completions = len(completion.choices)
assert num_completions == n, f"Num completions {num_completions} but expected {n}."
completion_repeats: dict[str, int] = {}
output_token_lengths = set()
for idx, choice in enumerate(completion.choices):
# Assert correct completion index & some finish reason.
assert choice.index == idx, f"Index {choice.index} but expected {idx}."
assert choice.finish_reason is not None, "None finish_reason is invalid."
text = choice.text
completion_repeats[text] = completion_repeats.get(text, 0) + 1
output_token_lengths.add(len(choice.logprobs.tokens))
# Assert subrequests finished at different times
assert len(output_token_lengths) > 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
@@ -403,7 +399,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
prompt = "What is an LLM?"
n = 3
max_tokens = 50 # we want some to finish earlier than others
max_tokens = 50
stream = await client.completions.create(
model=model_name,
@@ -427,19 +423,15 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
f"Expected {n} completions with valid indices and finish_reason."
)
completion_repeats: dict[str, int] = {}
chunk_lengths = set()
for chunk in chunks:
chunk_len = len(chunk)
# Assert correct number of completion tokens
chunk_lengths.add(chunk_len)
assert chunk_len <= max_tokens, (
f"max_tokens={max_tokens} but chunk len is {chunk_len}."
)
text = "".join(chunk)
completion_repeats[text] = completion_repeats.get(text, 0) + 1
print(text)
# Assert subrequests finished at different times
assert len(chunk_lengths) > 1
# Assert `n` unique completions
num_unique = len(completion_repeats)
if num_unique != n:
+105 -3
View File
@@ -21,7 +21,12 @@ pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True)
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.sample.gumbel import (
_uniform64_from_random53,
gumbel_sample,
murmur3_hash32,
)
DEVICE = "cuda"
VOCAB_SIZE = 200_000
@@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0
Z_TOLERANCE = 10.0
@triton.jit
def _murmur3_known_answer_kernel(
seeds_ptr,
positions_ptr,
offsets_ptr,
hashes_ptr,
domain_hashes_ptr,
numel: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
idx = tl.arange(0, BLOCK_SIZE)
mask = idx < numel
seed = tl.load(seeds_ptr + idx, mask=mask)
pos = tl.load(positions_ptr + idx, mask=mask)
offset = tl.load(offsets_ptr + idx, mask=mask)
hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64)
domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64)
tl.store(hashes_ptr + idx, hash32, mask=mask)
tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask)
@triton.jit
def _uniform64_endpoint_kernel(
random53_ptr,
uniform_ptr,
numel: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
idx = tl.arange(0, BLOCK_SIZE)
mask = idx < numel
random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64)
tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask)
def test_murmur3_known_answers():
seeds = torch.tensor(
[0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1],
dtype=torch.int64,
device=DEVICE,
)
positions = torch.tensor(
[0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7],
dtype=torch.int64,
device=DEVICE,
)
offsets = torch.tensor(
[0, 3, 1023, 199999, 4096, 0xFFFFFFFF],
dtype=torch.int64,
device=DEVICE,
)
hashes = torch.empty_like(seeds)
domain_hashes = torch.empty_like(seeds)
_murmur3_known_answer_kernel[(1,)](
seeds,
positions,
offsets,
hashes,
domain_hashes,
numel=seeds.numel(),
BLOCK_SIZE=8,
)
assert hashes.tolist() == [
0x8134CDF8,
0x23B53F91,
0xC8C13C14,
0x5DDE974F,
0xF1A9355B,
0xD952F9AA,
]
assert domain_hashes.tolist() == [
0xD28AD962,
0xE3AAF818,
0xD223C51C,
0x0547D2FE,
0x3AED160F,
0x49E8EA7F,
]
def test_uniform64_excludes_endpoints():
random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE)
uniform = torch.empty(2, dtype=torch.float64, device=DEVICE)
_uniform64_endpoint_kernel[(1,)](
random53,
uniform,
numel=random53.numel(),
BLOCK_SIZE=2,
)
assert torch.isfinite(uniform).all()
assert ((uniform > 0.0) & (uniform < 1.0)).all()
assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53]
def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor:
"""Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N."""
gen = torch.Generator(device=DEVICE).manual_seed(seed)
@@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity():
def test_drafting_uses_a_separate_noise_stream():
"""is_drafting salts the Philox offset: same inputs, different draws.
"""is_drafting salts the RNG position: same inputs, different draws.
The draft proposal and the residual resample after a rejection must be
independent. They key noise by the same (seed, pos), so only the salt keeps
@@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream():
tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional
consequence.
Relocating the offset must not distort the draw either, so both streams are
Salting the position must not distort the draw either, so both streams are
checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits
below the head -- the regime where fp32 Gumbel precision matters.
"""