diff --git a/tests/entrypoints/llm/test_struct_output_generate.py b/tests/entrypoints/llm/test_struct_output_generate.py index 0f55ad7adc..030b0207e4 100644 --- a/tests/entrypoints/llm/test_struct_output_generate.py +++ b/tests/entrypoints/llm/test_struct_output_generate.py @@ -948,7 +948,7 @@ def test_structured_output_batched_with_non_structured_outputs_requests( prompts = [structured_outputs_prompt, non_structured_outputs_prompt] sampling_params = [ SamplingParams( - temperature=1.0, + temperature=0, max_tokens=400, structured_outputs=StructuredOutputsParams(json=sample_json_schema), ), diff --git a/tests/entrypoints/openai/chat_completion/test_include_reasoning.py b/tests/entrypoints/openai/chat_completion/test_include_reasoning.py index 1d4e1ebc40..5eeca438fe 100644 --- a/tests/entrypoints/openai/chat_completion/test_include_reasoning.py +++ b/tests/entrypoints/openai/chat_completion/test_include_reasoning.py @@ -45,7 +45,8 @@ async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI): response = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES, - max_tokens=200, + max_tokens=512, + temperature=0, extra_body={"include_reasoning": True}, ) @@ -63,7 +64,8 @@ async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI) response = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES, - max_tokens=200, + max_tokens=512, + temperature=0, extra_body={"include_reasoning": False}, ) @@ -99,7 +101,8 @@ async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI): stream = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES, - max_tokens=200, + max_tokens=512, + temperature=0, stream=True, extra_body={"include_reasoning": True}, ) @@ -130,7 +133,8 @@ async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI): stream = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES, - max_tokens=200, + max_tokens=512, + temperature=0, stream=True, extra_body={"include_reasoning": False}, ) diff --git a/tests/entrypoints/openai/completion/test_completion.py b/tests/entrypoints/openai/completion/test_completion.py index af9f1def36..1bc6260cde 100644 --- a/tests/entrypoints/openai/completion/test_completion.py +++ b/tests/entrypoints/openai/completion/test_completion.py @@ -353,7 +353,7 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str prompt = "What is an LLM?" n = 3 - max_tokens = 50 # we want some to finish earlier than others + max_tokens = 50 # High temperature to maximize chance of unique completions. completion = await client.completions.create( @@ -371,16 +371,12 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str num_completions = len(completion.choices) assert num_completions == n, f"Num completions {num_completions} but expected {n}." completion_repeats: dict[str, int] = {} - output_token_lengths = set() for idx, choice in enumerate(completion.choices): # Assert correct completion index & some finish reason. assert choice.index == idx, f"Index {choice.index} but expected {idx}." assert choice.finish_reason is not None, "None finish_reason is invalid." text = choice.text completion_repeats[text] = completion_repeats.get(text, 0) + 1 - output_token_lengths.add(len(choice.logprobs.tokens)) - # Assert subrequests finished at different times - assert len(output_token_lengths) > 1 # Assert `n` unique completions num_unique = len(completion_repeats) if num_unique != n: @@ -403,7 +399,7 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str): prompt = "What is an LLM?" n = 3 - max_tokens = 50 # we want some to finish earlier than others + max_tokens = 50 stream = await client.completions.create( model=model_name, @@ -427,19 +423,15 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str): f"Expected {n} completions with valid indices and finish_reason." ) completion_repeats: dict[str, int] = {} - chunk_lengths = set() for chunk in chunks: chunk_len = len(chunk) # Assert correct number of completion tokens - chunk_lengths.add(chunk_len) assert chunk_len <= max_tokens, ( f"max_tokens={max_tokens} but chunk len is {chunk_len}." ) text = "".join(chunk) completion_repeats[text] = completion_repeats.get(text, 0) + 1 print(text) - # Assert subrequests finished at different times - assert len(chunk_lengths) > 1 # Assert `n` unique completions num_unique = len(completion_repeats) if num_unique != n: diff --git a/tests/v1/worker/test_gpu_gumbel_sample.py b/tests/v1/worker/test_gpu_gumbel_sample.py index 10a09b9de4..a66a1395d2 100644 --- a/tests/v1/worker/test_gpu_gumbel_sample.py +++ b/tests/v1/worker/test_gpu_gumbel_sample.py @@ -21,7 +21,12 @@ pytest.importorskip("triton") if not torch.cuda.is_available(): pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True) -from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample +from vllm.triton_utils import tl, triton +from vllm.v1.worker.gpu.sample.gumbel import ( + _uniform64_from_random53, + gumbel_sample, + murmur3_hash32, +) DEVICE = "cuda" VOCAB_SIZE = 200_000 @@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0 Z_TOLERANCE = 10.0 +@triton.jit +def _murmur3_known_answer_kernel( + seeds_ptr, + positions_ptr, + offsets_ptr, + hashes_ptr, + domain_hashes_ptr, + numel: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + idx = tl.arange(0, BLOCK_SIZE) + mask = idx < numel + seed = tl.load(seeds_ptr + idx, mask=mask) + pos = tl.load(positions_ptr + idx, mask=mask) + offset = tl.load(offsets_ptr + idx, mask=mask) + hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64) + domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64) + tl.store(hashes_ptr + idx, hash32, mask=mask) + tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask) + + +@triton.jit +def _uniform64_endpoint_kernel( + random53_ptr, + uniform_ptr, + numel: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + idx = tl.arange(0, BLOCK_SIZE) + mask = idx < numel + random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64) + tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask) + + +def test_murmur3_known_answers(): + seeds = torch.tensor( + [0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1], + dtype=torch.int64, + device=DEVICE, + ) + positions = torch.tensor( + [0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7], + dtype=torch.int64, + device=DEVICE, + ) + offsets = torch.tensor( + [0, 3, 1023, 199999, 4096, 0xFFFFFFFF], + dtype=torch.int64, + device=DEVICE, + ) + hashes = torch.empty_like(seeds) + domain_hashes = torch.empty_like(seeds) + + _murmur3_known_answer_kernel[(1,)]( + seeds, + positions, + offsets, + hashes, + domain_hashes, + numel=seeds.numel(), + BLOCK_SIZE=8, + ) + + assert hashes.tolist() == [ + 0x8134CDF8, + 0x23B53F91, + 0xC8C13C14, + 0x5DDE974F, + 0xF1A9355B, + 0xD952F9AA, + ] + assert domain_hashes.tolist() == [ + 0xD28AD962, + 0xE3AAF818, + 0xD223C51C, + 0x0547D2FE, + 0x3AED160F, + 0x49E8EA7F, + ] + + +def test_uniform64_excludes_endpoints(): + random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE) + uniform = torch.empty(2, dtype=torch.float64, device=DEVICE) + + _uniform64_endpoint_kernel[(1,)]( + random53, + uniform, + numel=random53.numel(), + BLOCK_SIZE=2, + ) + + assert torch.isfinite(uniform).all() + assert ((uniform > 0.0) & (uniform < 1.0)).all() + assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53] + + def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor: """Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N.""" gen = torch.Generator(device=DEVICE).manual_seed(seed) @@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity(): def test_drafting_uses_a_separate_noise_stream(): - """is_drafting salts the Philox offset: same inputs, different draws. + """is_drafting salts the RNG position: same inputs, different draws. The draft proposal and the residual resample after a rejection must be independent. They key noise by the same (seed, pos), so only the salt keeps @@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream(): tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional consequence. - Relocating the offset must not distort the draw either, so both streams are + Salting the position must not distort the draw either, so both streams are checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits below the head -- the regime where fp32 Gumbel precision matters. """ diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index 4bf1f47280..7220212369 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -2,10 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch -from vllm.triton_utils import HAS_TRITON, tl, tldevice, triton +from vllm.triton_utils import HAS_TRITON, tl, triton -# Smallest positive value produced by Triton's fp32 `tl.rand`. Used to clamp -# zero draws before the flipped Gumbel transform below. +# Smallest positive value produced by Triton's fp32 `tl.rand`. Used by the +# retained Philox helper for rejection sampling. # # Triton requires globals accessed from `@triton.jit` functions to be wrapped # in `tl.constexpr(...)`. We can only do that when Triton is actually @@ -87,6 +87,97 @@ def tl_rand32(seed, offset, includes_zero: tl.constexpr): return u +@triton.jit +def _murmur3_rotl32(value, shift: tl.constexpr): + return (value << shift) | (value >> (32 - shift)) + + +@triton.jit +def _murmur3_mix(h, key): + key *= 0xCC9E2D51 + key = _murmur3_rotl32(key, 15) + key *= 0x1B873593 + h ^= key + h = _murmur3_rotl32(h, 13) + return h * 5 + 0xE6546B64 + + +@triton.jit +def _murmur3_fmix32(h): + h ^= h >> 16 + h *= 0x85EBCA6B + h ^= h >> 13 + h *= 0xC2B2AE35 + return h ^ (h >> 16) + + +@triton.jit +def murmur3_hash32(seed, pos, offset, domain: tl.constexpr = 0): + seed = seed.to(tl.int64) + pos = pos.to(tl.int64) + offset = offset.to(tl.uint32) + # Keep the request-wide prefix scalar until the token offset is mixed in. + h = (seed ^ seed).to(tl.uint32) + h ^= domain + h = _murmur3_mix(h, (seed & 0xFFFFFFFF).to(tl.uint32)) + h = _murmur3_mix(h, ((seed >> 32) & 0xFFFFFFFF).to(tl.uint32)) + h = _murmur3_mix(h, (pos & 0xFFFFFFFF).to(tl.uint32)) + h = _murmur3_mix(h, offset) + return _murmur3_fmix32(h ^ 16) + + +@triton.jit +def murmur3_uniform32(seed, pos, offset): + random32 = murmur3_hash32(seed, pos, offset) + # Split the uint32 before converting to fp32 so backends without a native + # uint32-to-float conversion can still use all 32 source bits. Both 16-bit + # halves convert exactly; their sum is the correctly rounded fp32 value of + # (random32 + 0.5) * 2**-32. In particular, the u -> 0 winning tail keeps + # the full 32-bit source resolution instead of being truncated to 24 bits. + hi16 = (random32 >> 16).to(tl.int32) + lo16 = (random32 & 0xFFFF).to(tl.int32) + return ( + hi16.to(tl.float32) * 1.52587890625e-05 + + (lo16.to(tl.float32) + 0.5) * 2.3283064365386963e-10 + ) + + +@triton.jit +def _uniform64_from_random53(random53): + uniform = (random53.to(tl.float64) + 0.5) * 1.1102230246251565e-16 + # The largest midpoint rounds to 1.0 in fp64; keep the uniform open without + # relying on a near-one literal that Triton may materialize in fp32. + return tl.where(uniform == 1.0, uniform - 1.1102230246251565e-16, uniform) + + +@triton.jit +def murmur3_uniform64(seed, pos, offset): + lo = murmur3_hash32(seed, pos, offset).to(tl.uint64) + hi = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.uint64) + random53 = ((hi << 32) | lo) >> 11 + return _uniform64_from_random53(random53) + + +@triton.jit +def _log1p_neg_stable(value): + # Preserve precision for the positive Gumbel tail without relying on a + # backend-specific libdevice log1p. The degree-8 series has absolute error + # below 6e-7 on [0, 0.25]; subtraction is well-conditioned elsewhere for + # the part of the distribution that can win the argmax. + polynomial = 1.0 / 8.0 + polynomial = 1.0 / 7.0 + value * polynomial + polynomial = 1.0 / 6.0 + value * polynomial + polynomial = 1.0 / 5.0 + value * polynomial + polynomial = 1.0 / 4.0 + value * polynomial + polynomial = 1.0 / 3.0 + value * polynomial + polynomial = 1.0 / 2.0 + value * polynomial + polynomial = 1.0 + value * polynomial + series = -value * polynomial + + direct = tl.log(tl.maximum(1.0 - value, 5.960464477539063e-08)) + return tl.where(value < 0.25, series, direct) + + @triton.jit def gumbel_noised_argmax( logits, @@ -117,14 +208,15 @@ def gumbel_noised_argmax( if temp != 0.0: if IS_DRAFTING: pos = pos + _DRAFT_NOISE_SALT - gumbel_seed = tl.randint(seed, pos) if USE_FP64: - u = tl_rand64(gumbel_seed, keys, includes_zero=False) + u = murmur3_uniform64(seed, pos, keys) gumbel_noise = -tl.log(-tl.log(u)) else: - u = tl_rand32(gumbel_seed, keys, includes_zero=False) - # log1p keeps the winning tail at u -> 0, where fp32 resolves it. - gumbel_noise = -tl.log(-tldevice.log1p(-u)) + u = murmur3_uniform32(seed, pos, keys) + # Draw the large-noise tail (which decides the argmax winner) from + # u -> 0, where fp32 has fine resolution. Avoid backend-specific + # log1p while preserving precision in the winning tail. + gumbel_noise = -tl.log(-_log1p_neg_stable(u)) logits = tl.where(mask, logits + gumbel_noise, float("-inf")) return tl.max(logits, axis=0, return_indices=True)