[Kernel] Use Murmur3 RNG for Gumbel sampling (#51367)
Signed-off-by: freyfwt <110148731+freyfwt@users.noreply.github.com> Signed-off-by: freyfwt <freytian1996@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Jiangyun Zhu
parent
0f8fa53acf
commit
18553c5c9f
@@ -21,7 +21,12 @@ pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for Gumbel sampler tests", allow_module_level=True)
|
||||
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.sample.gumbel import (
|
||||
_uniform64_from_random53,
|
||||
gumbel_sample,
|
||||
murmur3_hash32,
|
||||
)
|
||||
|
||||
DEVICE = "cuda"
|
||||
VOCAB_SIZE = 200_000
|
||||
@@ -33,6 +38,103 @@ HEAD_LOG_GAP = 18.0
|
||||
Z_TOLERANCE = 10.0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _murmur3_known_answer_kernel(
|
||||
seeds_ptr,
|
||||
positions_ptr,
|
||||
offsets_ptr,
|
||||
hashes_ptr,
|
||||
domain_hashes_ptr,
|
||||
numel: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
idx = tl.arange(0, BLOCK_SIZE)
|
||||
mask = idx < numel
|
||||
seed = tl.load(seeds_ptr + idx, mask=mask)
|
||||
pos = tl.load(positions_ptr + idx, mask=mask)
|
||||
offset = tl.load(offsets_ptr + idx, mask=mask)
|
||||
hash32 = murmur3_hash32(seed, pos, offset).to(tl.int64)
|
||||
domain_hash32 = murmur3_hash32(seed, pos, offset, domain=0x9E3779B9).to(tl.int64)
|
||||
tl.store(hashes_ptr + idx, hash32, mask=mask)
|
||||
tl.store(domain_hashes_ptr + idx, domain_hash32, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _uniform64_endpoint_kernel(
|
||||
random53_ptr,
|
||||
uniform_ptr,
|
||||
numel: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
idx = tl.arange(0, BLOCK_SIZE)
|
||||
mask = idx < numel
|
||||
random53 = tl.load(random53_ptr + idx, mask=mask).to(tl.uint64)
|
||||
tl.store(uniform_ptr + idx, _uniform64_from_random53(random53), mask=mask)
|
||||
|
||||
|
||||
def test_murmur3_known_answers():
|
||||
seeds = torch.tensor(
|
||||
[0, 1, -1, 0x123456789ABCDEF, -(1 << 63), (1 << 63) - 1],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
positions = torch.tensor(
|
||||
[0, 2, 17, 123456789, 1 << 30, (1 << 30) + 7],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
offsets = torch.tensor(
|
||||
[0, 3, 1023, 199999, 4096, 0xFFFFFFFF],
|
||||
dtype=torch.int64,
|
||||
device=DEVICE,
|
||||
)
|
||||
hashes = torch.empty_like(seeds)
|
||||
domain_hashes = torch.empty_like(seeds)
|
||||
|
||||
_murmur3_known_answer_kernel[(1,)](
|
||||
seeds,
|
||||
positions,
|
||||
offsets,
|
||||
hashes,
|
||||
domain_hashes,
|
||||
numel=seeds.numel(),
|
||||
BLOCK_SIZE=8,
|
||||
)
|
||||
|
||||
assert hashes.tolist() == [
|
||||
0x8134CDF8,
|
||||
0x23B53F91,
|
||||
0xC8C13C14,
|
||||
0x5DDE974F,
|
||||
0xF1A9355B,
|
||||
0xD952F9AA,
|
||||
]
|
||||
assert domain_hashes.tolist() == [
|
||||
0xD28AD962,
|
||||
0xE3AAF818,
|
||||
0xD223C51C,
|
||||
0x0547D2FE,
|
||||
0x3AED160F,
|
||||
0x49E8EA7F,
|
||||
]
|
||||
|
||||
|
||||
def test_uniform64_excludes_endpoints():
|
||||
random53 = torch.tensor([0, (1 << 53) - 1], dtype=torch.int64, device=DEVICE)
|
||||
uniform = torch.empty(2, dtype=torch.float64, device=DEVICE)
|
||||
|
||||
_uniform64_endpoint_kernel[(1,)](
|
||||
random53,
|
||||
uniform,
|
||||
numel=random53.numel(),
|
||||
BLOCK_SIZE=2,
|
||||
)
|
||||
|
||||
assert torch.isfinite(uniform).all()
|
||||
assert ((uniform > 0.0) & (uniform < 1.0)).all()
|
||||
assert uniform.tolist() == [2.0**-54, 1.0 - 2.0**-53]
|
||||
|
||||
|
||||
def _make_heavy_tailed_counts(seed: int = 1234) -> torch.Tensor:
|
||||
"""Non-negative int64 counts of shape [VOCAB_SIZE]; target prob = counts/N."""
|
||||
gen = torch.Generator(device=DEVICE).manual_seed(seed)
|
||||
@@ -180,7 +282,7 @@ def test_full_vocab_distribution_fidelity():
|
||||
|
||||
|
||||
def test_drafting_uses_a_separate_noise_stream():
|
||||
"""is_drafting salts the Philox offset: same inputs, different draws.
|
||||
"""is_drafting salts the RNG position: same inputs, different draws.
|
||||
|
||||
The draft proposal and the residual resample after a rejection must be
|
||||
independent. They key noise by the same (seed, pos), so only the salt keeps
|
||||
@@ -189,7 +291,7 @@ def test_drafting_uses_a_separate_noise_stream():
|
||||
tests/v1/spec_decode/test_rejection_sampler_utils.py for the distributional
|
||||
consequence.
|
||||
|
||||
Relocating the offset must not distort the draw either, so both streams are
|
||||
Salting the position must not distort the draw either, so both streams are
|
||||
checked against the target's far-tail mass, which sits HEAD_LOG_GAP logits
|
||||
below the head -- the regime where fp32 Gumbel precision matters.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user