[BUG] Fix FP64 Gumbel precision coverage (#43150)
Signed-off-by: tianyu-z <zhangtianyupro@gmail.com> Signed-off-by: Tianyu Zhang <53099276+tianyu-z@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Nick Hill
OpenAI Codex
parent
8a83e6f2d7
commit
7fe7800fa4
@@ -544,13 +544,15 @@ def native_sample_recovered_tokens(
|
||||
target_probs: torch.Tensor, # [num_tokens, vocab_size]
|
||||
sampling_metadata: SamplingMetadata,
|
||||
device: torch.device,
|
||||
use_fp64_gumbel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
batch_size = len(num_draft_tokens)
|
||||
vocab_size = target_probs.shape[-1]
|
||||
q_dtype = torch.float64 if use_fp64_gumbel else torch.float32
|
||||
|
||||
q = torch.empty(
|
||||
(batch_size, vocab_size),
|
||||
dtype=torch.float32,
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
)
|
||||
q.exponential_()
|
||||
@@ -935,6 +937,67 @@ def test_sample_recovered_tokens(
|
||||
assert torch.equal(recovered_token_ids, ref_recovered_token_ids)
|
||||
|
||||
|
||||
def test_sample_recovered_tokens_uses_fp64_exponential_race_when_requested():
|
||||
batch_size = 2
|
||||
vocab_size = 64
|
||||
max_spec_len = 2
|
||||
num_tokens = batch_size * max_spec_len
|
||||
|
||||
draft_probs = torch.rand(
|
||||
num_tokens,
|
||||
vocab_size,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
draft_probs = F.softmax(draft_probs, dim=-1)
|
||||
target_probs = torch.rand(
|
||||
num_tokens,
|
||||
vocab_size,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
target_probs = F.softmax(target_probs, dim=-1)
|
||||
draft_token_ids = torch.multinomial(draft_probs, num_samples=1).to(torch.int32)
|
||||
|
||||
generators = {
|
||||
i: torch.Generator(device=DEVICE_TYPE).manual_seed(i) for i in range(batch_size)
|
||||
}
|
||||
sampling_metadata = create_sampling_metadata(
|
||||
all_greedy=False,
|
||||
temperature=torch.ones(batch_size, dtype=torch.float32, device=DEVICE_TYPE),
|
||||
generators=generators,
|
||||
)
|
||||
spec_decode_metadata = create_spec_decode_metadata(
|
||||
draft_token_ids.reshape(batch_size, max_spec_len).tolist(),
|
||||
target_probs.log(),
|
||||
)
|
||||
|
||||
expected = native_sample_recovered_tokens(
|
||||
max_spec_len,
|
||||
spec_decode_metadata.num_draft_tokens,
|
||||
spec_decode_metadata.cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
draft_probs,
|
||||
target_probs,
|
||||
sampling_metadata,
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
use_fp64_gumbel=True,
|
||||
)
|
||||
actual = sample_recovered_tokens(
|
||||
max_spec_len,
|
||||
spec_decode_metadata.num_draft_tokens,
|
||||
spec_decode_metadata.cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
draft_probs,
|
||||
target_probs,
|
||||
sampling_metadata,
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
use_fp64_gumbel=True,
|
||||
)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
########################### Tests for Synthetic Rejection Sampling #########
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,12 @@ from torch import Generator
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p_pytorch
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
apply_top_k_top_p_pytorch,
|
||||
random_sample,
|
||||
)
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
@@ -38,6 +43,10 @@ def _flashinfer_topk_topp_supported() -> bool:
|
||||
FLASHINFER_TOPK_TOPP_SUPPORTED = _flashinfer_topk_topp_supported()
|
||||
|
||||
|
||||
def _seed_default_generator(seed: int) -> None:
|
||||
set_random_seed(seed)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_default_device():
|
||||
"""
|
||||
@@ -49,6 +58,35 @@ def reset_default_device():
|
||||
torch.set_default_device(original_device)
|
||||
|
||||
|
||||
def test_sampler_threads_fp64_gumbel_to_topk_topp_sampler():
|
||||
sampler = Sampler(use_fp64_gumbel=True)
|
||||
|
||||
assert sampler.topk_topp_sampler.use_fp64_gumbel
|
||||
|
||||
|
||||
def test_random_sample_uses_fp64_exponential_race_when_requested():
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
probs = torch.tensor(
|
||||
[
|
||||
[0.70, 0.20, 0.10],
|
||||
[0.05, 0.15, 0.80],
|
||||
[0.25, 0.25, 0.50],
|
||||
],
|
||||
dtype=torch.float32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
|
||||
_seed_default_generator(12345)
|
||||
q = torch.empty(probs.shape, dtype=torch.float64, device=probs.device)
|
||||
q.exponential_()
|
||||
expected = q.reciprocal_().mul_(probs).argmax(dim=-1).view(-1)
|
||||
|
||||
_seed_default_generator(12345)
|
||||
actual = random_sample(probs.clone(), {}, use_fp64_gumbel=True)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
def test_topk_impl_equivalence():
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
generator = Generator(device=DEVICE_TYPE).manual_seed(33)
|
||||
|
||||
Reference in New Issue
Block a user