[BUG] Fix FP64 Gumbel precision coverage (#43150)
Signed-off-by: tianyu-z <zhangtianyupro@gmail.com> Signed-off-by: Tianyu Zhang <53099276+tianyu-z@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Nick Hill
OpenAI Codex
parent
8a83e6f2d7
commit
7fe7800fa4
@@ -544,13 +544,15 @@ def native_sample_recovered_tokens(
|
||||
target_probs: torch.Tensor, # [num_tokens, vocab_size]
|
||||
sampling_metadata: SamplingMetadata,
|
||||
device: torch.device,
|
||||
use_fp64_gumbel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
batch_size = len(num_draft_tokens)
|
||||
vocab_size = target_probs.shape[-1]
|
||||
q_dtype = torch.float64 if use_fp64_gumbel else torch.float32
|
||||
|
||||
q = torch.empty(
|
||||
(batch_size, vocab_size),
|
||||
dtype=torch.float32,
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
)
|
||||
q.exponential_()
|
||||
@@ -935,6 +937,67 @@ def test_sample_recovered_tokens(
|
||||
assert torch.equal(recovered_token_ids, ref_recovered_token_ids)
|
||||
|
||||
|
||||
def test_sample_recovered_tokens_uses_fp64_exponential_race_when_requested():
|
||||
batch_size = 2
|
||||
vocab_size = 64
|
||||
max_spec_len = 2
|
||||
num_tokens = batch_size * max_spec_len
|
||||
|
||||
draft_probs = torch.rand(
|
||||
num_tokens,
|
||||
vocab_size,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
draft_probs = F.softmax(draft_probs, dim=-1)
|
||||
target_probs = torch.rand(
|
||||
num_tokens,
|
||||
vocab_size,
|
||||
dtype=torch.float32,
|
||||
device=DEVICE_TYPE,
|
||||
)
|
||||
target_probs = F.softmax(target_probs, dim=-1)
|
||||
draft_token_ids = torch.multinomial(draft_probs, num_samples=1).to(torch.int32)
|
||||
|
||||
generators = {
|
||||
i: torch.Generator(device=DEVICE_TYPE).manual_seed(i) for i in range(batch_size)
|
||||
}
|
||||
sampling_metadata = create_sampling_metadata(
|
||||
all_greedy=False,
|
||||
temperature=torch.ones(batch_size, dtype=torch.float32, device=DEVICE_TYPE),
|
||||
generators=generators,
|
||||
)
|
||||
spec_decode_metadata = create_spec_decode_metadata(
|
||||
draft_token_ids.reshape(batch_size, max_spec_len).tolist(),
|
||||
target_probs.log(),
|
||||
)
|
||||
|
||||
expected = native_sample_recovered_tokens(
|
||||
max_spec_len,
|
||||
spec_decode_metadata.num_draft_tokens,
|
||||
spec_decode_metadata.cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
draft_probs,
|
||||
target_probs,
|
||||
sampling_metadata,
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
use_fp64_gumbel=True,
|
||||
)
|
||||
actual = sample_recovered_tokens(
|
||||
max_spec_len,
|
||||
spec_decode_metadata.num_draft_tokens,
|
||||
spec_decode_metadata.cu_num_draft_tokens,
|
||||
draft_token_ids,
|
||||
draft_probs,
|
||||
target_probs,
|
||||
sampling_metadata,
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
use_fp64_gumbel=True,
|
||||
)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
########################### Tests for Synthetic Rejection Sampling #########
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user