[BUG] Fix FP64 Gumbel precision coverage (#43150)
Signed-off-by: tianyu-z <zhangtianyupro@gmail.com> Signed-off-by: Tianyu Zhang <53099276+tianyu-z@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Nick Hill
OpenAI Codex
parent
8a83e6f2d7
commit
7fe7800fa4
@@ -65,6 +65,7 @@ class RejectionSampler(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.sampler = sampler
|
||||
self.use_fp64_gumbel = getattr(sampler, "use_fp64_gumbel", False)
|
||||
logprobs_mode = self.sampler.logprobs_mode
|
||||
self.is_processed_logprobs_mode = logprobs_mode.startswith("processed")
|
||||
self.is_logits_logprobs_mode = logprobs_mode.endswith("logits")
|
||||
@@ -176,6 +177,7 @@ class RejectionSampler(nn.Module):
|
||||
sampling_metadata,
|
||||
synthetic_mode=self.synthetic_mode,
|
||||
synthetic_conditional_rates=self.synthetic_conditional_rates,
|
||||
use_fp64_gumbel=self.use_fp64_gumbel,
|
||||
)
|
||||
|
||||
logprobs_tensors = None
|
||||
@@ -406,6 +408,7 @@ def rejection_sample(
|
||||
sampling_metadata: SamplingMetadata,
|
||||
synthetic_mode: bool = False,
|
||||
synthetic_conditional_rates: torch.Tensor | None = None,
|
||||
use_fp64_gumbel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
assert draft_token_ids.ndim == 1
|
||||
assert draft_probs is None or draft_probs.ndim == 2
|
||||
@@ -480,6 +483,7 @@ def rejection_sample(
|
||||
target_probs,
|
||||
sampling_metadata,
|
||||
device,
|
||||
use_fp64_gumbel,
|
||||
)
|
||||
|
||||
# Rejection sampling for random sampling requests.
|
||||
@@ -669,13 +673,15 @@ def sample_recovered_tokens(
|
||||
target_probs: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
device: torch.device,
|
||||
use_fp64_gumbel: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# NOTE(woosuk): Create only one distribution for each request.
|
||||
batch_size = len(num_draft_tokens)
|
||||
vocab_size = target_probs.shape[-1]
|
||||
q_dtype = torch.float64 if use_fp64_gumbel else torch.float32
|
||||
q = torch.empty(
|
||||
(batch_size, vocab_size),
|
||||
dtype=torch.float32,
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
)
|
||||
q.exponential_()
|
||||
@@ -699,6 +705,7 @@ def sample_recovered_tokens(
|
||||
vocab_size,
|
||||
BLOCK_SIZE,
|
||||
NO_DRAFT_PROBS=draft_probs is None,
|
||||
USE_FP64_GUMBEL=use_fp64_gumbel,
|
||||
)
|
||||
return recovered_token_ids
|
||||
|
||||
@@ -861,6 +868,7 @@ def sample_recovered_tokens_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
NO_DRAFT_PROBS: tl.constexpr,
|
||||
USE_FP64_GUMBEL: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
start_idx = 0 if req_idx == 0 else tl.load(cu_num_draft_tokens_ptr + req_idx - 1)
|
||||
@@ -877,7 +885,10 @@ def sample_recovered_tokens_kernel(
|
||||
if NO_DRAFT_PROBS:
|
||||
draft_token_id = tl.load(draft_token_ids_ptr + token_idx)
|
||||
|
||||
max_val = float("-inf")
|
||||
if USE_FP64_GUMBEL:
|
||||
max_val = tl.full((), float("-inf"), tl.float64)
|
||||
else:
|
||||
max_val = tl.full((), float("-inf"), tl.float32)
|
||||
recovered_id = 0
|
||||
for v in range(0, vocab_size, BLOCK_SIZE):
|
||||
vocab_offset = v + tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
Reference in New Issue
Block a user