[CPU][Bugfix][Speculative Decoding] Accept USE_FP64_GUMBEL in CPU recovered-tokens sampler (#46069)
Signed-off-by: hillel.darshan <hillel.darshan@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -300,6 +300,7 @@ def _sample_recovered_tokens_kernel_impl(
|
||||
vocab_size,
|
||||
BLOCK_SIZE=None,
|
||||
NO_DRAFT_PROBS=False,
|
||||
USE_FP64_GUMBEL=False,
|
||||
):
|
||||
# C++ reads integer tensors as int64_t*; ensure correct dtype.
|
||||
orig_dtype = output_token_ids.dtype
|
||||
@@ -310,7 +311,8 @@ def _sample_recovered_tokens_kernel_impl(
|
||||
_ensure_int64(draft_token_ids),
|
||||
draft_probs,
|
||||
target_probs,
|
||||
inv_q,
|
||||
# C++ kernel reads inv_q as float32.
|
||||
inv_q.to(torch.float32),
|
||||
vocab_size,
|
||||
NO_DRAFT_PROBS,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user