[CPU][Bugfix][Speculative Decoding] Accept USE_FP64_GUMBEL in CPU recovered-tokens sampler (#46069)
Signed-off-by: hillel.darshan <hillel.darshan@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -300,6 +300,7 @@ def _sample_recovered_tokens_kernel_impl(
|
|||||||
vocab_size,
|
vocab_size,
|
||||||
BLOCK_SIZE=None,
|
BLOCK_SIZE=None,
|
||||||
NO_DRAFT_PROBS=False,
|
NO_DRAFT_PROBS=False,
|
||||||
|
USE_FP64_GUMBEL=False,
|
||||||
):
|
):
|
||||||
# C++ reads integer tensors as int64_t*; ensure correct dtype.
|
# C++ reads integer tensors as int64_t*; ensure correct dtype.
|
||||||
orig_dtype = output_token_ids.dtype
|
orig_dtype = output_token_ids.dtype
|
||||||
@@ -310,7 +311,8 @@ def _sample_recovered_tokens_kernel_impl(
|
|||||||
_ensure_int64(draft_token_ids),
|
_ensure_int64(draft_token_ids),
|
||||||
draft_probs,
|
draft_probs,
|
||||||
target_probs,
|
target_probs,
|
||||||
inv_q,
|
# C++ kernel reads inv_q as float32.
|
||||||
|
inv_q.to(torch.float32),
|
||||||
vocab_size,
|
vocab_size,
|
||||||
NO_DRAFT_PROBS,
|
NO_DRAFT_PROBS,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user