[Spec Decode] DSpark (#46995)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Giancarlo Delfin
mgoin
parent
63fcce4de1
commit
f5a8d73377
@@ -223,6 +223,11 @@ def gumbel_sample(
|
||||
output_processed_logits_col: torch.Tensor | None = None,
|
||||
use_fp64: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Enforce contiguity on non-strided input tensors
|
||||
expanded_idx_mapping = expanded_idx_mapping.contiguous()
|
||||
pos = pos.contiguous()
|
||||
if output_processed_logits_col is not None:
|
||||
output_processed_logits_col = output_processed_logits_col.contiguous()
|
||||
num_tokens, vocab_size = logits.shape
|
||||
BLOCK_SIZE = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
|
||||
|
||||
Reference in New Issue
Block a user