[Bugfix][MRV2] Use int64 index mappings to fix row-offset overflow (#60736)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
vllmellm
Claude Opus 5.5
parent
51144dd83e
commit
ed7ae705a1
@@ -102,7 +102,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
# full buffer before a real batch has populated it, so zero would make
|
||||
# every padding row scatter into request slot 0.
|
||||
self.sample_idx_mapping = torch.full(
|
||||
(max_num_sampled_tokens,), -1, dtype=torch.int32, device=device
|
||||
(max_num_sampled_tokens,), -1, dtype=torch.int64, device=device
|
||||
)
|
||||
# [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into
|
||||
# draft_logits[req, step, :].
|
||||
|
||||
@@ -152,7 +152,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
device=device,
|
||||
)
|
||||
self.idx_mapping = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
self.max_num_reqs, dtype=torch.int64, device=device
|
||||
)
|
||||
self.temperature = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.float32, device=device
|
||||
|
||||
@@ -402,7 +402,7 @@ def _prepare_prefill_inputs_kernel(
|
||||
target_input_ids_ptr, # [src_tokens] int32
|
||||
target_positions_ptr, # [src_tokens] int64
|
||||
last_sampled_ptr, # [max_num_reqs] int32
|
||||
idx_mapping_ptr, # [num_reqs] int32
|
||||
idx_mapping_ptr, # [num_reqs] int64
|
||||
out_input_ids_ptr, # [src_tokens + num_reqs] int32
|
||||
out_positions_ptr, # [src_tokens + num_reqs] int64
|
||||
last_token_indices_ptr, # [max_num_reqs] int64
|
||||
|
||||
Reference in New Issue
Block a user