[Bugfix][MRV2] Use int64 index mappings to fix row-offset overflow (#60736)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
vllmellm
Claude Opus 5.5
parent
51144dd83e
commit
ed7ae705a1
@@ -214,21 +214,15 @@ class BatchSharder:
|
||||
local_logits_indices = torch.empty(
|
||||
num_local_logits, dtype=torch.int64, device=self.device
|
||||
)
|
||||
local_idx_mapping = torch.empty(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
local_idx_mapping = input_batch.idx_mapping.new_empty(num_local_reqs)
|
||||
local_cu_num_logits = input_batch.cu_num_logits.new_empty(num_local_reqs + 1)
|
||||
local_expanded_idx_mapping = input_batch.expanded_idx_mapping.new_empty(
|
||||
num_local_logits
|
||||
)
|
||||
local_cu_num_logits = torch.empty(
|
||||
num_local_reqs + 1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_expanded_idx_mapping = torch.empty(
|
||||
num_local_logits, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_expanded_local_pos = torch.empty(
|
||||
num_local_logits, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_seq_lens = torch.empty(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
local_expanded_local_pos = input_batch.expanded_local_pos.new_empty(
|
||||
num_local_logits
|
||||
)
|
||||
local_seq_lens = input_batch.seq_lens.new_empty(num_local_reqs)
|
||||
if num_reqs > 0:
|
||||
_build_shard_plan_kernel[(num_reqs,)](
|
||||
input_batch.idx_mapping,
|
||||
|
||||
@@ -62,7 +62,7 @@ class PenaltiesState(LogitsProcessor):
|
||||
def apply_staged_writes(self) -> None:
|
||||
if self._new_penalties_reqs:
|
||||
idx_mapping = async_tensor_h2d(
|
||||
self._new_penalties_reqs, dtype=torch.int32, device=self.device
|
||||
self._new_penalties_reqs, dtype=torch.int64, device=self.device
|
||||
)
|
||||
|
||||
prefill_lens = self.req_states.prefill_len.np[self._new_penalties_reqs]
|
||||
|
||||
Reference in New Issue
Block a user