[Bugfix][MRV2] Use int64 index mappings to fix row-offset overflow (#60736)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
vllmellm
Claude Opus 5.5
parent
51144dd83e
commit
ed7ae705a1
@@ -310,7 +310,7 @@ def _apply_write_kernel(
|
||||
MULTI_GROUP: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
row_idx = tl.load(write_indices_ptr + pid)
|
||||
row_idx = tl.load(write_indices_ptr + pid).to(tl.int64)
|
||||
start_idx = tl.load(write_starts_ptr + pid)
|
||||
|
||||
cu_start = tl.load(write_cu_lens_ptr + pid - 1) if pid > 0 else 0
|
||||
|
||||
@@ -137,7 +137,7 @@ class InputBatch:
|
||||
|
||||
req_ids = [f"req_{i}_{random_uuid()}" for i in range(num_reqs)]
|
||||
idx_mapping_np = np.arange(num_reqs, dtype=np.intp)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int64, device=device)
|
||||
expanded_idx_mapping = idx_mapping
|
||||
expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -715,7 +715,9 @@ def expand_idx_mapping(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
expanded_idx_mapping = idx_mapping.new_empty(total_num_logits)
|
||||
expanded_local_pos = idx_mapping.new_empty(total_num_logits)
|
||||
expanded_local_pos = torch.empty(
|
||||
total_num_logits, dtype=torch.int32, device=idx_mapping.device
|
||||
)
|
||||
_expand_idx_mapping_kernel[(num_reqs,)](
|
||||
idx_mapping,
|
||||
expanded_idx_mapping,
|
||||
|
||||
@@ -1361,7 +1361,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_scheduled_tokens_np = batch_req_state.num_scheduled_tokens
|
||||
idx_mapping_np = batch_req_state.idx_mapping_np
|
||||
idx_mapping = async_tensor_h2d(
|
||||
idx_mapping_np, device=self.device, dtype=torch.int32
|
||||
idx_mapping_np, device=self.device, dtype=torch.int64
|
||||
)
|
||||
num_reqs = len(req_ids)
|
||||
|
||||
|
||||
@@ -653,9 +653,7 @@ class PCPManager:
|
||||
idx_mapping=local_to_global_req_idx,
|
||||
idx_mapping_np=local_to_global_req_idx_np,
|
||||
expanded_idx_mapping=local_to_global_req_idx,
|
||||
expanded_local_pos=torch.zeros(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
),
|
||||
expanded_local_pos=input_batch.expanded_local_pos.new_zeros(num_local_reqs),
|
||||
num_scheduled_tokens=local_num_scheduled_tokens,
|
||||
num_tokens=num_local_tokens,
|
||||
num_tokens_after_padding=num_local_tokens_padded,
|
||||
|
||||
@@ -214,21 +214,15 @@ class BatchSharder:
|
||||
local_logits_indices = torch.empty(
|
||||
num_local_logits, dtype=torch.int64, device=self.device
|
||||
)
|
||||
local_idx_mapping = torch.empty(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
local_idx_mapping = input_batch.idx_mapping.new_empty(num_local_reqs)
|
||||
local_cu_num_logits = input_batch.cu_num_logits.new_empty(num_local_reqs + 1)
|
||||
local_expanded_idx_mapping = input_batch.expanded_idx_mapping.new_empty(
|
||||
num_local_logits
|
||||
)
|
||||
local_cu_num_logits = torch.empty(
|
||||
num_local_reqs + 1, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_expanded_idx_mapping = torch.empty(
|
||||
num_local_logits, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_expanded_local_pos = torch.empty(
|
||||
num_local_logits, dtype=torch.int32, device=self.device
|
||||
)
|
||||
local_seq_lens = torch.empty(
|
||||
num_local_reqs, dtype=torch.int32, device=self.device
|
||||
local_expanded_local_pos = input_batch.expanded_local_pos.new_empty(
|
||||
num_local_logits
|
||||
)
|
||||
local_seq_lens = input_batch.seq_lens.new_empty(num_local_reqs)
|
||||
if num_reqs > 0:
|
||||
_build_shard_plan_kernel[(num_reqs,)](
|
||||
input_batch.idx_mapping,
|
||||
|
||||
@@ -62,7 +62,7 @@ class PenaltiesState(LogitsProcessor):
|
||||
def apply_staged_writes(self) -> None:
|
||||
if self._new_penalties_reqs:
|
||||
idx_mapping = async_tensor_h2d(
|
||||
self._new_penalties_reqs, dtype=torch.int32, device=self.device
|
||||
self._new_penalties_reqs, dtype=torch.int64, device=self.device
|
||||
)
|
||||
|
||||
prefill_lens = self.req_states.prefill_len.np[self._new_penalties_reqs]
|
||||
|
||||
@@ -102,7 +102,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
# full buffer before a real batch has populated it, so zero would make
|
||||
# every padding row scatter into request slot 0.
|
||||
self.sample_idx_mapping = torch.full(
|
||||
(max_num_sampled_tokens,), -1, dtype=torch.int32, device=device
|
||||
(max_num_sampled_tokens,), -1, dtype=torch.int64, device=device
|
||||
)
|
||||
# [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into
|
||||
# draft_logits[req, step, :].
|
||||
|
||||
@@ -152,7 +152,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
device=device,
|
||||
)
|
||||
self.idx_mapping = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.int32, device=device
|
||||
self.max_num_reqs, dtype=torch.int64, device=device
|
||||
)
|
||||
self.temperature = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.float32, device=device
|
||||
|
||||
@@ -402,7 +402,7 @@ def _prepare_prefill_inputs_kernel(
|
||||
target_input_ids_ptr, # [src_tokens] int32
|
||||
target_positions_ptr, # [src_tokens] int64
|
||||
last_sampled_ptr, # [max_num_reqs] int32
|
||||
idx_mapping_ptr, # [num_reqs] int32
|
||||
idx_mapping_ptr, # [num_reqs] int64
|
||||
out_input_ids_ptr, # [src_tokens + num_reqs] int32
|
||||
out_positions_ptr, # [src_tokens + num_reqs] int64
|
||||
last_token_indices_ptr, # [max_num_reqs] int64
|
||||
|
||||
Reference in New Issue
Block a user