[Bugfix][Model Runner V2][Spec Decode] Fix int32 offset overflow in sampler kernels (#46560)

Signed-off-by: xiaojun.wei <jessiewei747@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
JessieWei
2026-06-24 11:00:57 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 Robert Shaw
parent 56ca5997ea
commit 3c43237233
10 changed files with 90 additions and 16 deletions
+1 -1
View File
@@ -114,7 +114,7 @@ def _bad_words_kernel(
input_ids_ptr,
expanded_local_pos_ptr,
):
token_idx = tl.program_id(0)
token_idx = tl.program_id(0).to(tl.int64)
bw_idx = tl.program_id(1)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
+3 -3
View File
@@ -23,7 +23,7 @@ def _temperature_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
token_idx = tl.program_id(0).to(tl.int64)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
temperature = tl.load(temperature_ptr + req_state_idx).to(tl.float32)
if temperature == 0.0 or temperature == 1.0:
@@ -91,7 +91,7 @@ def gumbel_block_argmax(
USE_FP64: tl.constexpr,
PER_TOKEN_COL: tl.constexpr = False,
):
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx).to(tl.int64)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
if temp != 0.0 and APPLY_TEMPERATURE:
# Apply temperature.
@@ -169,7 +169,7 @@ def _gumbel_sample_kernel(
USE_FP64: tl.constexpr,
PER_TOKEN_COL: tl.constexpr,
):
token_idx = tl.program_id(0)
token_idx = tl.program_id(0).to(tl.int64)
block_idx = tl.program_id(1)
block = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = block < vocab_size
+1 -1
View File
@@ -169,7 +169,7 @@ def _bias_kernel(
BLOCK_SIZE: tl.constexpr,
LOGITS_BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
token_idx = tl.program_id(0).to(tl.int64)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
block = tl.arange(0, BLOCK_SIZE)
+2 -2
View File
@@ -21,7 +21,7 @@ def _topk_log_softmax_kernel(
BLOCK_SIZE: tl.constexpr,
PADDED_TOPK: tl.constexpr,
):
req_idx = tl.program_id(0)
req_idx = tl.program_id(0).to(tl.int64)
row_ptr = logits_ptr + req_idx * logits_stride
max_val = float("-inf")
@@ -61,7 +61,7 @@ def _ranks_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
):
req_idx = tl.program_id(0)
req_idx = tl.program_id(0).to(tl.int64)
row_ptr = logits_ptr + req_idx * logits_stride
token_id = tl.load(token_ids_ptr + req_idx)
+1 -1
View File
@@ -14,7 +14,7 @@ def _min_p_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
token_idx = tl.program_id(0).to(tl.int64)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
min_p = tl.load(min_p_ptr + req_state_idx).to(tl.float32)
if min_p == 0.0:
+1 -1
View File
@@ -120,7 +120,7 @@ def _penalties_kernel(
vocab_size,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
token_idx = tl.program_id(0).to(tl.int64)
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
rep_penalty = tl.load(repetition_penalty_ptr + req_state_idx)
freq_penalty = tl.load(frequency_penalty_ptr + req_state_idx)