[Bugfix][Model Runner V2][Spec Decode] Fix int32 offset overflow in sampler kernels (#46560)
Signed-off-by: xiaojun.wei <jessiewei747@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Robert Shaw
parent
56ca5997ea
commit
3c43237233
@@ -114,7 +114,7 @@ def _bad_words_kernel(
|
||||
input_ids_ptr,
|
||||
expanded_local_pos_ptr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
bw_idx = tl.program_id(1)
|
||||
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
|
||||
@@ -23,7 +23,7 @@ def _temperature_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
temperature = tl.load(temperature_ptr + req_state_idx).to(tl.float32)
|
||||
if temperature == 0.0 or temperature == 1.0:
|
||||
@@ -91,7 +91,7 @@ def gumbel_block_argmax(
|
||||
USE_FP64: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr = False,
|
||||
):
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx).to(tl.int64)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Apply temperature.
|
||||
@@ -169,7 +169,7 @@ def _gumbel_sample_kernel(
|
||||
USE_FP64: tl.constexpr,
|
||||
PER_TOKEN_COL: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
block_idx = tl.program_id(1)
|
||||
block = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < vocab_size
|
||||
|
||||
@@ -169,7 +169,7 @@ def _bias_kernel(
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LOGITS_BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
@@ -21,7 +21,7 @@ def _topk_log_softmax_kernel(
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PADDED_TOPK: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
req_idx = tl.program_id(0).to(tl.int64)
|
||||
row_ptr = logits_ptr + req_idx * logits_stride
|
||||
|
||||
max_val = float("-inf")
|
||||
@@ -61,7 +61,7 @@ def _ranks_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
req_idx = tl.program_id(0)
|
||||
req_idx = tl.program_id(0).to(tl.int64)
|
||||
row_ptr = logits_ptr + req_idx * logits_stride
|
||||
|
||||
token_id = tl.load(token_ids_ptr + req_idx)
|
||||
|
||||
@@ -14,7 +14,7 @@ def _min_p_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
min_p = tl.load(min_p_ptr + req_state_idx).to(tl.float32)
|
||||
if min_p == 0.0:
|
||||
|
||||
@@ -120,7 +120,7 @@ def _penalties_kernel(
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx)
|
||||
rep_penalty = tl.load(repetition_penalty_ptr + req_state_idx)
|
||||
freq_penalty = tl.load(frequency_penalty_ptr + req_state_idx)
|
||||
|
||||
Reference in New Issue
Block a user