[Model Runner V2][Spec Decode] Fix stale values in idx_mapping from CG num reqs padding (#47066)
This commit is contained in:
@@ -100,7 +100,10 @@ def gumbel_block_argmax(
|
||||
PER_TOKEN_COL: tl.constexpr = False,
|
||||
):
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + token_idx).to(tl.int64)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
is_valid_req = req_state_idx >= 0
|
||||
temp = tl.load(temp_ptr + req_state_idx, mask=is_valid_req, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Apply temperature.
|
||||
# NOTE(woosuk): Match the behavior of _temperature_kernel.
|
||||
@@ -122,7 +125,7 @@ def gumbel_block_argmax(
|
||||
+ col * vocab_size
|
||||
+ block,
|
||||
logits,
|
||||
mask=mask,
|
||||
mask=mask & is_valid_req,
|
||||
)
|
||||
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
|
||||
@@ -131,7 +134,7 @@ def gumbel_block_argmax(
|
||||
logits = logits.to(tl.float64)
|
||||
if temp != 0.0:
|
||||
# Calculate the seed for gumbel noise.
|
||||
seed = tl.load(seeds_ptr + req_state_idx)
|
||||
seed = tl.load(seeds_ptr + req_state_idx, mask=is_valid_req, other=0)
|
||||
pos = tl.load(pos_ptr + token_idx)
|
||||
gumbel_seed = tl.randint(seed, pos)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user