[Bugfix] Route every speculation-capable row through the speculative path so recurrent state stays consistent (#56531)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor Agent
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4a705c7b54
commit
0eac152707
@@ -840,6 +840,12 @@ class GPUModelRunner(
|
||||
self.num_accepted_tokens = self._make_buffer(
|
||||
self.max_num_reqs, dtype=torch.int32
|
||||
)
|
||||
# The count includes the bonus token, so one is the minimum, and the
|
||||
# recurrent kernels pick their state slot as `accepted - 1`. Dummy runs
|
||||
# skip `_prepare_inputs`, so the zeros from the allocation would reach
|
||||
# those kernels as index -1 and fault on the out-of-bounds read.
|
||||
self.num_accepted_tokens.np.fill(1)
|
||||
self.num_accepted_tokens.copy_to_gpu()
|
||||
|
||||
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
|
||||
if self.uses_mrope:
|
||||
@@ -2199,7 +2205,7 @@ class GPUModelRunner(
|
||||
)
|
||||
self.mrope_positions.gpu[:, :total_num_scheduled_tokens] += drift
|
||||
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
use_spec_decode = self.speculative_config is not None
|
||||
if not use_spec_decode:
|
||||
# NOTE(woosuk): Due to chunked prefills, the batch may contain
|
||||
# partial requests. While we should not sample any token
|
||||
@@ -2226,6 +2232,18 @@ class GPUModelRunner(
|
||||
num_draft_tokens[req_idx] = draft_len
|
||||
if num_scheduled_tokens[req_idx] == draft_len + 1:
|
||||
num_decode_draft_tokens[req_idx] = draft_len
|
||||
# Some recurrent backends may read the previous step's state at an
|
||||
# offset that only the speculative path applies. So a decode step
|
||||
# that got no drafts must still count as a speculative row with
|
||||
# zero drafts, not as a plain decode. Prompt chunks are left out:
|
||||
# their tokens are prompt tokens and carry no such offset.
|
||||
zero_draft_decode_mask = (
|
||||
self.input_batch.num_computed_tokens_cpu[:num_reqs]
|
||||
>= self.input_batch.num_prompt_tokens[:num_reqs]
|
||||
) & (num_scheduled_tokens[:num_reqs] == 1)
|
||||
num_decode_draft_tokens[
|
||||
(num_decode_draft_tokens < 0) & zero_draft_decode_mask
|
||||
] = 0
|
||||
spec_decode_metadata = self._calc_spec_decode_metadata(
|
||||
num_draft_tokens, cu_num_tokens
|
||||
)
|
||||
@@ -4312,7 +4330,7 @@ class GPUModelRunner(
|
||||
self.mamba_state_idx,
|
||||
)
|
||||
|
||||
use_spec_decode = len(scheduler_output.scheduled_spec_decode_tokens) > 0
|
||||
use_spec_decode = self.speculative_config is not None
|
||||
ubatch_slices_attn = ubatch_slices_padded if pad_attn else ubatch_slices
|
||||
|
||||
slot_mappings_by_group, slot_mappings = self._get_slot_mappings(
|
||||
|
||||
Reference in New Issue
Block a user