[Bugfix][PP] Never drop a decoding request from the sampled-token broadcast (#54436)

Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Ausar
2026-08-31 23:49:54 +00:00
committed by GitHub
co-authored by Claude
parent b05acd2ae0
commit 6bafc049aa
6 changed files with 87 additions and 22 deletions
-1
View File
@@ -96,7 +96,6 @@ def _make_batch(
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE),
positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE),
is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE),
+83
View File
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Which rows the PP sampled-token broadcast must carry."""
from unittest.mock import Mock
import numpy as np
from vllm.v1.worker.gpu import pp_utils
def _batch(num_computed, prefill_len, num_scheduled):
return Mock(
num_reqs=len(num_computed),
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
prefill_len_np=np.array(prefill_len, dtype=np.int32),
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
)
def test_excludes_non_final_prefill_chunks():
"""Unchanged behaviour: a chunk that does not finish its prefill is skipped."""
# Row 0 is a middle prefill chunk and produces no sample; row 1 finishes its
# prefill this step and therefore does.
batch = _batch(
num_computed=[512, 1000],
prefill_len=[4096, 1004],
num_scheduled=[448, 4],
)
mask = pp_utils.compute_need_sampled_mask(batch)
assert mask is not None
assert mask.tolist() == [False, True]
def test_none_when_no_row_samples():
"""Unchanged behaviour: an all-prefill batch needs no broadcast at all."""
batch = _batch(
num_computed=[0, 512],
prefill_len=[4096, 4096],
num_scheduled=[448, 448],
)
assert pp_utils.compute_need_sampled_mask(batch) is None
def test_keeps_decoding_request_past_its_length_cap():
"""A decoding request must never be dropped from the broadcast.
Speculative decoding advances `num_computed_tokens` several tokens per step,
so it can overrun `prompt_len + max_tokens` while the scheduler is still
running the request. Predicting "this one is finishing" and skipping its
broadcast freezes the earlier pipeline stages' `last_sampled_tokens` and
`draft_tokens` while the last rank keeps advancing its own, and the stages
then diverge permanently.
"""
batch = _batch(
# 14176 computed tokens is already past this request's own
# prompt_len + max_tokens; the scheduler is still running it.
num_computed=[14176],
prefill_len=[12175],
num_scheduled=[8],
)
mask = pp_utils.compute_need_sampled_mask(batch)
assert mask is not None
assert mask.tolist() == [True]
def test_decode_row_ahead_of_a_prefill_chunk():
"""Row order does not matter: only whether the row finishes its prefill."""
batch = _batch(
num_computed=[10, 512],
prefill_len=[8, 4096],
num_scheduled=[1, 448],
)
mask = pp_utils.compute_need_sampled_mask(batch)
assert mask is not None
assert mask.tolist() == [True, False]
-4
View File
@@ -85,9 +85,6 @@ class InputBatch:
# == np.any(is_prefilling_np)
has_prefill: bool
# [num_reqs] only populated when pipeline parallelism is enabled.
max_seq_len_np: np.ndarray | None
# [num_tokens_after_padding]
input_ids: torch.Tensor
# [num_tokens_after_padding]
@@ -194,7 +191,6 @@ class InputBatch:
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
has_prefill=False,
max_seq_len_np=None,
input_ids=input_ids,
positions=positions,
is_padding=is_padding,
-6
View File
@@ -1311,11 +1311,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
max_seq_len_np = None
if self.use_pp:
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
prompt_lens = None
if self.model_config.rswa_window is not None:
# prompt_lens is only used in R-SWA case.
@@ -1344,7 +1339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
is_prefilling_np=batch_req_state.is_prefilling_np,
has_prefill=batch_req_state.has_prefill,
max_seq_len_np=max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
is_padding=self.input_buffers.is_padding[:num_tokens_after_padding],
-3
View File
@@ -575,9 +575,6 @@ class PCPManager:
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
is_prefilling_np=local_is_prefilling_np,
has_prefill=bool(local_is_prefilling_np.any()),
max_seq_len_np=global_batch.max_seq_len_np[local_to_global_batch_req_idx_np]
if global_batch.max_seq_len_np is not None
else None,
input_ids=input_buffers.input_ids[:num_local_tokens_padded],
positions=input_buffers.positions[:num_local_tokens_padded],
is_padding=is_padding,
+4 -8
View File
@@ -34,19 +34,15 @@ class PendingRecv:
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
with outputs that might be needed in a subsequent (decode) step.
Returns None if no sampled outputs are needed in the requests' next step."""
that produce a sampled token this step, and therefore must have that token
(and the draft block proposed from it) propagated to the earlier PP stages.
Returns None if no request in the batch produces a sample."""
old_computed = input_batch.num_computed_tokens_np
prefill_len = input_batch.prefill_len_np
max_seq_len = input_batch.max_seq_len_np
assert max_seq_len is not None # always populated under PP
# Exclude non-final prefill chunks (they don't produce a sample).
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
# Exclude requests that we know are finished.
not_finishing = np.maximum(old_computed, prefill_len) + 1 < max_seq_len
need_sampled_mask = produces_sample & not_finishing
return need_sampled_mask if need_sampled_mask.any() else None
return produces_sample if produces_sample.any() else None
class PPHandler: