[Bugfix][PP] Never drop a decoding request from the sampled-token broadcast (#54436)
Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -96,7 +96,6 @@ def _make_batch(
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
max_seq_len_np=None,
|
||||
input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE),
|
||||
positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE),
|
||||
is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE),
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Which rows the PP sampled-token broadcast must carry."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.worker.gpu import pp_utils
|
||||
|
||||
|
||||
def _batch(num_computed, prefill_len, num_scheduled):
|
||||
return Mock(
|
||||
num_reqs=len(num_computed),
|
||||
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
|
||||
prefill_len_np=np.array(prefill_len, dtype=np.int32),
|
||||
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_excludes_non_final_prefill_chunks():
|
||||
"""Unchanged behaviour: a chunk that does not finish its prefill is skipped."""
|
||||
# Row 0 is a middle prefill chunk and produces no sample; row 1 finishes its
|
||||
# prefill this step and therefore does.
|
||||
batch = _batch(
|
||||
num_computed=[512, 1000],
|
||||
prefill_len=[4096, 1004],
|
||||
num_scheduled=[448, 4],
|
||||
)
|
||||
|
||||
mask = pp_utils.compute_need_sampled_mask(batch)
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [False, True]
|
||||
|
||||
|
||||
def test_none_when_no_row_samples():
|
||||
"""Unchanged behaviour: an all-prefill batch needs no broadcast at all."""
|
||||
batch = _batch(
|
||||
num_computed=[0, 512],
|
||||
prefill_len=[4096, 4096],
|
||||
num_scheduled=[448, 448],
|
||||
)
|
||||
|
||||
assert pp_utils.compute_need_sampled_mask(batch) is None
|
||||
|
||||
|
||||
def test_keeps_decoding_request_past_its_length_cap():
|
||||
"""A decoding request must never be dropped from the broadcast.
|
||||
|
||||
Speculative decoding advances `num_computed_tokens` several tokens per step,
|
||||
so it can overrun `prompt_len + max_tokens` while the scheduler is still
|
||||
running the request. Predicting "this one is finishing" and skipping its
|
||||
broadcast freezes the earlier pipeline stages' `last_sampled_tokens` and
|
||||
`draft_tokens` while the last rank keeps advancing its own, and the stages
|
||||
then diverge permanently.
|
||||
"""
|
||||
batch = _batch(
|
||||
# 14176 computed tokens is already past this request's own
|
||||
# prompt_len + max_tokens; the scheduler is still running it.
|
||||
num_computed=[14176],
|
||||
prefill_len=[12175],
|
||||
num_scheduled=[8],
|
||||
)
|
||||
|
||||
mask = pp_utils.compute_need_sampled_mask(batch)
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [True]
|
||||
|
||||
|
||||
def test_decode_row_ahead_of_a_prefill_chunk():
|
||||
"""Row order does not matter: only whether the row finishes its prefill."""
|
||||
batch = _batch(
|
||||
num_computed=[10, 512],
|
||||
prefill_len=[8, 4096],
|
||||
num_scheduled=[1, 448],
|
||||
)
|
||||
|
||||
mask = pp_utils.compute_need_sampled_mask(batch)
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [True, False]
|
||||
@@ -85,9 +85,6 @@ class InputBatch:
|
||||
# == np.any(is_prefilling_np)
|
||||
has_prefill: bool
|
||||
|
||||
# [num_reqs] only populated when pipeline parallelism is enabled.
|
||||
max_seq_len_np: np.ndarray | None
|
||||
|
||||
# [num_tokens_after_padding]
|
||||
input_ids: torch.Tensor
|
||||
# [num_tokens_after_padding]
|
||||
@@ -194,7 +191,6 @@ class InputBatch:
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
has_prefill=False,
|
||||
max_seq_len_np=None,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
is_padding=is_padding,
|
||||
|
||||
@@ -1311,11 +1311,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
|
||||
|
||||
max_seq_len_np = None
|
||||
if self.use_pp:
|
||||
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
|
||||
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
|
||||
|
||||
prompt_lens = None
|
||||
if self.model_config.rswa_window is not None:
|
||||
# prompt_lens is only used in R-SWA case.
|
||||
@@ -1344,7 +1339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=batch_req_state.is_prefilling_np,
|
||||
has_prefill=batch_req_state.has_prefill,
|
||||
max_seq_len_np=max_seq_len_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
is_padding=self.input_buffers.is_padding[:num_tokens_after_padding],
|
||||
|
||||
@@ -575,9 +575,6 @@ class PCPManager:
|
||||
num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=local_is_prefilling_np,
|
||||
has_prefill=bool(local_is_prefilling_np.any()),
|
||||
max_seq_len_np=global_batch.max_seq_len_np[local_to_global_batch_req_idx_np]
|
||||
if global_batch.max_seq_len_np is not None
|
||||
else None,
|
||||
input_ids=input_buffers.input_ids[:num_local_tokens_padded],
|
||||
positions=input_buffers.positions[:num_local_tokens_padded],
|
||||
is_padding=is_padding,
|
||||
|
||||
@@ -34,19 +34,15 @@ class PendingRecv:
|
||||
|
||||
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
|
||||
with outputs that might be needed in a subsequent (decode) step.
|
||||
Returns None if no sampled outputs are needed in the requests' next step."""
|
||||
that produce a sampled token this step, and therefore must have that token
|
||||
(and the draft block proposed from it) propagated to the earlier PP stages.
|
||||
Returns None if no request in the batch produces a sample."""
|
||||
|
||||
old_computed = input_batch.num_computed_tokens_np
|
||||
prefill_len = input_batch.prefill_len_np
|
||||
max_seq_len = input_batch.max_seq_len_np
|
||||
assert max_seq_len is not None # always populated under PP
|
||||
# Exclude non-final prefill chunks (they don't produce a sample).
|
||||
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
|
||||
# Exclude requests that we know are finished.
|
||||
not_finishing = np.maximum(old_computed, prefill_len) + 1 < max_seq_len
|
||||
need_sampled_mask = produces_sample & not_finishing
|
||||
return need_sampled_mask if need_sampled_mask.any() else None
|
||||
return produces_sample if produces_sample.any() else None
|
||||
|
||||
|
||||
class PPHandler:
|
||||
|
||||
Reference in New Issue
Block a user