[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)

Signed-off-by: LostFox11 <wangziyue17@huawei.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: LostFox11 <wangziyue17@huawei.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
LostFox11
2026-09-30 19:18:55 -07:00
committed by GitHub
co-authored by LostFox11 Nick Hill
parent 34c458d27f
commit 7314360c9e
7 changed files with 111 additions and 6 deletions
@@ -57,6 +57,7 @@ def _make_runner(
runner.decode_query_len = decode_query_len
runner.adaptive_verification = None
runner.pcp_manager = None
runner.use_pp = False
runner.req_states = SimpleNamespace(
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
# The runner keeps this as min(num_computed_tokens, prefill_len).
@@ -246,6 +247,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
runner.decode_query_len = 8
runner.adaptive_verification = None
runner.pcp_manager = None
runner.use_pp = False
# State arrays in state-index order: a prefilling request, then two decodes.
runner.req_states = SimpleNamespace(
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
+86 -1
View File
@@ -9,12 +9,26 @@ import numpy as np
from vllm.v1.worker.gpu import pp_utils
def _batch(num_computed, prefill_len, num_scheduled):
def _batch(
num_computed,
prefill_len,
num_scheduled,
is_prefilling=None,
max_seq_len=None,
):
return Mock(
num_reqs=len(num_computed),
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
prefill_len_np=np.array(prefill_len, dtype=np.int32),
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
is_prefilling_np=(
np.array(is_prefilling, dtype=np.bool_)
if is_prefilling is not None
else None
),
max_seq_len_np=(
np.array(max_seq_len, dtype=np.int32) if max_seq_len is not None else None
),
)
@@ -81,3 +95,74 @@ def test_decode_row_ahead_of_a_prefill_chunk():
assert mask is not None
assert mask.tolist() == [True, False]
def test_excludes_final_prefill_chunk_reaching_its_length_cap():
"""A max_tokens=1 final chunk hands its token off and leaves the engine.
Disaggregated prefill submits max_tokens=1, so the request finishes with
the final chunk's single sample and no follow-up step on this engine
reads the broadcast payload.
"""
batch = _batch(
num_computed=[1000],
prefill_len=[1004],
num_scheduled=[4],
is_prefilling=[True],
max_seq_len=[1005],
)
assert pp_utils.compute_need_sampled_mask(batch) is None
def test_keeps_final_prefill_chunk_with_generation_budget_left():
"""The same final chunk, but the request keeps decoding here."""
batch = _batch(
num_computed=[1000],
prefill_len=[1004],
num_scheduled=[4],
is_prefilling=[True],
max_seq_len=[1004 + 256],
)
mask = pp_utils.compute_need_sampled_mask(batch)
assert mask is not None
assert mask.tolist() == [True]
def test_length_cap_overrun_decode_row_is_kept():
"""Decoding rows are never excluded, even past their length cap.
Speculative decoding can transiently overrun prompt_len + max_tokens
while the scheduler still runs the request; dropping the row would
desynchronize the PP stages.
"""
batch = _batch(
num_computed=[14176],
prefill_len=[12175],
num_scheduled=[8],
is_prefilling=[False],
max_seq_len=[12176],
)
mask = pp_utils.compute_need_sampled_mask(batch)
assert mask is not None
assert mask.tolist() == [True]
def test_finishing_prefill_row_does_not_drop_a_sibling_row():
"""Only finishing rows are excluded; a decoding sibling keeps its row."""
batch = _batch(
num_computed=[1000, 10],
prefill_len=[1004, 8],
num_scheduled=[4, 1],
is_prefilling=[True, False],
max_seq_len=[1005, 1024],
)
mask = pp_utils.compute_need_sampled_mask(batch)
assert mask is not None
assert mask.tolist() == [False, True]