[Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542)
Signed-off-by: LostFox11 <wangziyue17@huawei.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: LostFox11 <wangziyue17@huawei.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
LostFox11
Nick Hill
parent
34c458d27f
commit
7314360c9e
@@ -57,6 +57,7 @@ def _make_runner(
|
||||
runner.decode_query_len = decode_query_len
|
||||
runner.adaptive_verification = None
|
||||
runner.pcp_manager = None
|
||||
runner.use_pp = False
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_states)},
|
||||
# The runner keeps this as min(num_computed_tokens, prefill_len).
|
||||
@@ -246,6 +247,7 @@ def test_uniform_decode_uses_state_index_not_batch_position():
|
||||
runner.decode_query_len = 8
|
||||
runner.adaptive_verification = None
|
||||
runner.pcp_manager = None
|
||||
runner.use_pp = False
|
||||
# State arrays in state-index order: a prefilling request, then two decodes.
|
||||
runner.req_states = SimpleNamespace(
|
||||
req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2},
|
||||
|
||||
@@ -9,12 +9,26 @@ import numpy as np
|
||||
from vllm.v1.worker.gpu import pp_utils
|
||||
|
||||
|
||||
def _batch(num_computed, prefill_len, num_scheduled):
|
||||
def _batch(
|
||||
num_computed,
|
||||
prefill_len,
|
||||
num_scheduled,
|
||||
is_prefilling=None,
|
||||
max_seq_len=None,
|
||||
):
|
||||
return Mock(
|
||||
num_reqs=len(num_computed),
|
||||
num_computed_tokens_np=np.array(num_computed, dtype=np.int32),
|
||||
prefill_len_np=np.array(prefill_len, dtype=np.int32),
|
||||
num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32),
|
||||
is_prefilling_np=(
|
||||
np.array(is_prefilling, dtype=np.bool_)
|
||||
if is_prefilling is not None
|
||||
else None
|
||||
),
|
||||
max_seq_len_np=(
|
||||
np.array(max_seq_len, dtype=np.int32) if max_seq_len is not None else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -81,3 +95,74 @@ def test_decode_row_ahead_of_a_prefill_chunk():
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [True, False]
|
||||
|
||||
|
||||
def test_excludes_final_prefill_chunk_reaching_its_length_cap():
|
||||
"""A max_tokens=1 final chunk hands its token off and leaves the engine.
|
||||
|
||||
Disaggregated prefill submits max_tokens=1, so the request finishes with
|
||||
the final chunk's single sample and no follow-up step on this engine
|
||||
reads the broadcast payload.
|
||||
"""
|
||||
batch = _batch(
|
||||
num_computed=[1000],
|
||||
prefill_len=[1004],
|
||||
num_scheduled=[4],
|
||||
is_prefilling=[True],
|
||||
max_seq_len=[1005],
|
||||
)
|
||||
|
||||
assert pp_utils.compute_need_sampled_mask(batch) is None
|
||||
|
||||
|
||||
def test_keeps_final_prefill_chunk_with_generation_budget_left():
|
||||
"""The same final chunk, but the request keeps decoding here."""
|
||||
batch = _batch(
|
||||
num_computed=[1000],
|
||||
prefill_len=[1004],
|
||||
num_scheduled=[4],
|
||||
is_prefilling=[True],
|
||||
max_seq_len=[1004 + 256],
|
||||
)
|
||||
|
||||
mask = pp_utils.compute_need_sampled_mask(batch)
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [True]
|
||||
|
||||
|
||||
def test_length_cap_overrun_decode_row_is_kept():
|
||||
"""Decoding rows are never excluded, even past their length cap.
|
||||
|
||||
Speculative decoding can transiently overrun prompt_len + max_tokens
|
||||
while the scheduler still runs the request; dropping the row would
|
||||
desynchronize the PP stages.
|
||||
"""
|
||||
batch = _batch(
|
||||
num_computed=[14176],
|
||||
prefill_len=[12175],
|
||||
num_scheduled=[8],
|
||||
is_prefilling=[False],
|
||||
max_seq_len=[12176],
|
||||
)
|
||||
|
||||
mask = pp_utils.compute_need_sampled_mask(batch)
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [True]
|
||||
|
||||
|
||||
def test_finishing_prefill_row_does_not_drop_a_sibling_row():
|
||||
"""Only finishing rows are excluded; a decoding sibling keeps its row."""
|
||||
batch = _batch(
|
||||
num_computed=[1000, 10],
|
||||
prefill_len=[1004, 8],
|
||||
num_scheduled=[4, 1],
|
||||
is_prefilling=[True, False],
|
||||
max_seq_len=[1005, 1024],
|
||||
)
|
||||
|
||||
mask = pp_utils.compute_need_sampled_mask(batch)
|
||||
|
||||
assert mask is not None
|
||||
assert mask.tolist() == [False, True]
|
||||
|
||||
Reference in New Issue
Block a user