[Core] Apply prefill_schedule_interval outside data parallelism (#54627)
Signed-off-by: lifulu <fululi12@amd.com> Signed-off-by: lifulu <lifulu@users.noreply.github.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: lifulu <lifulu@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cursor Agent
lifulu
Claude Opus 5
parent
c825e1d235
commit
ed3050add4
@@ -558,6 +558,249 @@ def test_schedule_prefills_gating(has_running: bool):
|
||||
assert any(r.req_id == "new0" for r in output.scheduled_new_reqs)
|
||||
|
||||
|
||||
def _decode_one(scheduler, output, req_ids: list[str]) -> None:
|
||||
"""Feed back one sampled token per request, so they keep decoding."""
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
|
||||
sampled_token_ids=[[0] for _ in req_ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _start_decoding(scheduler, req_id: str = "run0"):
|
||||
"""Prefill one request and take it into the decode state."""
|
||||
(request,) = create_requests(
|
||||
num_requests=1, num_tokens=8, max_tokens=100, req_ids=[req_id]
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
output = scheduler.schedule()
|
||||
assert req_id in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, [req_id])
|
||||
assert len(scheduler.running) == 1
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [2, 4])
|
||||
def test_local_prefill_interval_holds_prefill_between_bursts(interval: int):
|
||||
"""A single engine applies `prefill_schedule_interval` in the scheduler,
|
||||
timed from the last prefill-carrying step: a waiting prefill is held for the
|
||||
remainder of the interval while running requests keep decoding, then
|
||||
admitted once the interval has elapsed.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=interval)
|
||||
# This prefill starts the interval.
|
||||
_start_decoding(scheduler)
|
||||
assert scheduler.last_prefill_step == scheduler.num_executed_steps
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
|
||||
# Steps inside the interval decode alone; the waiting prefill is held.
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "new0" not in output.num_scheduled_tokens
|
||||
assert "run0" in output.num_scheduled_tokens
|
||||
assert new_req.status == RequestStatus.WAITING
|
||||
_decode_one(scheduler, output, ["run0"])
|
||||
|
||||
# The interval has now elapsed, so prefill is admitted again.
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
assert scheduler.last_prefill_step == scheduler.num_executed_steps
|
||||
|
||||
|
||||
def test_empty_schedule_does_not_advance_the_prefill_cadence():
|
||||
"""An empty schedule() must not consume interval budget.
|
||||
|
||||
`current_step` counts every schedule() call. With async scheduling,
|
||||
step_with_batch_queue() calls schedule() again whenever the batch queue has
|
||||
room and has_requests() is true -- so a call lands while the previous batch
|
||||
is still in flight and nothing new can be scheduled. An async KV connector
|
||||
makes this common, since requests sit in lookup or WAITING_FOR_REMOTE_KVS.
|
||||
Timing the cadence on current_step lets those empty calls expire the hold
|
||||
early. Reported on #54627 against the LMCache MP connector.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=4)
|
||||
_start_decoding(scheduler)
|
||||
|
||||
# One real decode step, left in flight (no update_from_output yet).
|
||||
output = scheduler.schedule()
|
||||
assert output.total_num_scheduled_tokens > 0
|
||||
executed_before = scheduler.num_executed_steps
|
||||
step_before = scheduler.current_step
|
||||
|
||||
# Re-entering schedule() while that batch is still unaccounted for
|
||||
# schedules nothing -- this is the async-scheduling shape.
|
||||
for _ in range(3):
|
||||
empty = scheduler.schedule()
|
||||
assert empty.total_num_scheduled_tokens == 0
|
||||
|
||||
assert scheduler.current_step == step_before + 3, "current_step should advance"
|
||||
assert scheduler.num_executed_steps == executed_before, (
|
||||
"an empty schedule() must not count as an executed step"
|
||||
)
|
||||
|
||||
|
||||
def test_prefill_cadence_counts_executed_steps_not_schedule_calls():
|
||||
"""The hold spans `interval` EXECUTED steps, however many empty schedule()
|
||||
calls are interleaved.
|
||||
|
||||
Counting schedule() calls instead lets each empty call eat interval budget,
|
||||
so with an async KV connector prefill lands on nearly every step and the
|
||||
feature stops doing anything.
|
||||
"""
|
||||
interval = 4
|
||||
scheduler = create_scheduler(prefill_schedule_interval=interval)
|
||||
_start_decoding(scheduler)
|
||||
prefill_at = scheduler.num_executed_steps
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
|
||||
empty_calls = 0
|
||||
admitted_at = None
|
||||
for _ in range(50):
|
||||
output = scheduler.schedule()
|
||||
if "new0" in output.num_scheduled_tokens:
|
||||
admitted_at = scheduler.num_executed_steps
|
||||
break
|
||||
if output.total_num_scheduled_tokens == 0:
|
||||
empty_calls += 1
|
||||
continue
|
||||
# A real decode step. Leave it in flight and re-enter schedule() twice,
|
||||
# as async scheduling does while the batch queue has room.
|
||||
for _ in range(2):
|
||||
extra = scheduler.schedule()
|
||||
if "new0" in extra.num_scheduled_tokens:
|
||||
admitted_at = scheduler.num_executed_steps
|
||||
break
|
||||
empty_calls += 1
|
||||
if admitted_at is not None:
|
||||
break
|
||||
_decode_one(scheduler, output, ["run0"])
|
||||
|
||||
assert admitted_at is not None, "prefill was never admitted"
|
||||
assert empty_calls > 0, "test never exercised an empty schedule() call"
|
||||
# The hold is measured in executed steps. Counting schedule() calls would
|
||||
# admit after far fewer, because the empty re-entries above would count.
|
||||
assert admitted_at - prefill_at == interval, (
|
||||
f"hold lasted {admitted_at - prefill_at} executed steps, expected "
|
||||
f"{interval}; empty schedule() calls were counted against the cadence"
|
||||
)
|
||||
assert scheduler.current_step > scheduler.num_executed_steps
|
||||
|
||||
|
||||
def test_local_prefill_interval_defaults_to_no_gating():
|
||||
"""The default interval of 1 leaves admission exactly as it was: a waiting
|
||||
prefill is admitted on the very next step.
|
||||
"""
|
||||
scheduler = create_scheduler()
|
||||
assert scheduler.local_prefill_interval == 1
|
||||
_start_decoding(scheduler)
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_local_prefill_interval_left_to_engine_under_dp():
|
||||
"""Under data parallelism the cadence must stay aligned across ranks, so the
|
||||
DP engine core drives it and the scheduler must not gate on its own.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=4, data_parallel_size=2)
|
||||
assert scheduler.local_prefill_interval == 1
|
||||
_start_decoding(scheduler)
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_local_prefill_interval_never_idles_the_model():
|
||||
"""With nothing running to decode in prefill's place, the interval must not
|
||||
hold prefill back, or the step would be wasted entirely.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=8)
|
||||
_start_decoding(scheduler)
|
||||
|
||||
# Finish the only running request, leaving nothing to decode.
|
||||
scheduler.finish_requests("run0", RequestStatus.FINISHED_ABORTED)
|
||||
assert not scheduler.running
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [2, 3])
|
||||
def test_local_prefill_interval_defers_inflight_prefill_chunk(interval: int):
|
||||
"""The single-engine interval gates prefill WORK, not just new admissions.
|
||||
|
||||
A request that is already partially prefilled is held for the remainder of
|
||||
the interval too, exactly as `throttle_prefills` does for DP. That is the
|
||||
point of the cadence -- a held step must carry no prefill at all, or decode
|
||||
does not get the uninterrupted stretch the interval exists to give it.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16,
|
||||
max_num_batched_tokens=50,
|
||||
enable_chunked_prefill=True,
|
||||
prefill_schedule_interval=interval,
|
||||
)
|
||||
|
||||
# A short request that finishes prefill in one step -> a running decode.
|
||||
(decode_req,) = create_requests(num_requests=1, num_tokens=4, req_ids=["dec0"])
|
||||
scheduler.add_request(decode_req)
|
||||
output = scheduler.schedule()
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
assert decode_req in scheduler.running and not decode_req.is_prefill_chunk
|
||||
|
||||
# A long request (80 tokens, budget 50) -> prefilled in chunks. That first
|
||||
# prefill closed the interval, so walk to the next step that opens it.
|
||||
(chunk_req,) = create_requests(num_requests=1, num_tokens=80, req_ids=["chk0"])
|
||||
scheduler.add_request(chunk_req)
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" not in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens["chk0"] > 0
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=["dec0", "chk0"],
|
||||
req_id_to_index={"dec0": 0, "chk0": 1},
|
||||
sampled_token_ids=[[0], []], # no token sampled for partial prefill
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
assert chunk_req.is_prefill_chunk # still mid-prefill, in running
|
||||
|
||||
# Held steps: the in-flight chunk is deferred along with new admissions,
|
||||
# while the decode keeps running.
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" not in output.num_scheduled_tokens
|
||||
assert "dec0" in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
|
||||
# Once the interval has elapsed the chunk resumes.
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def _setup_remote_kv_resume(num_prompt_tokens: int, matched_tokens: int):
|
||||
"""Drive a remote-KV request `r2` to the resume point (async load complete)
|
||||
while another request `r1` is already decoding, so the step is throttle-
|
||||
|
||||
@@ -85,6 +85,7 @@ def create_scheduler(
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
runner: RunnerOption = "auto",
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
prefill_schedule_interval: int = 1,
|
||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||
diffusion_canvas_length: int | None = None,
|
||||
scheduler_cls: type[Scheduler] | None = None,
|
||||
@@ -134,6 +135,7 @@ def create_scheduler(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
async_scheduling=async_scheduling,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
prefill_schedule_interval=prefill_schedule_interval,
|
||||
# Ensure admission/preemption mechanics are deterministic
|
||||
watermark=0.0,
|
||||
policy=scheduling_policy,
|
||||
|
||||
Reference in New Issue
Block a user