[BugFix] Fix P/D preemption race condition (#50297)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
0a31372e5f
commit
437e0b7f8e
@@ -638,3 +638,76 @@ def test_reset_prefix_cache_with_inflight_output_under_kv_pressure(pp_size: int)
|
||||
_assert_positions_consistent(req, engine)
|
||||
# All stale shares fully drained by the end.
|
||||
assert getattr(req, "num_stale_output_tokens", 0) == 0
|
||||
|
||||
|
||||
def test_requires_kv_delivery_defaults_to_producer_role():
|
||||
# No connector: nothing is handed off, so keep the lossless deliver-stale
|
||||
# path on preemption.
|
||||
assert create_scheduler(async_scheduling=True).requires_kv_delivery is False
|
||||
# Only a producer hands KV off when a request completes.
|
||||
for role, expected in (
|
||||
("kv_producer", True),
|
||||
("kv_both", True),
|
||||
("kv_consumer", False),
|
||||
):
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True, use_kv_connector=True, kv_role=role
|
||||
)
|
||||
assert scheduler.requires_kv_delivery is expected, role
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_role", ["kv_producer", "kv_consumer"])
|
||||
def test_kv_pressure_preempt_mid_handoff(kv_role: str):
|
||||
"""P/D race: a request is KV-pressure preempted while the output of its
|
||||
final prefill chunk -- the hand-off token that would finish it -- is in
|
||||
flight.
|
||||
|
||||
On a producer, that output must be dropped so the request recomputes;
|
||||
delivering it would finish the request and hand off blocks the preemption
|
||||
already freed, so the consumer pulls garbage. A consumer hands nothing off,
|
||||
so it keeps the lossless deliver-stale path.
|
||||
"""
|
||||
is_producer = kv_role == "kv_producer"
|
||||
scheduler = create_scheduler(
|
||||
async_scheduling=True,
|
||||
use_kv_connector=True,
|
||||
kv_role=kv_role,
|
||||
num_blocks=5,
|
||||
block_size=16,
|
||||
max_num_batched_tokens=512,
|
||||
)
|
||||
assert scheduler.requires_kv_delivery is is_producer
|
||||
|
||||
# 32-token prompts fill 2 blocks each, exhausting the usable pool, so the
|
||||
# next decode allocation preempts the tail of the running queue (the handoff
|
||||
# request) while its prefill output is still in flight.
|
||||
decoder = create_requests(
|
||||
num_requests=1, num_tokens=32, max_tokens=8, req_ids=["decoder"]
|
||||
)[0]
|
||||
handoff = create_requests(
|
||||
num_requests=1, num_tokens=32, max_tokens=1, req_ids=["handoff"]
|
||||
)[0]
|
||||
scheduler.add_request(decoder)
|
||||
scheduler.add_request(handoff)
|
||||
sched_output = scheduler.schedule()
|
||||
assert handoff.status == RequestStatus.RUNNING
|
||||
assert handoff.num_output_placeholders == 1
|
||||
|
||||
scheduler.schedule()
|
||||
assert handoff.status == RequestStatus.PREEMPTED
|
||||
assert handoff.num_stale_output_tokens == handoff.num_prompt_tokens
|
||||
assert handoff.drop_stale_output is is_producer
|
||||
|
||||
scheduler.update_from_output(sched_output, _make_model_runner_output(sched_output))
|
||||
|
||||
assert handoff.num_stale_output_tokens == 0
|
||||
if is_producer:
|
||||
# Dropped: recomputed from the waiting queue, so the hand-off happens
|
||||
# against real KV.
|
||||
assert not handoff.is_finished()
|
||||
assert handoff.status == RequestStatus.PREEMPTED
|
||||
assert handoff.num_output_tokens == 0
|
||||
assert handoff.request_id in scheduler.requests
|
||||
else:
|
||||
assert handoff.is_finished()
|
||||
assert handoff.num_output_tokens == 1
|
||||
|
||||
@@ -55,6 +55,7 @@ def create_scheduler(
|
||||
long_prefill_token_threshold: int = 0,
|
||||
disable_chunked_mm_input: bool = False,
|
||||
use_kv_connector: None | bool | str | MockKVConfig = None,
|
||||
kv_role: str = "kv_both",
|
||||
num_blocks: int = 10000,
|
||||
block_size: int = 16,
|
||||
max_model_len: int | None = None,
|
||||
@@ -115,7 +116,7 @@ def create_scheduler(
|
||||
if isinstance(use_kv_connector, MockKVConfig):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="MockKVConnector",
|
||||
kv_role="kv_both",
|
||||
kv_role=kv_role,
|
||||
kv_connector_extra_config={
|
||||
"matched_tokens": use_kv_connector.matched_tokens,
|
||||
"is_async": use_kv_connector.is_async,
|
||||
@@ -127,12 +128,12 @@ def create_scheduler(
|
||||
elif isinstance(use_kv_connector, str):
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector=use_kv_connector,
|
||||
kv_role="kv_both",
|
||||
kv_role=kv_role,
|
||||
)
|
||||
elif use_kv_connector:
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="ExampleConnector",
|
||||
kv_role="kv_both",
|
||||
kv_role=kv_role,
|
||||
kv_connector_extra_config={"shared_storage_path": "local_storage"},
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user