[BugFix] Fix P/D preemption race condition (#50297)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-07-30 01:11:18 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 0a31372e5f
commit 437e0b7f8e
7 changed files with 117 additions and 5 deletions
+73
View File
@@ -638,3 +638,76 @@ def test_reset_prefix_cache_with_inflight_output_under_kv_pressure(pp_size: int)
_assert_positions_consistent(req, engine)
# All stale shares fully drained by the end.
assert getattr(req, "num_stale_output_tokens", 0) == 0
def test_requires_kv_delivery_defaults_to_producer_role():
# No connector: nothing is handed off, so keep the lossless deliver-stale
# path on preemption.
assert create_scheduler(async_scheduling=True).requires_kv_delivery is False
# Only a producer hands KV off when a request completes.
for role, expected in (
("kv_producer", True),
("kv_both", True),
("kv_consumer", False),
):
scheduler = create_scheduler(
async_scheduling=True, use_kv_connector=True, kv_role=role
)
assert scheduler.requires_kv_delivery is expected, role
@pytest.mark.parametrize("kv_role", ["kv_producer", "kv_consumer"])
def test_kv_pressure_preempt_mid_handoff(kv_role: str):
"""P/D race: a request is KV-pressure preempted while the output of its
final prefill chunk -- the hand-off token that would finish it -- is in
flight.
On a producer, that output must be dropped so the request recomputes;
delivering it would finish the request and hand off blocks the preemption
already freed, so the consumer pulls garbage. A consumer hands nothing off,
so it keeps the lossless deliver-stale path.
"""
is_producer = kv_role == "kv_producer"
scheduler = create_scheduler(
async_scheduling=True,
use_kv_connector=True,
kv_role=kv_role,
num_blocks=5,
block_size=16,
max_num_batched_tokens=512,
)
assert scheduler.requires_kv_delivery is is_producer
# 32-token prompts fill 2 blocks each, exhausting the usable pool, so the
# next decode allocation preempts the tail of the running queue (the handoff
# request) while its prefill output is still in flight.
decoder = create_requests(
num_requests=1, num_tokens=32, max_tokens=8, req_ids=["decoder"]
)[0]
handoff = create_requests(
num_requests=1, num_tokens=32, max_tokens=1, req_ids=["handoff"]
)[0]
scheduler.add_request(decoder)
scheduler.add_request(handoff)
sched_output = scheduler.schedule()
assert handoff.status == RequestStatus.RUNNING
assert handoff.num_output_placeholders == 1
scheduler.schedule()
assert handoff.status == RequestStatus.PREEMPTED
assert handoff.num_stale_output_tokens == handoff.num_prompt_tokens
assert handoff.drop_stale_output is is_producer
scheduler.update_from_output(sched_output, _make_model_runner_output(sched_output))
assert handoff.num_stale_output_tokens == 0
if is_producer:
# Dropped: recomputed from the waiting queue, so the hand-off happens
# against real KV.
assert not handoff.is_finished()
assert handoff.status == RequestStatus.PREEMPTED
assert handoff.num_output_tokens == 0
assert handoff.request_id in scheduler.requests
else:
assert handoff.is_finished()
assert handoff.num_output_tokens == 1
+4 -3
View File
@@ -55,6 +55,7 @@ def create_scheduler(
long_prefill_token_threshold: int = 0,
disable_chunked_mm_input: bool = False,
use_kv_connector: None | bool | str | MockKVConfig = None,
kv_role: str = "kv_both",
num_blocks: int = 10000,
block_size: int = 16,
max_model_len: int | None = None,
@@ -115,7 +116,7 @@ def create_scheduler(
if isinstance(use_kv_connector, MockKVConfig):
kv_transfer_config = KVTransferConfig(
kv_connector="MockKVConnector",
kv_role="kv_both",
kv_role=kv_role,
kv_connector_extra_config={
"matched_tokens": use_kv_connector.matched_tokens,
"is_async": use_kv_connector.is_async,
@@ -127,12 +128,12 @@ def create_scheduler(
elif isinstance(use_kv_connector, str):
kv_transfer_config = KVTransferConfig(
kv_connector=use_kv_connector,
kv_role="kv_both",
kv_role=kv_role,
)
elif use_kv_connector:
kv_transfer_config = KVTransferConfig(
kv_connector="ExampleConnector",
kv_role="kv_both",
kv_role=kv_role,
kv_connector_extra_config={"shared_storage_path": "local_storage"},
)