[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)

This commit is contained in:
Dao007forever
2026-05-09 22:52:09 -07:00
committed by GitHub
parent fb1ac806c5
commit 3f5bd482f5
11 changed files with 156 additions and 2 deletions
+6
View File
@@ -122,6 +122,12 @@ class EngineCoreRequest(
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, Any] | None = None
# If True, the request should be added to the scheduler's waiting queue
# and immediately aborted, so connector-side cleanup runs via the standard
# request_finished hook. Used to free P-side prefill blocks when a
# KV-transfer request is rejected on the D node before engine admission.
abort_immediately: bool = False
@property
def params(self) -> SamplingParams | PoolingParams:
"""Return the processed params (sampling or pooling)."""
+27
View File
@@ -720,6 +720,33 @@ class AsyncLLM(EngineClient):
if self.log_requests:
logger.info("Aborted request(s) %s.", ",".join(request_ids))
async def notify_kv_transfer_request_rejected(
self,
request_id: str,
kv_transfer_params: dict[str, Any],
*,
data_parallel_rank: int | None = None,
) -> None:
"""Submit a pre-aborted request so the connector's request_finished
hook runs to free any pre-admission KV-transfer resources (e.g. NIXL
prefill blocks pinned on the P node)."""
request = EngineCoreRequest(
request_id=request_id,
prompt_token_ids=[0],
mm_features=None,
sampling_params=SamplingParams(
max_tokens=1,
extra_args={"kv_transfer_params": dict(kv_transfer_params)},
),
pooling_params=None,
arrival_time=time.time(),
lora_request=None,
cache_salt=None,
data_parallel_rank=data_parallel_rank,
abort_immediately=True,
)
await self.engine_core.add_request_async(request)
async def pause_generation(
self,
*,
+4
View File
@@ -344,6 +344,10 @@ class EngineCore:
)
self.scheduler.add_request(request)
if request.abort_immediately:
# Immediately abort so the connector's request_finished hook runs
# to free any pre-admission KV-transfer resources.
self.abort_requests([request.request_id])
def abort_requests(self, request_ids: list[str]):
"""Abort requests from the scheduler."""