[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)
This commit is contained in:
@@ -122,6 +122,12 @@ class EngineCoreRequest(
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None
|
||||
|
||||
# If True, the request should be added to the scheduler's waiting queue
|
||||
# and immediately aborted, so connector-side cleanup runs via the standard
|
||||
# request_finished hook. Used to free P-side prefill blocks when a
|
||||
# KV-transfer request is rejected on the D node before engine admission.
|
||||
abort_immediately: bool = False
|
||||
|
||||
@property
|
||||
def params(self) -> SamplingParams | PoolingParams:
|
||||
"""Return the processed params (sampling or pooling)."""
|
||||
|
||||
@@ -720,6 +720,33 @@ class AsyncLLM(EngineClient):
|
||||
if self.log_requests:
|
||||
logger.info("Aborted request(s) %s.", ",".join(request_ids))
|
||||
|
||||
async def notify_kv_transfer_request_rejected(
|
||||
self,
|
||||
request_id: str,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
*,
|
||||
data_parallel_rank: int | None = None,
|
||||
) -> None:
|
||||
"""Submit a pre-aborted request so the connector's request_finished
|
||||
hook runs to free any pre-admission KV-transfer resources (e.g. NIXL
|
||||
prefill blocks pinned on the P node)."""
|
||||
request = EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=[0],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(
|
||||
max_tokens=1,
|
||||
extra_args={"kv_transfer_params": dict(kv_transfer_params)},
|
||||
),
|
||||
pooling_params=None,
|
||||
arrival_time=time.time(),
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
abort_immediately=True,
|
||||
)
|
||||
await self.engine_core.add_request_async(request)
|
||||
|
||||
async def pause_generation(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -344,6 +344,10 @@ class EngineCore:
|
||||
)
|
||||
|
||||
self.scheduler.add_request(request)
|
||||
if request.abort_immediately:
|
||||
# Immediately abort so the connector's request_finished hook runs
|
||||
# to free any pre-admission KV-transfer resources.
|
||||
self.abort_requests([request.request_id])
|
||||
|
||||
def abort_requests(self, request_ids: list[str]):
|
||||
"""Abort requests from the scheduler."""
|
||||
|
||||
Reference in New Issue
Block a user