[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)
This commit is contained in:
@@ -720,6 +720,33 @@ class AsyncLLM(EngineClient):
|
||||
if self.log_requests:
|
||||
logger.info("Aborted request(s) %s.", ",".join(request_ids))
|
||||
|
||||
async def notify_kv_transfer_request_rejected(
|
||||
self,
|
||||
request_id: str,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
*,
|
||||
data_parallel_rank: int | None = None,
|
||||
) -> None:
|
||||
"""Submit a pre-aborted request so the connector's request_finished
|
||||
hook runs to free any pre-admission KV-transfer resources (e.g. NIXL
|
||||
prefill blocks pinned on the P node)."""
|
||||
request = EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=[0],
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(
|
||||
max_tokens=1,
|
||||
extra_args={"kv_transfer_params": dict(kv_transfer_params)},
|
||||
),
|
||||
pooling_params=None,
|
||||
arrival_time=time.time(),
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
abort_immediately=True,
|
||||
)
|
||||
await self.engine_core.add_request_async(request)
|
||||
|
||||
async def pause_generation(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user