[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)

This commit is contained in:
Dao007forever
2026-05-09 22:52:09 -07:00
committed by GitHub
parent fb1ac806c5
commit 3f5bd482f5
11 changed files with 156 additions and 2 deletions
@@ -234,6 +234,15 @@ class OpenAIServingChat(OpenAIServing):
for the API specification. This API mimics the OpenAI
Chat Completion API.
"""
return await self._with_kv_transfer_rejection_cleanup(
self._create_chat_completion(request, raw_request), request, raw_request
)
async def _create_chat_completion(
self,
request: ChatCompletionRequest,
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
# Streaming response
tokenizer = self.renderer.tokenizer
assert tokenizer is not None
@@ -118,6 +118,15 @@ class OpenAIServingCompletion(OpenAIServing):
- suffix (the language models we currently support do not support
suffix)
"""
return await self._with_kv_transfer_rejection_cleanup(
self._create_completion(request, raw_request), request, raw_request
)
async def _create_completion(
self,
request: CompletionRequest,
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
if request.stream and request.use_beam_search:
return self.create_error_response(
"Streaming is not currently supported with beam search"
+39 -1
View File
@@ -4,7 +4,7 @@ import asyncio
import contextlib
import json
import time
from collections.abc import AsyncGenerator, Mapping
from collections.abc import AsyncGenerator, Awaitable, Mapping
from dataclasses import dataclass, field
from http import HTTPStatus
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
@@ -118,6 +118,7 @@ AnyResponse: TypeAlias = (
)
RequestT = TypeVar("RequestT", bound=AnyRequest)
_T = TypeVar("_T")
@dataclass(kw_only=True)
@@ -156,6 +157,9 @@ class OpenAIServing:
self.model_config = engine_client.model_config
self.renderer = engine_client.renderer
self.input_processor = engine_client.input_processor
vllm_config = getattr(engine_client, "vllm_config", None)
kv_transfer_config = getattr(vllm_config, "kv_transfer_config", None)
self.has_kv_connector = kv_transfer_config is not None
# Computed once at startup (cached by ``vllm_config`` identity) and
# stamped on non-streaming responses. Streaming chunks deliberately
@@ -616,6 +620,40 @@ class OpenAIServing:
except ValueError:
return None
async def _with_kv_transfer_rejection_cleanup(
self,
awaitable: Awaitable[_T],
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
raw_request: Request | None,
) -> _T:
"""Wrap a `create_*` coroutine so that, if it raises or returns an
ErrorResponse (i.e. the request never reached the engine), the KV
connector is notified to free any pinned remote-prefill blocks."""
kv_transfer_params = self.has_kv_connector and request.kv_transfer_params
if not kv_transfer_params or not kv_transfer_params.get("do_remote_prefill"):
return await awaitable
notify = True
try:
result = await awaitable
if not isinstance(result, ErrorResponse):
notify = False
return result
finally:
if notify:
try:
await self.engine_client.notify_kv_transfer_request_rejected(
request.request_id,
kv_transfer_params,
data_parallel_rank=self._get_data_parallel_rank(raw_request),
)
except Exception:
logger.warning(
"Failed to notify KV connector about rejected request %s",
request.request_id,
exc_info=True,
)
@staticmethod
def _parse_tool_calls_from_content(
request: ResponsesRequest | ChatCompletionRequest,
@@ -323,6 +323,17 @@ class OpenAIServingResponses(OpenAIServing):
AsyncGenerator[StreamingResponsesResponse, None]
| ResponsesResponse
| ErrorResponse
):
return await self._with_kv_transfer_rejection_cleanup(
self._create_responses(request, raw_request), request, raw_request
)
async def _create_responses(
self, request: ResponsesRequest, raw_request: Request | None = None
) -> (
AsyncGenerator[StreamingResponsesResponse, None]
| ResponsesResponse
| ErrorResponse
):
error_check_ret = await self._check_model(request)
if error_check_ret is not None: