[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)
This commit is contained in:
@@ -234,6 +234,15 @@ class OpenAIServingChat(OpenAIServing):
|
||||
for the API specification. This API mimics the OpenAI
|
||||
Chat Completion API.
|
||||
"""
|
||||
return await self._with_kv_transfer_rejection_cleanup(
|
||||
self._create_chat_completion(request, raw_request), request, raw_request
|
||||
)
|
||||
|
||||
async def _create_chat_completion(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
|
||||
# Streaming response
|
||||
tokenizer = self.renderer.tokenizer
|
||||
assert tokenizer is not None
|
||||
|
||||
@@ -118,6 +118,15 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
- suffix (the language models we currently support do not support
|
||||
suffix)
|
||||
"""
|
||||
return await self._with_kv_transfer_rejection_cleanup(
|
||||
self._create_completion(request, raw_request), request, raw_request
|
||||
)
|
||||
|
||||
async def _create_completion(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
|
||||
if request.stream and request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Streaming is not currently supported with beam search"
|
||||
|
||||
@@ -4,7 +4,7 @@ import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from collections.abc import AsyncGenerator, Awaitable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from http import HTTPStatus
|
||||
from typing import Any, ClassVar, Generic, Protocol, TypeAlias, TypeVar
|
||||
@@ -118,6 +118,7 @@ AnyResponse: TypeAlias = (
|
||||
)
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
@@ -156,6 +157,9 @@ class OpenAIServing:
|
||||
self.model_config = engine_client.model_config
|
||||
self.renderer = engine_client.renderer
|
||||
self.input_processor = engine_client.input_processor
|
||||
vllm_config = getattr(engine_client, "vllm_config", None)
|
||||
kv_transfer_config = getattr(vllm_config, "kv_transfer_config", None)
|
||||
self.has_kv_connector = kv_transfer_config is not None
|
||||
|
||||
# Computed once at startup (cached by ``vllm_config`` identity) and
|
||||
# stamped on non-streaming responses. Streaming chunks deliberately
|
||||
@@ -616,6 +620,40 @@ class OpenAIServing:
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def _with_kv_transfer_rejection_cleanup(
|
||||
self,
|
||||
awaitable: Awaitable[_T],
|
||||
request: ChatCompletionRequest | CompletionRequest | ResponsesRequest,
|
||||
raw_request: Request | None,
|
||||
) -> _T:
|
||||
"""Wrap a `create_*` coroutine so that, if it raises or returns an
|
||||
ErrorResponse (i.e. the request never reached the engine), the KV
|
||||
connector is notified to free any pinned remote-prefill blocks."""
|
||||
kv_transfer_params = self.has_kv_connector and request.kv_transfer_params
|
||||
if not kv_transfer_params or not kv_transfer_params.get("do_remote_prefill"):
|
||||
return await awaitable
|
||||
|
||||
notify = True
|
||||
try:
|
||||
result = await awaitable
|
||||
if not isinstance(result, ErrorResponse):
|
||||
notify = False
|
||||
return result
|
||||
finally:
|
||||
if notify:
|
||||
try:
|
||||
await self.engine_client.notify_kv_transfer_request_rejected(
|
||||
request.request_id,
|
||||
kv_transfer_params,
|
||||
data_parallel_rank=self._get_data_parallel_rank(raw_request),
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to notify KV connector about rejected request %s",
|
||||
request.request_id,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_tool_calls_from_content(
|
||||
request: ResponsesRequest | ChatCompletionRequest,
|
||||
|
||||
@@ -323,6 +323,17 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
AsyncGenerator[StreamingResponsesResponse, None]
|
||||
| ResponsesResponse
|
||||
| ErrorResponse
|
||||
):
|
||||
return await self._with_kv_transfer_rejection_cleanup(
|
||||
self._create_responses(request, raw_request), request, raw_request
|
||||
)
|
||||
|
||||
async def _create_responses(
|
||||
self, request: ResponsesRequest, raw_request: Request | None = None
|
||||
) -> (
|
||||
AsyncGenerator[StreamingResponsesResponse, None]
|
||||
| ResponsesResponse
|
||||
| ErrorResponse
|
||||
):
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
|
||||
Reference in New Issue
Block a user