[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)
This commit is contained in:
@@ -323,6 +323,17 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
AsyncGenerator[StreamingResponsesResponse, None]
|
||||
| ResponsesResponse
|
||||
| ErrorResponse
|
||||
):
|
||||
return await self._with_kv_transfer_rejection_cleanup(
|
||||
self._create_responses(request, raw_request), request, raw_request
|
||||
)
|
||||
|
||||
async def _create_responses(
|
||||
self, request: ResponsesRequest, raw_request: Request | None = None
|
||||
) -> (
|
||||
AsyncGenerator[StreamingResponsesResponse, None]
|
||||
| ResponsesResponse
|
||||
| ErrorResponse
|
||||
):
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
|
||||
Reference in New Issue
Block a user