[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)

This commit is contained in:
Dao007forever
2026-05-09 22:52:09 -07:00
committed by GitHub
parent fb1ac806c5
commit 3f5bd482f5
11 changed files with 156 additions and 2 deletions
@@ -234,6 +234,15 @@ class OpenAIServingChat(OpenAIServing):
for the API specification. This API mimics the OpenAI
Chat Completion API.
"""
return await self._with_kv_transfer_rejection_cleanup(
self._create_chat_completion(request, raw_request), request, raw_request
)
async def _create_chat_completion(
self,
request: ChatCompletionRequest,
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
# Streaming response
tokenizer = self.renderer.tokenizer
assert tokenizer is not None