[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)

This commit is contained in:
Dao007forever
2026-05-09 22:52:09 -07:00
committed by GitHub
parent fb1ac806c5
commit 3f5bd482f5
11 changed files with 156 additions and 2 deletions
@@ -118,6 +118,15 @@ class OpenAIServingCompletion(OpenAIServing):
- suffix (the language models we currently support do not support
suffix)
"""
return await self._with_kv_transfer_rejection_cleanup(
self._create_completion(request, raw_request), request, raw_request
)
async def _create_completion(
self,
request: CompletionRequest,
raw_request: Request | None = None,
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
if request.stream and request.use_beam_search:
return self.create_error_response(
"Streaming is not currently supported with beam search"