[Bugfix][KV Transfer][NIXL] Notify P node on pre-admission rejection to free stranded KV blocks (#41269)
This commit is contained in:
@@ -118,6 +118,15 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
- suffix (the language models we currently support do not support
|
||||
suffix)
|
||||
"""
|
||||
return await self._with_kv_transfer_rejection_cleanup(
|
||||
self._create_completion(request, raw_request), request, raw_request
|
||||
)
|
||||
|
||||
async def _create_completion(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> AsyncGenerator[str, None] | CompletionResponse | ErrorResponse:
|
||||
if request.stream and request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Streaming is not currently supported with beam search"
|
||||
|
||||
Reference in New Issue
Block a user