[Bugfix] Fix request cancellation without polling (#11190)

This commit is contained in:
Joe Runde
2024-12-17 12:26:32 -08:00
committed by GitHub
parent f9ecbb18bf
commit 2d1b9baa8f
12 changed files with 164 additions and 103 deletions
@@ -159,8 +159,7 @@ class OpenAIServingCompletion(OpenAIServing):
# TODO: Use a vllm-specific Validation Error
return self.create_error_response(str(e))
result_generator = merge_async_iterators(
*generators, is_cancelled=raw_request.is_disconnected)
result_generator = merge_async_iterators(*generators)
model_name = self._get_model_name(lora_request)
num_prompts = len(engine_prompts)