[Bugfix] Fix request cancellation without polling (#11190)

This commit is contained in:
Joe Runde
2024-12-17 12:26:32 -08:00
committed by GitHub
parent f9ecbb18bf
commit 2d1b9baa8f
12 changed files with 164 additions and 103 deletions
+1 -4
View File
@@ -202,10 +202,7 @@ class OpenAIServingEmbedding(OpenAIServing):
# TODO: Use a vllm-specific Validation Error
return self.create_error_response(str(e))
result_generator = merge_async_iterators(
*generators,
is_cancelled=raw_request.is_disconnected if raw_request else None,
)
result_generator = merge_async_iterators(*generators)
num_prompts = len(engine_prompts)