[Bugfix] Abort requests when the connection to /v1/completions is interrupted (#4363)

This commit is contained in:
Ruoyu Qin
2024-04-27 09:48:37 -07:00
committed by GitHub
parent 7134303cbb
commit dfea173148
2 changed files with 53 additions and 5 deletions
+12 -5
View File
@@ -225,11 +225,18 @@ def merge_async_iterators(
]
async def consumer():
while not all(finished) or not queue.empty():
item = await queue.get()
if isinstance(item, Exception):
raise item
yield item
try:
while not all(finished) or not queue.empty():
item = await queue.get()
if isinstance(item, Exception):
raise item
yield item
except (Exception, asyncio.CancelledError) as e:
for task in _tasks:
# NOTE: Pass the error msg in cancel()
# when only Python 3.9+ is supported.
task.cancel()
raise e
await asyncio.gather(*_tasks)
return consumer()