[Frontend] don't block event loop in tokenization (preprocess) in OpenAI compatible server (#10635)
Signed-off-by: Tomer Asida <tomera@ai21.com>
This commit is contained in:
+6
-2
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import concurrent
|
||||
import contextlib
|
||||
import datetime
|
||||
import enum
|
||||
@@ -351,7 +352,10 @@ def in_wsl() -> bool:
|
||||
return "microsoft" in " ".join(uname()).lower()
|
||||
|
||||
|
||||
def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
|
||||
def make_async(
|
||||
func: Callable[P, T],
|
||||
executor: Optional[concurrent.futures.Executor] = None
|
||||
) -> Callable[P, Awaitable[T]]:
|
||||
"""Take a blocking function, and run it on in an executor thread.
|
||||
|
||||
This function prevents the blocking function from blocking the
|
||||
@@ -362,7 +366,7 @@ def make_async(func: Callable[P, T]) -> Callable[P, Awaitable[T]]:
|
||||
def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future:
|
||||
loop = asyncio.get_event_loop()
|
||||
p_func = partial(func, *args, **kwargs)
|
||||
return loop.run_in_executor(executor=None, func=p_func)
|
||||
return loop.run_in_executor(executor=executor, func=p_func)
|
||||
|
||||
return _async_wrapper
|
||||
|
||||
|
||||
Reference in New Issue
Block a user