[Renderer] Offload derender CPU work to renderer thread pool (#49396)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
@@ -24,6 +24,7 @@ from vllm.parser import Parser, ParserManager
|
|||||||
from vllm.renderers import BaseRenderer
|
from vllm.renderers import BaseRenderer
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
|
from vllm.utils.async_utils import make_async
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
@@ -73,10 +74,26 @@ class OnlineDerenderer:
|
|||||||
self.supports_browsing = False
|
self.supports_browsing = False
|
||||||
self.supports_code_interpreter = False
|
self.supports_code_interpreter = False
|
||||||
|
|
||||||
|
# Detokenization, logprob resolution and parsing are CPU-bound;
|
||||||
|
# offload them in one hop to keep the event loop responsive.
|
||||||
|
self._derender_chat_async = make_async(
|
||||||
|
self._derender_chat, executor=renderer._executor
|
||||||
|
)
|
||||||
|
self._derender_completion_async = make_async(
|
||||||
|
self._derender_completion, executor=renderer._executor
|
||||||
|
)
|
||||||
|
|
||||||
async def derender_chat(
|
async def derender_chat(
|
||||||
self,
|
self,
|
||||||
generate_response: GenerateResponse,
|
generate_response: GenerateResponse,
|
||||||
chat_request: ChatCompletionRequest | None = None,
|
chat_request: ChatCompletionRequest | None = None,
|
||||||
|
) -> list[ChatCompletionResponseChoice]:
|
||||||
|
return await self._derender_chat_async(generate_response, chat_request)
|
||||||
|
|
||||||
|
def _derender_chat(
|
||||||
|
self,
|
||||||
|
generate_response: GenerateResponse,
|
||||||
|
chat_request: ChatCompletionRequest | None = None,
|
||||||
) -> list[ChatCompletionResponseChoice]:
|
) -> list[ChatCompletionResponseChoice]:
|
||||||
tokenizer = self.renderer.get_tokenizer()
|
tokenizer = self.renderer.get_tokenizer()
|
||||||
choices: list[ChatCompletionResponseChoice] = []
|
choices: list[ChatCompletionResponseChoice] = []
|
||||||
@@ -172,6 +189,13 @@ class OnlineDerenderer:
|
|||||||
self,
|
self,
|
||||||
generate_responses: list[GenerateResponse],
|
generate_responses: list[GenerateResponse],
|
||||||
prompt_tokens: list[int] | None = None,
|
prompt_tokens: list[int] | None = None,
|
||||||
|
) -> tuple[list[CompletionResponseChoice], int, int]:
|
||||||
|
return await self._derender_completion_async(generate_responses, prompt_tokens)
|
||||||
|
|
||||||
|
def _derender_completion(
|
||||||
|
self,
|
||||||
|
generate_responses: list[GenerateResponse],
|
||||||
|
prompt_tokens: list[int] | None = None,
|
||||||
) -> tuple[list[CompletionResponseChoice], int, int]:
|
) -> tuple[list[CompletionResponseChoice], int, int]:
|
||||||
n = len(generate_responses)
|
n = len(generate_responses)
|
||||||
prompt_tokens_list: list[int] = (
|
prompt_tokens_list: list[int] = (
|
||||||
|
|||||||
Reference in New Issue
Block a user