[Renderer] Move Processor out of AsyncLLM (#24138)
Signed-off-by: Yang <lymailforjob@gmail.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
DarkLight1337
parent
5f2cacdb1e
commit
812b7f54a8
@@ -7,7 +7,8 @@ import traceback
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union
|
||||
from typing import (Any, Callable, ClassVar, Generic, NamedTuple, Optional,
|
||||
TypeVar, Union)
|
||||
|
||||
import torch
|
||||
from fastapi import Request
|
||||
@@ -15,6 +16,11 @@ from pydantic import BaseModel, ConfigDict, Field
|
||||
from starlette.datastructures import Headers
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.entrypoints.utils import _validate_truncation_size
|
||||
from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.processor import Processor
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import TypedDict
|
||||
else:
|
||||
@@ -134,6 +140,12 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:
|
||||
and "prompt_embeds" in prompt)
|
||||
|
||||
|
||||
class PromptComponents(NamedTuple):
|
||||
text: Optional[str] = None
|
||||
token_ids: Optional[list[int]] = None
|
||||
embeds: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
|
||||
|
||||
@@ -239,6 +251,16 @@ class OpenAIServing:
|
||||
AsyncMicrobatchTokenizer] = {}
|
||||
self.log_error_stack = log_error_stack
|
||||
|
||||
async def _get_processor(self) -> Processor:
|
||||
if not hasattr(self, "_processor"):
|
||||
vllm_config = await self.engine_client.get_vllm_config()
|
||||
if self.model_config.skip_tokenizer_init:
|
||||
tokenizer = None
|
||||
else:
|
||||
tokenizer = init_tokenizer_from_configs(self.model_config)
|
||||
self._processor = Processor(vllm_config, tokenizer)
|
||||
return self._processor
|
||||
|
||||
def _get_renderer(self, tokenizer: Optional[AnyTokenizer]) -> BaseRenderer:
|
||||
"""
|
||||
Get a Renderer instance with the provided tokenizer.
|
||||
@@ -850,6 +872,36 @@ class OpenAIServing:
|
||||
|
||||
return conversation, [request_prompt], [engine_prompt]
|
||||
|
||||
async def _process_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: PromptType,
|
||||
sampling_params: SamplingParams,
|
||||
*,
|
||||
lora_request: Optional[LoRARequest],
|
||||
trace_headers: Optional[Mapping[str, str]],
|
||||
priority: int,
|
||||
) -> tuple[EngineCoreRequest, dict[str, Any]]:
|
||||
"""
|
||||
using the Processor to process inputs for AsyncLLM
|
||||
"""
|
||||
tokenization_kwargs: dict[str, Any] = {}
|
||||
_validate_truncation_size(self.max_model_len,
|
||||
sampling_params.truncate_prompt_tokens,
|
||||
tokenization_kwargs)
|
||||
|
||||
processor = await self._get_processor()
|
||||
engine_request = processor.process_inputs(
|
||||
request_id,
|
||||
engine_prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
)
|
||||
return engine_request, tokenization_kwargs
|
||||
|
||||
async def _generate_with_builtin_tools(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -861,6 +913,7 @@ class OpenAIServing:
|
||||
priority: int = 0,
|
||||
**kwargs,
|
||||
):
|
||||
prompt_text, _, _ = self._get_prompt_components(request_prompt)
|
||||
orig_priority = priority
|
||||
while True:
|
||||
self._log_inputs(
|
||||
@@ -869,14 +922,27 @@ class OpenAIServing:
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
generator = self.engine_client.generate(
|
||||
trace_headers = kwargs.get("trace_headers")
|
||||
engine_request, tokenization_kwargs = (await self._process_inputs(
|
||||
request_id,
|
||||
engine_prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
))
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_request,
|
||||
sampling_params,
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
prompt_text=prompt_text,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
context.append_output(res)
|
||||
# NOTE(woosuk): The stop condition is handled by the engine.
|
||||
@@ -905,6 +971,28 @@ class OpenAIServing:
|
||||
# OPTIMIZATION
|
||||
priority = orig_priority - 1
|
||||
|
||||
def _get_prompt_components(
|
||||
self,
|
||||
inputs: Union[RequestPrompt, PromptType],
|
||||
) -> PromptComponents:
|
||||
if isinstance(inputs, str):
|
||||
return PromptComponents(text=inputs)
|
||||
if isinstance(inputs, list):
|
||||
return PromptComponents(token_ids=inputs)
|
||||
if isinstance(inputs, dict):
|
||||
return PromptComponents(
|
||||
text=inputs.get("prompt"), # type: ignore[arg-type]
|
||||
token_ids=inputs.get(
|
||||
"prompt_token_ids"), # type: ignore[arg-type]
|
||||
embeds=inputs.get("prompt_embeds"),
|
||||
)
|
||||
|
||||
return PromptComponents(
|
||||
text=getattr(inputs, "prompt", None),
|
||||
token_ids=getattr(inputs, "prompt_token_ids", None),
|
||||
embeds=getattr(inputs, "prompt_embeds", None),
|
||||
)
|
||||
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
@@ -915,14 +1003,9 @@ class OpenAIServing:
|
||||
) -> None:
|
||||
if self.request_logger is None:
|
||||
return
|
||||
prompt, prompt_token_ids, prompt_embeds = None, None, None
|
||||
if isinstance(inputs, str):
|
||||
prompt = inputs
|
||||
elif isinstance(inputs, list):
|
||||
prompt_token_ids = inputs
|
||||
else:
|
||||
prompt = getattr(inputs, 'prompt', None)
|
||||
prompt_token_ids = getattr(inputs, 'prompt_token_ids', None)
|
||||
|
||||
prompt, prompt_token_ids, prompt_embeds = (
|
||||
self._get_prompt_components(inputs))
|
||||
|
||||
self.request_logger.log_inputs(
|
||||
request_id,
|
||||
|
||||
Reference in New Issue
Block a user