[Renderer] Move Processor out of LLMEngine (#26165)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -7,8 +7,7 @@ import traceback
|
||||
from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from http import HTTPStatus
|
||||
from typing import (Any, Callable, ClassVar, Generic, NamedTuple, Optional,
|
||||
TypeVar, Union)
|
||||
from typing import Any, Callable, ClassVar, Generic, Optional, TypeVar, Union
|
||||
|
||||
import torch
|
||||
from fastapi import Request
|
||||
@@ -69,6 +68,7 @@ from vllm.entrypoints.renderer import (BaseRenderer, CompletionRenderer,
|
||||
# yapf: enable
|
||||
from vllm.inputs.data import PromptType
|
||||
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
|
||||
from vllm.inputs.parse import PromptComponents, get_prompt_components
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -140,12 +140,6 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]:
|
||||
and "prompt_embeds" in prompt)
|
||||
|
||||
|
||||
class PromptComponents(NamedTuple):
|
||||
text: Optional[str] = None
|
||||
token_ids: Optional[list[int]] = None
|
||||
embeds: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
RequestT = TypeVar("RequestT", bound=AnyRequest)
|
||||
|
||||
|
||||
@@ -876,25 +870,23 @@ class OpenAIServing:
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: PromptType,
|
||||
sampling_params: SamplingParams,
|
||||
params: Union[SamplingParams, PoolingParams],
|
||||
*,
|
||||
lora_request: Optional[LoRARequest],
|
||||
trace_headers: Optional[Mapping[str, str]],
|
||||
priority: int,
|
||||
) -> tuple[EngineCoreRequest, dict[str, Any]]:
|
||||
"""
|
||||
using the Processor to process inputs for AsyncLLM
|
||||
"""
|
||||
"""Use the Processor to process inputs for AsyncLLM."""
|
||||
tokenization_kwargs: dict[str, Any] = {}
|
||||
_validate_truncation_size(self.max_model_len,
|
||||
sampling_params.truncate_prompt_tokens,
|
||||
params.truncate_prompt_tokens,
|
||||
tokenization_kwargs)
|
||||
|
||||
processor = await self._get_processor()
|
||||
engine_request = processor.process_inputs(
|
||||
request_id,
|
||||
engine_prompt,
|
||||
sampling_params,
|
||||
params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
@@ -973,25 +965,12 @@ class OpenAIServing:
|
||||
|
||||
def _get_prompt_components(
|
||||
self,
|
||||
inputs: Union[RequestPrompt, PromptType],
|
||||
prompt: Union[RequestPrompt, PromptType],
|
||||
) -> PromptComponents:
|
||||
if isinstance(inputs, str):
|
||||
return PromptComponents(text=inputs)
|
||||
if isinstance(inputs, list):
|
||||
return PromptComponents(token_ids=inputs)
|
||||
if isinstance(inputs, dict):
|
||||
return PromptComponents(
|
||||
text=inputs.get("prompt"), # type: ignore[arg-type]
|
||||
token_ids=inputs.get(
|
||||
"prompt_token_ids"), # type: ignore[arg-type]
|
||||
embeds=inputs.get("prompt_embeds"),
|
||||
)
|
||||
if isinstance(prompt, list):
|
||||
return PromptComponents(token_ids=prompt)
|
||||
|
||||
return PromptComponents(
|
||||
text=getattr(inputs, "prompt", None),
|
||||
token_ids=getattr(inputs, "prompt_token_ids", None),
|
||||
embeds=getattr(inputs, "prompt_embeds", None),
|
||||
)
|
||||
return get_prompt_components(prompt) # type: ignore[arg-type]
|
||||
|
||||
def _log_inputs(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user