[Frontend] Split the offline inference APIs and utils. (#43553)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
f815c99954
commit
d5cf7b4a2c
@@ -1,34 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Any
|
||||
|
||||
from tqdm.auto import tqdm
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateConfig
|
||||
from vllm.inputs import (
|
||||
DataPrompt,
|
||||
EngineInput,
|
||||
PromptType,
|
||||
)
|
||||
from vllm.entrypoints.offline_utils import OfflineInferenceMixin
|
||||
from vllm.inputs import DataPrompt, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import (
|
||||
ClassificationRequestOutput,
|
||||
EmbeddingRequestOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
ScoringRequestOutput,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
from .factories import init_pooling_io_processors
|
||||
from .scoring.io_processor import ScoringIOProcessor
|
||||
@@ -37,20 +27,10 @@ from .typing import OfflineInputsContext, OfflineOutputsContext
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_P = TypeVar("_P", bound=SamplingParams | PoolingParams | None)
|
||||
_O = TypeVar(
|
||||
"_O",
|
||||
bound=RequestOutput | PoolingRequestOutput,
|
||||
default=RequestOutput | PoolingRequestOutput,
|
||||
)
|
||||
|
||||
|
||||
class PoolingOfflineMixin(ABC):
|
||||
class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
"""Offline inference for pooling models"""
|
||||
|
||||
renderer: BaseRenderer
|
||||
llm_engine: "LLMEngine"
|
||||
model_config: ModelConfig
|
||||
runner_type: str
|
||||
chat_template: str | None
|
||||
supported_tasks: tuple[SupportedTask, ...]
|
||||
@@ -464,47 +444,3 @@ class PoolingOfflineMixin(ABC):
|
||||
)
|
||||
|
||||
return [ScoringRequestOutput.from_base(item) for item in outputs]
|
||||
|
||||
@abstractmethod
|
||||
def _params_to_seq(
|
||||
self,
|
||||
params: _P | Sequence[_P],
|
||||
num_requests: int,
|
||||
) -> Sequence[_P]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _lora_request_to_seq(
|
||||
self,
|
||||
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
|
||||
num_requests: int,
|
||||
) -> Sequence[LoRARequest | None]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: list[int] | None,
|
||||
num_requests: int,
|
||||
) -> Sequence[int]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _render_and_add_requests(
|
||||
self,
|
||||
prompts: Iterable[EngineInput],
|
||||
params: Sequence[SamplingParams | PoolingParams],
|
||||
*,
|
||||
lora_requests: Sequence[LoRARequest | None] | None = None,
|
||||
priorities: Sequence[int] | None = None,
|
||||
) -> list[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _run_engine(
|
||||
self,
|
||||
output_type: type[_O] | tuple[type[_O], ...],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
) -> list[_O]:
|
||||
raise NotImplementedError
|
||||
|
||||
Reference in New Issue
Block a user