[Frontend] Split the offline inference APIs and utils. (#43553)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
wang.yuqi
2026-05-26 05:20:24 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent f815c99954
commit d5cf7b4a2c
5 changed files with 642 additions and 713 deletions
+4 -68
View File
@@ -1,34 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Sequence
from collections.abc import Callable, Sequence
from typing import Any
from tqdm.auto import tqdm
from typing_extensions import TypeVar
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ChatTemplateConfig
from vllm.inputs import (
DataPrompt,
EngineInput,
PromptType,
)
from vllm.entrypoints.offline_utils import OfflineInferenceMixin
from vllm.inputs import DataPrompt, PromptType
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.outputs import (
ClassificationRequestOutput,
EmbeddingRequestOutput,
PoolingRequestOutput,
RequestOutput,
ScoringRequestOutput,
)
from vllm.pooling_params import PoolingParams
from vllm.renderers import BaseRenderer
from vllm.sampling_params import SamplingParams
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
from vllm.v1.engine.llm_engine import LLMEngine
from .factories import init_pooling_io_processors
from .scoring.io_processor import ScoringIOProcessor
@@ -37,20 +27,10 @@ from .typing import OfflineInputsContext, OfflineOutputsContext
logger = init_logger(__name__)
_P = TypeVar("_P", bound=SamplingParams | PoolingParams | None)
_O = TypeVar(
"_O",
bound=RequestOutput | PoolingRequestOutput,
default=RequestOutput | PoolingRequestOutput,
)
class PoolingOfflineMixin(ABC):
class PoolingOfflineMixin(OfflineInferenceMixin):
"""Offline inference for pooling models"""
renderer: BaseRenderer
llm_engine: "LLMEngine"
model_config: ModelConfig
runner_type: str
chat_template: str | None
supported_tasks: tuple[SupportedTask, ...]
@@ -464,47 +444,3 @@ class PoolingOfflineMixin(ABC):
)
return [ScoringRequestOutput.from_base(item) for item in outputs]
@abstractmethod
def _params_to_seq(
self,
params: _P | Sequence[_P],
num_requests: int,
) -> Sequence[_P]:
raise NotImplementedError
@abstractmethod
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
raise NotImplementedError
@abstractmethod
def _priority_to_seq(
self,
priority: list[int] | None,
num_requests: int,
) -> Sequence[int]:
raise NotImplementedError
@abstractmethod
def _render_and_add_requests(
self,
prompts: Iterable[EngineInput],
params: Sequence[SamplingParams | PoolingParams],
*,
lora_requests: Sequence[LoRARequest | None] | None = None,
priorities: Sequence[int] | None = None,
) -> list[str]:
raise NotImplementedError
@abstractmethod
def _run_engine(
self,
output_type: type[_O] | tuple[type[_O], ...],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
) -> list[_O]:
raise NotImplementedError