[Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,17 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Sequence
|
||||
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeAlias, TypeVar
|
||||
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.inputs import DataPrompt, EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.inputs import DictPrompt
|
||||
|
||||
from ...tasks import PoolingTask
|
||||
from .classify.protocol import (
|
||||
ClassificationChatRequest,
|
||||
ClassificationCompletionRequest,
|
||||
@@ -34,7 +38,7 @@ from .pooling.protocol import (
|
||||
PoolingResponse,
|
||||
)
|
||||
from .scoring.protocol import ScoringRequest, ScoringResponse
|
||||
from .scoring.typing import ScoringData
|
||||
from .scoring.typing import ScoreData, ScoringData
|
||||
|
||||
PoolingCompletionLikeRequest: TypeAlias = (
|
||||
EmbeddingCompletionRequest
|
||||
@@ -111,13 +115,36 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
|
||||
@dataclass
|
||||
class OfflineInputsContext:
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt | ScoringData
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
tokenization_kwargs: dict[str, Any] | None = None
|
||||
chat_template: str | None = None
|
||||
pooling_task: PoolingTask
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: Sequence[int] | None
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
|
||||
@dataclass
|
||||
class OfflineEncodeInputsContext(OfflineInputsContext):
|
||||
prompts: PromptType | Sequence[PromptType]
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineScoringInputsContext(OfflineInputsContext):
|
||||
scoring_data: ScoringData
|
||||
chat_template: str | None
|
||||
pooling_params: PoolingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflinePluginInputsContext(OfflineInputsContext):
|
||||
prompts: DataPrompt
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
ALLOfflineInputsContext: TypeAlias = (
|
||||
OfflineEncodeInputsContext
|
||||
| OfflineScoringInputsContext
|
||||
| OfflinePluginInputsContext
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -126,3 +153,41 @@ class OfflineOutputsContext:
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
|
||||
|
||||
class RenderParams(TypedDict):
|
||||
tok_params: TokenizeParams
|
||||
prompt_extras: dict[str, Any] | None
|
||||
skip_mm_cache: bool
|
||||
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
|
||||
|
||||
class EncodeCMPLRenderParams(RenderParams):
|
||||
prompts: DictPrompt
|
||||
|
||||
|
||||
class EncodeChatRenderParams(RenderParams):
|
||||
conversations: list["ChatCompletionMessageParam"]
|
||||
chat_params: ChatParams
|
||||
|
||||
|
||||
class ScoringRenderParams(RenderParams):
|
||||
data_1: ScoreData
|
||||
data_2: ScoreData
|
||||
chat_template: str | None
|
||||
|
||||
|
||||
class PoolingEngineInput(TypedDict):
|
||||
prompts: EngineInput
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
|
||||
|
||||
RequestGenerator: TypeAlias = Generator[
|
||||
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
|
||||
]
|
||||
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
|
||||
|
||||
Reference in New Issue
Block a user