[Frontend][3/n] Improve pooling entrypoints | scoring. (#28631)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,14 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
from collections.abc import AsyncGenerator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeAlias, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from vllm import PoolingRequestOutput
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.entrypoints.pooling.classify.protocol import (
|
||||
ClassificationChatRequest,
|
||||
ClassificationCompletionRequest,
|
||||
@@ -23,15 +23,13 @@ from vllm.entrypoints.pooling.embed.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.pooling.pooling.protocol import (
|
||||
IOProcessorRequest,
|
||||
PoolingBytesResponse,
|
||||
PoolingChatRequest,
|
||||
PoolingCompletionRequest,
|
||||
PoolingResponse,
|
||||
)
|
||||
from vllm.entrypoints.pooling.score.protocol import (
|
||||
RerankRequest,
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.protocol import ScoringRequest, ScoringResponse
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoringData
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
@@ -49,8 +47,7 @@ AnyPoolingRequest: TypeAlias = (
|
||||
PoolingCompletionLikeRequest
|
||||
| PoolingChatLikeRequest
|
||||
| IOProcessorRequest
|
||||
| RerankRequest
|
||||
| ScoreRequest
|
||||
| ScoringRequest
|
||||
| CohereEmbedRequest
|
||||
)
|
||||
|
||||
@@ -59,7 +56,8 @@ AnyPoolingResponse: TypeAlias = (
|
||||
| EmbeddingResponse
|
||||
| EmbeddingBytesResponse
|
||||
| PoolingResponse
|
||||
| ScoreResponse
|
||||
| PoolingBytesResponse
|
||||
| ScoringResponse
|
||||
)
|
||||
|
||||
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
|
||||
@@ -73,8 +71,8 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
request_id: str
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
lora_request: LoRARequest | None = None
|
||||
|
||||
engine_inputs: list[EngineInput] | None = None
|
||||
pooling_params: PoolingParams | list[PoolingParams] | None = None
|
||||
engine_inputs: Sequence[EngineInput] | None = None
|
||||
prompt_request_ids: list[str] | None = None
|
||||
intermediates: Any | None = None
|
||||
|
||||
@@ -84,3 +82,22 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
|
||||
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineInputsContext:
|
||||
prompts: PromptType | Sequence[PromptType] | ScoringData
|
||||
pooling_params: PoolingParams | list[PoolingParams] | None = None
|
||||
tokenization_kwargs: dict[str, Any] | None = None
|
||||
chat_template: str | None = None
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
offset: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineOutputsContext:
|
||||
outputs: list[PoolingRequestOutput]
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
offset: int | None = None
|
||||
|
||||
Reference in New Issue
Block a user