[Core] Set pooling params based on task and model (#21128)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
import asyncio
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import cached_property
|
||||
from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
|
||||
Union)
|
||||
|
||||
@@ -15,6 +16,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.pooling_params import PoolingTask
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sequence import ExecuteModelRequest, PoolerOutput
|
||||
from vllm.utils import make_async
|
||||
@@ -135,6 +137,11 @@ class ExecutorBase(ABC):
|
||||
|
||||
return self.collective_rpc(rpc_func)
|
||||
|
||||
@cached_property # Avoid unnecessary RPC calls
|
||||
def supported_pooling_tasks(self) -> tuple[PoolingTask, ...]:
|
||||
output = self.collective_rpc("get_supported_pooling_tasks")
|
||||
return tuple({task for tasks in output for task in tasks})
|
||||
|
||||
def execute_model(
|
||||
self, execute_model_req: ExecuteModelRequest
|
||||
) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
|
||||
|
||||
Reference in New Issue
Block a user