[Core] Set pooling params based on task and model (#21128)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import dataclasses
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type, Union
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces_base import VllmModelForPooling
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.multimodal import MultiModalKwargs
|
||||
from vllm.pooling_params import PoolingParams
|
||||
@@ -195,7 +196,20 @@ class PoolingModelRunner(
|
||||
seq_groups: List[Tuple[List[int], PoolingParams]] = []
|
||||
for i, seq_group_metadata in enumerate(seq_group_metadata_list):
|
||||
seq_ids = list(seq_group_metadata.seq_data.keys())
|
||||
|
||||
pooling_params = seq_group_metadata.pooling_params
|
||||
assert pooling_params is not None
|
||||
assert pooling_params.task is not None, (
|
||||
"You did not set `task` in the API")
|
||||
|
||||
to_update = (cast(VllmModelForPooling,
|
||||
self.model).pooler.get_pooling_updates(
|
||||
pooling_params.task))
|
||||
assert to_update is not None, (
|
||||
f"{pooling_params.task=} is not supported by the model")
|
||||
|
||||
to_update.apply(pooling_params)
|
||||
|
||||
seq_groups.append((seq_ids, pooling_params))
|
||||
|
||||
seq_data: Dict[int, SequenceData] = {}
|
||||
|
||||
Reference in New Issue
Block a user