[Model][1/N] Support multiple poolers at model level (#21227)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-07-21 02:22:21 -07:00
committed by GitHub
parent 378d33c392
commit 042af0c8d3
22 changed files with 549 additions and 413 deletions
+4 -12
View File
@@ -5,7 +5,7 @@ import copy
import gc
import time
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Optional, Union, cast, get_args
from typing import TYPE_CHECKING, Any, Optional, Union, cast
import numpy as np
import torch
@@ -415,15 +415,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
generator = None
if pooling_params:
assert pooling_params.task is not None, (
assert (task := pooling_params.task) is not None, (
"You did not set `task` in the API")
model = cast(VllmModelForPooling, self.model)
to_update = (model.pooler.get_pooling_updates(
pooling_params.task))
assert to_update is not None, (
f"{pooling_params.task=} is not supported by the model")
to_update = model.pooler.get_pooling_updates(task)
to_update.apply(pooling_params)
self.requests[req_id] = CachedRequestState(
@@ -1122,10 +1118,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if not is_pooling_model(model):
return []
return [
task for task in get_args(PoolingTask)
if model.pooler.get_pooling_updates(task)
]
return list(model.pooler.get_supported_tasks())
def apply_grammar_bitmask(
self,
@@ -2247,7 +2240,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dummy_pooling_params = PoolingParams(task=dummy_task)
to_update = model.pooler.get_pooling_updates(dummy_task)
assert to_update is not None
to_update.apply(dummy_pooling_params)
dummy_metadata = PoolingMetadata(