[Model] Consolidate score logic by introduce score_type (#36479)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1584,8 +1584,11 @@ class LLM:
|
||||
)
|
||||
|
||||
supported_tasks = self.supported_tasks
|
||||
score_type = self.model_config.score_type
|
||||
is_late_interaction = score_type == "late-interaction"
|
||||
is_cross_encoder = score_type == "cross-encoder"
|
||||
|
||||
# Late interaction models (e.g., ColBERT) use token_embed for scoring
|
||||
is_late_interaction = model_config.is_late_interaction
|
||||
if not is_late_interaction and all(
|
||||
t not in supported_tasks for t in ("embed", "classify")
|
||||
):
|
||||
@@ -1595,13 +1598,10 @@ class LLM:
|
||||
"`--convert embed` or `--convert classify`."
|
||||
)
|
||||
|
||||
if (
|
||||
model_config.is_cross_encoder
|
||||
and getattr(model_config.hf_config, "num_labels", 0) != 1
|
||||
):
|
||||
if is_cross_encoder and getattr(model_config.hf_config, "num_labels", 0) != 1:
|
||||
raise ValueError("Score API is only enabled for num_labels == 1.")
|
||||
|
||||
if not model_config.is_cross_encoder and chat_template is not None:
|
||||
if not is_cross_encoder and chat_template is not None:
|
||||
raise ValueError(
|
||||
"chat_template is only supported for cross-encoder models."
|
||||
)
|
||||
@@ -1622,7 +1622,7 @@ class LLM:
|
||||
)
|
||||
encode_kwargs = tok_params.get_encode_kwargs()
|
||||
|
||||
if model_config.is_cross_encoder:
|
||||
if is_cross_encoder:
|
||||
return self._cross_encoding_score(
|
||||
score_data_1,
|
||||
score_data_2,
|
||||
|
||||
@@ -37,10 +37,10 @@ def register_pooling_api_routers(
|
||||
|
||||
app.include_router(embed_router)
|
||||
|
||||
# Score/rerank endpoints are available for:
|
||||
# - "score" task (cross-encoder models)
|
||||
# - "embed" task (bi-encoder models)
|
||||
# - "token_embed" task (late interaction models like ColBERT)
|
||||
# Score API handles score/rerank for:
|
||||
# - "score" task (score_type: cross-encoder models)
|
||||
# - "embed" task (score_type: bi-encoder models)
|
||||
# - "token_embed" task (score_type: late interaction models)
|
||||
if any(t in supported_tasks for t in ("score", "embed", "token_embed")):
|
||||
from vllm.entrypoints.pooling.score.api_router import router as score_router
|
||||
|
||||
@@ -101,10 +101,10 @@ def init_pooling_state(
|
||||
if "classify" in supported_tasks
|
||||
else None
|
||||
)
|
||||
# ServingScores handles score/rerank for:
|
||||
# - "score" task (cross-encoder models)
|
||||
# - "embed" task (bi-encoder models)
|
||||
# - "token_embed" task (late interaction models like ColBERT)
|
||||
# Score API handles score/rerank for:
|
||||
# - "score" task (score_type: cross-encoder models)
|
||||
# - "embed" task (score_type: bi-encoder models)
|
||||
# - "token_embed" task (score_type: late interaction models)
|
||||
state.serving_scores = (
|
||||
ServingScores(
|
||||
engine_client,
|
||||
|
||||
@@ -69,16 +69,15 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
self.is_cross_encoder = self.model_config.is_cross_encoder
|
||||
self.is_multimodal_model = self.model_config.is_multimodal_model
|
||||
self.score_type = self.model_config.score_type
|
||||
self.architecture = self.model_config.architecture
|
||||
self.is_late_interaction = self.model_config.is_late_interaction
|
||||
self.is_multimodal_model = self.model_config.is_multimodal_model
|
||||
|
||||
if self.is_cross_encoder:
|
||||
if self.score_type == "cross-encoder":
|
||||
self._score_func = self._cross_encoding_score
|
||||
elif self.is_late_interaction:
|
||||
elif self.score_type == "late-interaction":
|
||||
self._score_func = self._late_interaction_score
|
||||
else:
|
||||
else: # "bi-encoder"
|
||||
self._score_func = self._embedding_score
|
||||
|
||||
async def _embedding_score(
|
||||
|
||||
Reference in New Issue
Block a user