[Model][2/N] Automatic conversion of CrossEncoding model (#19978)

Signed-off-by: wang.yuqi <noooop@126.com>
This commit is contained in:
wang.yuqi
2025-07-03 13:59:23 +00:00
committed by GitHub
parent 1819fbda63
commit 6f1229f91d
16 changed files with 199 additions and 92 deletions
+7 -3
View File
@@ -1289,9 +1289,13 @@ class LLM:
raise ValueError(" ".join(messages))
if self.llm_engine.model_config.task not in ("embed", "score"):
raise ValueError(
"Score API is only enabled for `--task embed or --task score`")
if self.llm_engine.model_config.task not in ("embed", "classify"):
raise ValueError("Score API is only enabled for "
"`--task embed or --task classify`.")
if (self.llm_engine.model_config.task == "classify"
and self.llm_engine.model_config.hf_config.num_labels != 1):
raise ValueError("Score API is only enabled for num_labels == 1.")
# the tokenizer for models such as
# "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing
+11 -8
View File
@@ -1311,24 +1311,27 @@ async def init_app_state(
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
) if model_config.task == "embed" else None
state.openai_serving_scores = ServingScores(
engine_client,
model_config,
state.openai_serving_models,
request_logger=request_logger) if model_config.task in (
"score", "embed", "pooling") else None
state.openai_serving_classification = ServingClassification(
engine_client,
model_config,
state.openai_serving_models,
request_logger=request_logger,
) if model_config.task == "classify" else None
enable_serving_reranking = (model_config.task == "classify" and getattr(
model_config.hf_config, "num_labels", 0) == 1)
state.jinaai_serving_reranking = ServingScores(
engine_client,
model_config,
state.openai_serving_models,
request_logger=request_logger
) if model_config.task == "score" else None
request_logger=request_logger) if enable_serving_reranking else None
state.openai_serving_scores = ServingScores(
engine_client,
model_config,
state.openai_serving_models,
request_logger=request_logger) if (
model_config.task == "embed" or enable_serving_reranking) else None
state.openai_serving_tokenization = OpenAIServingTokenization(
engine_client,
model_config,
+5 -1
View File
@@ -357,12 +357,16 @@ async def main(args):
chat_template=None,
chat_template_content_format="auto",
) if model_config.task == "embed" else None
enable_serving_reranking = (model_config.task == "classify" and getattr(
model_config.hf_config, "num_labels", 0) == 1)
openai_serving_scores = (ServingScores(
engine,
model_config,
openai_serving_models,
request_logger=request_logger,
) if model_config.task == "score" else None)
) if (model_config.task == "embed" or enable_serving_reranking) else None)
tracker = BatchProgressTracker()
logger.info("Reading batch from %s...", args.input_file)