[Frontend] Support multimodal inputs for late-interaction scoring (ColQwen3) + NewModel: nvidia/nemotron-colembed (#34574)
Signed-off-by: craftsangjae <craftsangjae@gmail.com>
This commit is contained in:
+9
-20
@@ -50,6 +50,7 @@ from vllm.entrypoints.pooling.score.utils import (
|
||||
compress_token_type_ids,
|
||||
compute_maxsim_score,
|
||||
get_score_prompt,
|
||||
score_data_to_prompts,
|
||||
validate_score_input,
|
||||
)
|
||||
from vllm.entrypoints.utils import log_non_default_args
|
||||
@@ -1395,25 +1396,13 @@ class LLM:
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
# Extract text from ScoreData
|
||||
text_1: list[str] = []
|
||||
for text in data_1:
|
||||
if not isinstance(text, str):
|
||||
raise NotImplementedError(
|
||||
"Late interaction scores currently do not support multimodal input."
|
||||
)
|
||||
text_1.append(text)
|
||||
# Convert ScoreData to PromptType (handles both text and multimodal)
|
||||
model_config = self.model_config
|
||||
prompts_1 = score_data_to_prompts(data_1, "query", model_config)
|
||||
prompts_2 = score_data_to_prompts(data_2, "document", model_config)
|
||||
|
||||
text_2: list[str] = []
|
||||
for text in data_2:
|
||||
if not isinstance(text, str):
|
||||
raise NotImplementedError(
|
||||
"Late interaction scores currently do not support multimodal input."
|
||||
)
|
||||
text_2.append(text)
|
||||
|
||||
encoded_output = self.encode(
|
||||
text_1 + text_2,
|
||||
encoded_output: list[PoolingRequestOutput] = self.encode(
|
||||
prompts_1 + prompts_2,
|
||||
use_tqdm=use_tqdm,
|
||||
lora_request=lora_request,
|
||||
pooling_params=pooling_params,
|
||||
@@ -1421,8 +1410,8 @@ class LLM:
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
encoded_output_1 = encoded_output[0 : len(text_1)]
|
||||
encoded_output_2 = encoded_output[len(text_1) :]
|
||||
encoded_output_1: list[PoolingRequestOutput] = encoded_output[: len(prompts_1)]
|
||||
encoded_output_2: list[PoolingRequestOutput] = encoded_output[len(prompts_1) :]
|
||||
|
||||
if len(encoded_output_1) == 1:
|
||||
encoded_output_1 = encoded_output_1 * len(encoded_output_2)
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.entrypoints.pooling.score.utils import (
|
||||
compress_token_type_ids,
|
||||
compute_maxsim_score,
|
||||
get_score_prompt,
|
||||
parse_score_data_single,
|
||||
validate_score_input,
|
||||
)
|
||||
from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs
|
||||
@@ -174,6 +175,43 @@ class ServingScores(OpenAIServing):
|
||||
|
||||
return final_res_batch
|
||||
|
||||
def _preprocess_late_interaction_item(
|
||||
self,
|
||||
data: ScoreData,
|
||||
role: str,
|
||||
request: RerankRequest | ScoreRequest,
|
||||
tokenizer: TokenizerLike,
|
||||
tokenization_kwargs: dict[str, Any],
|
||||
) -> tuple[str, TokensPrompt]:
|
||||
"""Parse a single ScoreData into a text + optional multimodal
|
||||
TokensPrompt for late-interaction encoding.
|
||||
|
||||
For plain strings, tokenises directly.
|
||||
For multimodal content parts, extracts text and multi_modal_data.
|
||||
"""
|
||||
model_config = self.model_config
|
||||
|
||||
if isinstance(data, str):
|
||||
text, mm_data, mm_uuids = data, None, None
|
||||
else:
|
||||
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
||||
|
||||
prompt_inputs = tokenizer(text, **tokenization_kwargs)
|
||||
self._validate_input(request, prompt_inputs["input_ids"], text)
|
||||
|
||||
engine_prompt = TokensPrompt(
|
||||
prompt_token_ids=prompt_inputs["input_ids"],
|
||||
)
|
||||
|
||||
if mm_data is not None:
|
||||
engine_prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
engine_prompt["multi_modal_uuids"] = mm_uuids
|
||||
if request.mm_processor_kwargs is not None:
|
||||
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
|
||||
|
||||
return text, engine_prompt
|
||||
|
||||
async def _late_interaction_score(
|
||||
self,
|
||||
data_1: list[ScoreData],
|
||||
@@ -189,37 +227,36 @@ class ServingScores(OpenAIServing):
|
||||
Encodes queries and documents into per-token embeddings, then computes
|
||||
MaxSim: sum over query tokens of max similarity to any document token.
|
||||
"""
|
||||
input_texts: list[str] = []
|
||||
for text in data_1 + data_2:
|
||||
if not isinstance(text, str):
|
||||
raise NotImplementedError(
|
||||
"Late interaction scores currently do not support multimodal input."
|
||||
)
|
||||
input_texts.append(text)
|
||||
|
||||
model_config = self.model_config
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
tokenization_kwargs = request.build_tok_params(model_config).get_encode_kwargs()
|
||||
|
||||
encode_async = make_async(
|
||||
tokenizer.encode,
|
||||
all_data = data_1 + data_2
|
||||
roles = ["query"] * len(data_1) + ["document"] * len(data_2)
|
||||
|
||||
preprocess_async = make_async(
|
||||
self._preprocess_late_interaction_item,
|
||||
executor=self._tokenizer_executor,
|
||||
)
|
||||
|
||||
tokenization_kwargs = request.build_tok_params(model_config).get_encode_kwargs()
|
||||
tokenized_prompts = await asyncio.gather(
|
||||
*(encode_async(t, **tokenization_kwargs) for t in input_texts)
|
||||
preprocessed = await asyncio.gather(
|
||||
*(
|
||||
preprocess_async(
|
||||
data=d,
|
||||
role=r,
|
||||
request=request,
|
||||
tokenizer=tokenizer,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
for d, r in zip(all_data, roles)
|
||||
)
|
||||
)
|
||||
|
||||
engine_prompts: list[ProcessorInputs] = []
|
||||
for tok_result, input_text in zip(tokenized_prompts, input_texts):
|
||||
text_token_prompt = self._validate_input(request, tok_result, input_text)
|
||||
|
||||
engine_prompts.append(
|
||||
token_inputs(
|
||||
text_token_prompt["prompt_token_ids"],
|
||||
prompt=input_text,
|
||||
)
|
||||
)
|
||||
input_texts: list[str] = []
|
||||
engine_prompts: list[TokensPrompt] = []
|
||||
for text, engine_prompt in preprocessed:
|
||||
input_texts.append(text)
|
||||
engine_prompts.append(engine_prompt)
|
||||
|
||||
# Schedule the request and get the result generator.
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
_parse_chat_message_content_parts,
|
||||
)
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.inputs.data import PromptType, TextPrompt
|
||||
from vllm.model_executor.models.interfaces import supports_score_template
|
||||
from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
@@ -153,31 +154,91 @@ def validate_score_input(
|
||||
return score_input_1, score_input_2
|
||||
|
||||
|
||||
def _ensure_str(content: list[ConversationMessage]) -> str:
|
||||
"""Extract a single string prompt from parsed conversation content."""
|
||||
assert len(content) == 1
|
||||
prompt = content[0]["content"]
|
||||
if prompt is not None and isinstance(prompt, str):
|
||||
return cast(str, prompt)
|
||||
raise ValueError(f"Only string content is supported, but got {content}.")
|
||||
|
||||
|
||||
def parse_score_data(
|
||||
data_1: ScoreData,
|
||||
data_2: ScoreData,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
"""Parse a query-document pair into text prompts and shared multi-modal
|
||||
data.
|
||||
|
||||
Uses a **single** :class:`MultiModalItemTracker` so that multi-modal
|
||||
items from both inputs are merged into one ``mm_data`` dict. This is
|
||||
the correct behaviour for cross-encoder scoring, where query and
|
||||
document are concatenated into a single model prompt.
|
||||
"""
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
|
||||
content_1 = _parse_score_content("query", data_1, mm_tracker)
|
||||
content_2 = _parse_score_content("document", data_2, mm_tracker)
|
||||
|
||||
def ensure_str(content: list[ConversationMessage]) -> str:
|
||||
assert len(content) == 1
|
||||
prompt = content[0]["content"]
|
||||
if prompt is not None and isinstance(prompt, str):
|
||||
return cast(str, prompt)
|
||||
else:
|
||||
raise ValueError(f"Only string content is supported, but got {content}.")
|
||||
|
||||
prompt_1 = ensure_str(content_1)
|
||||
prompt_2 = ensure_str(content_2)
|
||||
prompt_1 = _ensure_str(content_1)
|
||||
prompt_2 = _ensure_str(content_2)
|
||||
mm_items, mm_uuids = mm_tracker.resolve_items()
|
||||
|
||||
return prompt_1, prompt_2, mm_items, mm_uuids
|
||||
|
||||
|
||||
def parse_score_data_single(
|
||||
data: ScoreData,
|
||||
role: str,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
"""Parse **one** ScoreData into a text prompt and its own multi-modal
|
||||
data.
|
||||
|
||||
Unlike :func:`parse_score_data`, each call creates an **independent**
|
||||
:class:`MultiModalItemTracker` so multi-modal items are kept separate.
|
||||
This is the correct behaviour for late-interaction scoring, where
|
||||
query and document are encoded independently.
|
||||
"""
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
content = _parse_score_content(role, data, mm_tracker)
|
||||
|
||||
prompt = _ensure_str(content)
|
||||
mm_items, mm_uuids = mm_tracker.resolve_items()
|
||||
return prompt, mm_items, mm_uuids
|
||||
|
||||
|
||||
def score_data_to_prompts(
|
||||
data_list: list[ScoreData],
|
||||
role: str,
|
||||
model_config: ModelConfig,
|
||||
) -> list[PromptType]:
|
||||
"""Convert a list of ScoreData into PromptType objects.
|
||||
|
||||
For plain text inputs, returns the string directly.
|
||||
For multimodal inputs (list of content parts), parses them into
|
||||
a :class:`TextPrompt` with attached ``multi_modal_data`` /
|
||||
``multi_modal_uuids``.
|
||||
|
||||
This is used by late-interaction scoring where each query/document
|
||||
is encoded independently.
|
||||
"""
|
||||
prompts: list[PromptType] = []
|
||||
for data in data_list:
|
||||
if isinstance(data, str):
|
||||
prompts.append(data)
|
||||
else:
|
||||
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
||||
prompt: TextPrompt = TextPrompt(prompt=text)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
prompts.append(prompt)
|
||||
return prompts
|
||||
|
||||
|
||||
def _parse_score_content(
|
||||
role: str,
|
||||
data: ScoreData,
|
||||
|
||||
@@ -16,6 +16,7 @@ Based on: Qwen3-VL backbone with custom text projection
|
||||
Target models:
|
||||
- TomoroAI/tomoro-colqwen3-embed-8b
|
||||
- OpenSearch-AI/Ops-Colqwen3-4B
|
||||
- nvidia/nemotron-colembed-vl-4b-v2
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
@@ -229,13 +230,14 @@ class ColQwen3Model(
|
||||
if not isinstance(hidden_states, torch.Tensor):
|
||||
return hidden_states # type: ignore
|
||||
|
||||
proj_dtype = self.custom_text_proj.weight.dtype # type: ignore
|
||||
if hidden_states.dtype != proj_dtype:
|
||||
hidden_states = hidden_states.to(proj_dtype)
|
||||
if self.custom_text_proj is not None:
|
||||
proj_dtype = self.custom_text_proj.weight.dtype
|
||||
if hidden_states.dtype != proj_dtype:
|
||||
hidden_states = hidden_states.to(proj_dtype)
|
||||
hidden_states = self.custom_text_proj(hidden_states)
|
||||
|
||||
# Project to embedding dimension and L2 normalize
|
||||
proj = self.custom_text_proj(hidden_states) # type: ignore
|
||||
return torch.nn.functional.normalize(proj, p=2, dim=-1)
|
||||
# L2 normalize
|
||||
return torch.nn.functional.normalize(hidden_states, p=2, dim=-1)
|
||||
|
||||
# Names used for the projection layer across different ColQwen3 variants
|
||||
_PROJ_LAYER_NAMES = {
|
||||
|
||||
@@ -256,6 +256,7 @@ _EMBEDDING_MODELS = {
|
||||
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
|
||||
"ColQwen3": ("colqwen3", "ColQwen3Model"),
|
||||
"OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
|
||||
"Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
|
||||
"SiglipModel": ("siglip", "SiglipEmbeddingModel"),
|
||||
# Technically Terratorch models work on images, both in
|
||||
# input and output. I am adding it here because it piggy-backs on embedding
|
||||
|
||||
@@ -76,6 +76,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
chatglm="ChatGLMConfig",
|
||||
colqwen3="ColQwen3Config",
|
||||
ops_colqwen3="OpsColQwen3Config",
|
||||
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
|
||||
deepseek_vl_v2="DeepseekVLV2Config",
|
||||
deepseek_v32="DeepseekV3Config",
|
||||
flex_olmo="FlexOlmoConfig",
|
||||
|
||||
Reference in New Issue
Block a user