[Bugfix] Fix ScoreMultiModalParam multi-document scoring returning single result (#33837)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
wang.yuqi
parent
07daee132b
commit
1f70313e59
@@ -1,6 +1,5 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import cast
|
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import transformers
|
import transformers
|
||||||
@@ -117,7 +116,7 @@ def _normalize_image(image_val: str) -> str:
|
|||||||
|
|
||||||
def create_score_multimodal_param(
|
def create_score_multimodal_param(
|
||||||
content_parts: list[dict],
|
content_parts: list[dict],
|
||||||
) -> ScoreMultiModalParam:
|
) -> list[ScoreMultiModalParam]:
|
||||||
"""
|
"""
|
||||||
Create a ScoreMultiModalParam from a list of content dictionaries.
|
Create a ScoreMultiModalParam from a list of content dictionaries.
|
||||||
|
|
||||||
@@ -152,7 +151,7 @@ def create_score_multimodal_param(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
return ScoreMultiModalParam(content=formatted_content)
|
return [ScoreMultiModalParam(content=[content]) for content in formatted_content]
|
||||||
|
|
||||||
|
|
||||||
def _run_vllm(
|
def _run_vllm(
|
||||||
@@ -198,23 +197,7 @@ def _run_hf(
|
|||||||
else:
|
else:
|
||||||
raise ValueError("Unsupported query format")
|
raise ValueError("Unsupported query format")
|
||||||
|
|
||||||
# Separate documents by type
|
scores: list[float] = []
|
||||||
text_docs: list[str] = []
|
|
||||||
image_docs: list[str] = []
|
|
||||||
text_indices: list[int] = []
|
|
||||||
image_indices: list[int] = []
|
|
||||||
|
|
||||||
for idx, doc in enumerate(document_strs):
|
|
||||||
if "text" in doc:
|
|
||||||
text_docs.append(doc["text"])
|
|
||||||
text_indices.append(idx)
|
|
||||||
elif "image" in doc:
|
|
||||||
image_docs.append(_normalize_image(doc["image"]))
|
|
||||||
image_indices.append(idx)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Unsupported document format at index {idx}")
|
|
||||||
|
|
||||||
scores: list[None | float] = [None] * len(document_strs)
|
|
||||||
|
|
||||||
with hf_runner(
|
with hf_runner(
|
||||||
model,
|
model,
|
||||||
@@ -223,30 +206,24 @@ def _run_hf(
|
|||||||
auto_cls=AutoModel,
|
auto_cls=AutoModel,
|
||||||
model_kwargs={"key_mapping": CHECKPOINT_TO_HF_MAPPER},
|
model_kwargs={"key_mapping": CHECKPOINT_TO_HF_MAPPER},
|
||||||
) as hf_model:
|
) as hf_model:
|
||||||
# Score text documents
|
for doc in document_strs:
|
||||||
if text_docs:
|
if "text" in doc:
|
||||||
text_scores = hf_model.model.compute_score(
|
score = hf_model.model.compute_score(
|
||||||
[[query_data, d] for d in text_docs],
|
[[query_data, doc["text"]]],
|
||||||
max_length=2048,
|
max_length=2048,
|
||||||
query_type=query_type,
|
query_type=query_type,
|
||||||
doc_type="text",
|
doc_type="text",
|
||||||
)
|
)
|
||||||
for i, s in zip(text_indices, text_scores):
|
scores.append(score)
|
||||||
scores[i] = s
|
elif "image" in doc:
|
||||||
|
score = hf_model.model.compute_score(
|
||||||
# Score image documents
|
[[query_data, doc["image"]]],
|
||||||
if image_docs:
|
max_length=2048,
|
||||||
image_scores = hf_model.model.compute_score(
|
query_type=query_type,
|
||||||
[[query_data, d] for d in image_docs],
|
doc_type="image",
|
||||||
max_length=2048,
|
)
|
||||||
query_type=query_type,
|
scores.append(score)
|
||||||
doc_type="image",
|
return scores
|
||||||
)
|
|
||||||
for i, s in zip(image_indices, image_scores):
|
|
||||||
scores[i] = s
|
|
||||||
|
|
||||||
assert all(s is not None for s in scores)
|
|
||||||
return cast(list[float], scores)
|
|
||||||
|
|
||||||
|
|
||||||
def _run_test(
|
def _run_test(
|
||||||
|
|||||||
Reference in New Issue
Block a user