[Bugfix][Multimodal] Parse decoded video frame lists as a single video (#55326)
Signed-off-by: waizuichougou <2082431897@qq.com> Signed-off-by: waizuichougou <105572299+waizuichougou@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7ef4d9bfed
commit
295ac4e52e
@@ -56,6 +56,23 @@ def test_frame_size_hwc_chw(frame):
|
|||||||
assert items.get_frame_size(0) == (W, H)
|
assert items.get_frame_size(0) == (W, H)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"frames",
|
||||||
|
[
|
||||||
|
[np.zeros((H, W, 3), dtype=np.uint8) for _ in range(2)],
|
||||||
|
[torch.zeros((H, W, 3), dtype=torch.uint8) for _ in range(2)],
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_parse_video_frame_list_as_single_video(frames):
|
||||||
|
"""A list of decoded frames must represent one video item."""
|
||||||
|
items = MultiModalDataParser().parse_mm_data({"video": frames})["video"]
|
||||||
|
|
||||||
|
assert items.get_count() == 1
|
||||||
|
video = items.get(0)
|
||||||
|
assert isinstance(video, np.ndarray)
|
||||||
|
np.testing.assert_array_equal(video, np.stack([np.asarray(f) for f in frames]))
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"modality,processor_cls",
|
"modality,processor_cls",
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -751,8 +751,14 @@ class MultiModalDataParser:
|
|||||||
return VideoEmbeddingItems(data, self.expected_hidden_size)
|
return VideoEmbeddingItems(data, self.expected_hidden_size)
|
||||||
|
|
||||||
data_items: list[VideoItem]
|
data_items: list[VideoItem]
|
||||||
if (is_list_of(data, PILImage.Image) and len(data) > 0) or (
|
if (
|
||||||
isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 4
|
(is_list_of(data, PILImage.Image) and len(data) > 0)
|
||||||
|
or (
|
||||||
|
is_list_of(data, (np.ndarray, torch.Tensor), check="all")
|
||||||
|
and len(data) > 0
|
||||||
|
and all(item.ndim == 3 for item in data)
|
||||||
|
)
|
||||||
|
or (isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 4)
|
||||||
):
|
):
|
||||||
data_items = [data]
|
data_items = [data]
|
||||||
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
elif isinstance(data, (np.ndarray, torch.Tensor)):
|
||||||
|
|||||||
Reference in New Issue
Block a user