[Bugfix][Multimodal] Fix PyNvVideoCodec video backend returning NCHW instead of NHWC (#51076)
Signed-off-by: Junpu Yu <davidyu@nvidia.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.multimodal.video import (
|
|||||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||||
VIDEO_LOADER_REGISTRY,
|
VIDEO_LOADER_REGISTRY,
|
||||||
VideoLoader,
|
VideoLoader,
|
||||||
|
_pynvvc_frames_to_nhwc,
|
||||||
)
|
)
|
||||||
|
|
||||||
from ..utils import cosine_similarity, create_video_from_image, normalize_image
|
from ..utils import cosine_similarity, create_video_from_image, normalize_image
|
||||||
@@ -498,3 +499,22 @@ class TestMergeKwargsGpuBackendPolicy:
|
|||||||
|
|
||||||
def test_unknown_backend_not_treated_as_gpu(self):
|
def test_unknown_backend_not_treated_as_gpu(self):
|
||||||
assert not VIDEO_LOADER_REGISTRY.backend_requires_gpu("totally_unknown")
|
assert not VIDEO_LOADER_REGISTRY.backend_requires_gpu("totally_unknown")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("layout", ["nhwc", "nchw"])
|
||||||
|
def test_pynvvc_frames_normalized_to_nhwc(layout: str):
|
||||||
|
"""PyNvVideoCodec frame batches are normalized to NHWC regardless of the
|
||||||
|
per-frame layout the decoder emits (it has varied across versions), so the
|
||||||
|
HF video processors (which materialize a PIL image per frame) receive the
|
||||||
|
same NHWC shape as every other video backend."""
|
||||||
|
torch = pytest.importorskip("torch")
|
||||||
|
|
||||||
|
n, h, w, c = 4, 5, 6, 3
|
||||||
|
nhwc = torch.arange(n * h * w * c, dtype=torch.uint8).reshape(n, h, w, c)
|
||||||
|
frames = nhwc if layout == "nhwc" else nhwc.permute(0, 3, 1, 2).contiguous()
|
||||||
|
|
||||||
|
out = _pynvvc_frames_to_nhwc(frames)
|
||||||
|
|
||||||
|
assert out.shape == (n, h, w, c)
|
||||||
|
assert out.is_contiguous()
|
||||||
|
assert torch.equal(out, nhwc) # content preserved / correctly transposed
|
||||||
|
|||||||
@@ -637,6 +637,24 @@ class TorchCodecVideoBackendMixin:
|
|||||||
return batch.data.numpy(), list(frame_indices)
|
return batch.data.numpy(), list(frame_indices)
|
||||||
|
|
||||||
|
|
||||||
|
def _pynvvc_frames_to_nhwc(frames):
|
||||||
|
"""Return a stacked PyNvVideoCodec frame batch as contiguous NHWC.
|
||||||
|
|
||||||
|
PyNvVideoCodec's per-frame layout has varied across versions (HWC vs CHW),
|
||||||
|
so detect the channel axis rather than assuming a fixed order. NHWC is the
|
||||||
|
layout the other video backends return and the HF video processors expect.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frames: A ``(N, ?, ?, ?)`` uint8 tensor in either NHWC or NCHW order.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The same frames as a contiguous ``(N, H, W, C)`` tensor.
|
||||||
|
"""
|
||||||
|
if frames.shape[-1] != 3 and frames.shape[-3] == 3:
|
||||||
|
frames = frames.permute(0, 2, 3, 1) # NCHW -> NHWC
|
||||||
|
return frames.contiguous()
|
||||||
|
|
||||||
|
|
||||||
class PyNvVideoCodecVideoBackendMixin:
|
class PyNvVideoCodecVideoBackendMixin:
|
||||||
"""PyNvVideoCodec utilities for GPU-backed frame decode."""
|
"""PyNvVideoCodec utilities for GPU-backed frame decode."""
|
||||||
|
|
||||||
@@ -807,7 +825,7 @@ class PyNvVideoCodecVideoBackendMixin:
|
|||||||
"PyNvVideoCodec returned frames with unexpected shape "
|
"PyNvVideoCodec returned frames with unexpected shape "
|
||||||
f"{tuple(device_frames.shape)}"
|
f"{tuple(device_frames.shape)}"
|
||||||
)
|
)
|
||||||
device_frames = device_frames.permute(0, 3, 1, 2).contiguous()
|
device_frames = _pynvvc_frames_to_nhwc(device_frames)
|
||||||
host_frames = torch.empty(
|
host_frames = torch.empty(
|
||||||
device_frames.shape,
|
device_frames.shape,
|
||||||
dtype=device_frames.dtype,
|
dtype=device_frames.dtype,
|
||||||
|
|||||||
Reference in New Issue
Block a user