From d3621c1eb3c4cb8d8a97e05d8c58d7f7571e34db Mon Sep 17 00:00:00 2001 From: Junpu Yu <109769073+davidjpyu@users.noreply.github.com> Date: Fri, 7 Aug 2026 21:49:06 -0700 Subject: [PATCH] [Bugfix][Multimodal] Fix PyNvVideoCodec video backend returning NCHW instead of NHWC (#51076) Signed-off-by: Junpu Yu --- tests/multimodal/media/test_video.py | 20 ++++++++++++++++++++ vllm/multimodal/video.py | 20 +++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index d417b07fa9..6ce20f6621 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -20,6 +20,7 @@ from vllm.multimodal.video import ( PYNVVIDEOCODEC_VIDEO_BACKEND, VIDEO_LOADER_REGISTRY, VideoLoader, + _pynvvc_frames_to_nhwc, ) from ..utils import cosine_similarity, create_video_from_image, normalize_image @@ -498,3 +499,22 @@ class TestMergeKwargsGpuBackendPolicy: def test_unknown_backend_not_treated_as_gpu(self): assert not VIDEO_LOADER_REGISTRY.backend_requires_gpu("totally_unknown") + + +@pytest.mark.parametrize("layout", ["nhwc", "nchw"]) +def test_pynvvc_frames_normalized_to_nhwc(layout: str): + """PyNvVideoCodec frame batches are normalized to NHWC regardless of the + per-frame layout the decoder emits (it has varied across versions), so the + HF video processors (which materialize a PIL image per frame) receive the + same NHWC shape as every other video backend.""" + torch = pytest.importorskip("torch") + + n, h, w, c = 4, 5, 6, 3 + nhwc = torch.arange(n * h * w * c, dtype=torch.uint8).reshape(n, h, w, c) + frames = nhwc if layout == "nhwc" else nhwc.permute(0, 3, 1, 2).contiguous() + + out = _pynvvc_frames_to_nhwc(frames) + + assert out.shape == (n, h, w, c) + assert out.is_contiguous() + assert torch.equal(out, nhwc) # content preserved / correctly transposed diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index d45e18d084..05c6daf3fc 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -637,6 +637,24 @@ class TorchCodecVideoBackendMixin: return batch.data.numpy(), list(frame_indices) +def _pynvvc_frames_to_nhwc(frames): + """Return a stacked PyNvVideoCodec frame batch as contiguous NHWC. + + PyNvVideoCodec's per-frame layout has varied across versions (HWC vs CHW), + so detect the channel axis rather than assuming a fixed order. NHWC is the + layout the other video backends return and the HF video processors expect. + + Args: + frames: A ``(N, ?, ?, ?)`` uint8 tensor in either NHWC or NCHW order. + + Returns: + The same frames as a contiguous ``(N, H, W, C)`` tensor. + """ + if frames.shape[-1] != 3 and frames.shape[-3] == 3: + frames = frames.permute(0, 2, 3, 1) # NCHW -> NHWC + return frames.contiguous() + + class PyNvVideoCodecVideoBackendMixin: """PyNvVideoCodec utilities for GPU-backed frame decode.""" @@ -807,7 +825,7 @@ class PyNvVideoCodecVideoBackendMixin: "PyNvVideoCodec returned frames with unexpected shape " f"{tuple(device_frames.shape)}" ) - device_frames = device_frames.permute(0, 3, 1, 2).contiguous() + device_frames = _pynvvc_frames_to_nhwc(device_frames) host_frames = torch.empty( device_frames.shape, dtype=device_frames.dtype,