diff --git a/tests/multimodal/assets/unsupported_8k_h264.mp4 b/tests/multimodal/assets/unsupported_8k_h264.mp4 new file mode 100644 index 0000000000..c81f83823e Binary files /dev/null and b/tests/multimodal/assets/unsupported_8k_h264.mp4 differ diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index b5d05301b5..8e27a5c58f 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -328,6 +328,129 @@ def test_pynvvideocodec_decoder_slots_are_configured_once( PyNvVideoCodecVideoBackend._configure_decoder_slots(3) +def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot(): + events: list[tuple[str, str]] = [] + + class FakeStream: + cuda_stream = "cuda-stream" + + class FakeDecoder: + poisoned = False + + def reconfigure_decoder(self, file_path: str): + self.poisoned = True + events.append(("reconfigure", file_path)) + raise RuntimeError("reconfigure failed") + + old_decoder = FakeDecoder() + slot = PyNvVideoCodecDecoderSlot(FakeStream()) + slot.decoder = old_decoder + slot.source_path = "valid.mp4" + + class FakeNvc: + class OutputColorType: + RGB = "rgb" + + @staticmethod + def SimpleDecoder(file_path: str, **kwargs): + events.append(("construct", file_path)) + assert slot.decoder is None + assert slot.source_path is None + raise RuntimeError("construct failed") + + old_slots = PyNvVideoCodecVideoBackend._decoder_slots + old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots + old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond + old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots + try: + PyNvVideoCodecVideoBackend._decoder_slots = [slot] + PyNvVideoCodecVideoBackend._active_decoder_slots = 1 + PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition() + PyNvVideoCodecVideoBackend._max_decoder_slots = 1 + + with ( + pytest.raises(RuntimeError, match="construct failed"), + PyNvVideoCodecVideoBackend._borrow_decoder_slot() as borrowed, + ): + assert borrowed is slot + borrowed.get_decoder( + "unsupported-8k.mp4", + FakeNvc, + device_index=0, + ) + + assert events == [ + ("reconfigure", "unsupported-8k.mp4"), + ("construct", "unsupported-8k.mp4"), + ] + assert old_decoder.poisoned + assert slot.decoder is None + assert slot.source_path is None + assert PyNvVideoCodecVideoBackend._decoder_slots == [slot] + finally: + PyNvVideoCodecVideoBackend._decoder_slots = old_slots + PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots + PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond + PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_pynvvideocodec_h200_recovers_after_unsupported_8k(): + import PyNvVideoCodec as nvc + import torch + + if "H200" not in torch.cuda.get_device_name(0): + pytest.skip("Requires H200 NVDEC resolution limits") + + valid_video = create_long_gop_video(num_frames=2, width=64, height=64) + unsupported_video = (ASSETS_DIR / "unsupported_8k_h264.mp4").read_bytes() + + old_slots = PyNvVideoCodecVideoBackend._decoder_slots + old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots + old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond + old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots + try: + PyNvVideoCodecVideoBackend._decoder_slots = [] + PyNvVideoCodecVideoBackend._active_decoder_slots = 0 + PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition() + PyNvVideoCodecVideoBackend._max_decoder_slots = None + + loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND) + frames_before, _ = loader.load_bytes( + valid_video, + num_frames=1, + hw_decoders=1, + ) + + with pytest.raises(Exception) as exc_info: + loader.load_bytes( + unsupported_video, + num_frames=1, + hw_decoders=1, + ) + + root_cause = exc_info.value + while root_cause.__cause__ is not None: + root_cause = root_cause.__cause__ + assert isinstance(root_cause, nvc.PyNvVCExceptionUnsupported) + assert "MBCount not supported" in str(root_cause) + + frames_after, _ = loader.load_bytes( + valid_video, + num_frames=1, + hw_decoders=1, + ) + + assert frames_after.shape == frames_before.shape + finally: + for slot in PyNvVideoCodecVideoBackend._decoder_slots: + slot.invalidate() + PyNvVideoCodecVideoBackend._decoder_slots = old_slots + PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots + PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond + PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots + + @pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"]) def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object): with pytest.raises(ValueError, match="hw_decoders must be a positive integer"): diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 4af65dea29..8d4d7090ed 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -261,8 +261,13 @@ class PyNvVideoCodecDecoderSlot: self.decoder = None self.source_path: str | None = None + def invalidate(self) -> None: + self.decoder = None + self.source_path = None + def _construct(self, file_path: str, nvc, device_index: int) -> None: - self.decoder = nvc.SimpleDecoder( + self.invalidate() + decoder = nvc.SimpleDecoder( file_path, output_color_type=nvc.OutputColorType.RGB, use_device_memory=True, @@ -271,6 +276,7 @@ class PyNvVideoCodecDecoderSlot: cuda_stream=self.stream.cuda_stream, decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE, ) + self.decoder = decoder self.source_path = file_path def get_decoder(self, file_path: str, nvc, device_index: int): @@ -747,9 +753,13 @@ class PyNvVideoCodecVideoBackendMixin: cls._decoder_slot_cond.notify() raise + borrow_succeeded = False try: yield slot + borrow_succeeded = True finally: + if not borrow_succeeded: + slot.invalidate() with cls._decoder_slot_cond: cls._decoder_slots.append(slot) cls._decoder_slot_cond.notify()