diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 0804f6f9bb..562e38109f 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -581,6 +581,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
| `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + IE+ | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
| `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
+| `LlavaOnevision2ForConditionalGeneration` | LLaVA-OneVision-2 | T + I+ + V+ | `lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct` | | |
| `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I+ + V+ | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
| `MiDashengLMModel` | MiDashengLM | T + A+ | `mispeech/midashenglm-7b` | | ✅︎ |
| `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + IE+ + VE+ + A+ | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ |
diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py
index f785a68f97..1ef39cfaa7 100644
--- a/tests/models/multimodal/processing/test_common.py
+++ b/tests/models/multimodal/processing/test_common.py
@@ -452,6 +452,15 @@ def test_processing_correctness(
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
+ if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
+ pytest.skip(
+ "LLaVA-OneVision-2 video processing routes frames through custom "
+ "video backends (qwen_vl_utils / codec) that require real encoded "
+ "video bytes and metadata. The synthetic numpy-array videos used by "
+ "this test yield empty video features, so the generic correctness "
+ "check cannot exercise the video path. Image processing is covered "
+ "by registration/inference tests."
+ )
_test_processing_correctness(
model_id,
diff --git a/tests/models/multimodal/processing/test_llava_onevision2.py b/tests/models/multimodal/processing/test_llava_onevision2.py
new file mode 100644
index 0000000000..817d9a4f67
--- /dev/null
+++ b/tests/models/multimodal/processing/test_llava_onevision2.py
@@ -0,0 +1,302 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism.
+
+The codec video backend cannot be exercised end-to-end in CI because it
+relies on the model's ``trust_remote_code`` package and real video bytes.
+These tests cover the pure-Python marker plumbing that wraps a video *path*
+for vLLM's ``MultiModalDataParser``:
+
+* :func:`prepare_codec_video_input` must encode a per-path hash into its
+ dummy ndarray so distinct codec videos do not collide on ``mm_hash``
+ (otherwise ``EncoderCacheManager`` would skip the encoder for every video
+ after the first).
+* :func:`_extract_codec_video_paths` must round-trip the marker back to the
+ original path(s) after the parser strips the metadata dict, and must return
+ ``None`` for non-codec inputs.
+"""
+
+import types
+
+import numpy as np
+import pytest
+
+from vllm.model_executor.models.llava_onevision2 import (
+ _CODEC_VIDEO_MARKER,
+ LlavaOnevision2VideoBackend,
+ _extract_codec_video_paths,
+ _frame_video_to_pil_and_timestamps,
+ _validate_video_source,
+ prepare_codec_video_input,
+)
+from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata
+
+
+def _model_config(local: str = "", domains=None):
+ # Minimal stand-in for vLLM's ModelConfig: the validator only reads
+ # ``allowed_local_media_path`` and ``allowed_media_domains``.
+ return types.SimpleNamespace(
+ allowed_local_media_path=local,
+ allowed_media_domains=domains,
+ )
+
+
+def test_prepare_codec_video_input_shape_and_marker():
+ dummy, meta = prepare_codec_video_input("/data/foo.mp4")
+
+ # 4-D ndarray satisfies MultiModalDataParser's video shape check.
+ assert isinstance(dummy, np.ndarray)
+ assert dummy.shape == (1, 1, 16, 3)
+ assert dummy.dtype == np.uint8
+
+ # Metadata carries the exact path under the codec marker key.
+ assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"}
+
+
+def test_prepare_codec_video_input_is_deterministic():
+ # Same path must yield identical dummy bytes (stable mm_hash).
+ a, _ = prepare_codec_video_input("/data/foo.mp4")
+ b, _ = prepare_codec_video_input("/data/foo.mp4")
+ assert a.tobytes() == b.tobytes()
+
+
+def test_prepare_codec_video_input_distinct_paths_distinct_bytes():
+ # Distinct paths must yield distinct dummy bytes so the parser-visible
+ # ndarray (the only part reaching MultiModalHasher) varies per video.
+ paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"]
+ payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths}
+ assert len(payloads) == len(paths)
+
+
+def test_extract_codec_video_paths_parser_list_shape():
+ # Parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
+ items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")]
+ assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"]
+
+
+def test_extract_codec_video_paths_single_raw_tuple():
+ # Single raw (ndarray, dict) tuple (pre-parser path) is also accepted.
+ item = prepare_codec_video_input("/x/solo.mp4")
+ assert _extract_codec_video_paths(item) == ["/x/solo.mp4"]
+
+
+def test_extract_codec_video_paths_non_codec_returns_none():
+ # Plain decoded-frame inputs (ndarray / list of ndarray) are not codec
+ # markers and must fall through to the frame backend.
+ plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
+ assert _extract_codec_video_paths(plain) is None
+ assert _extract_codec_video_paths([plain, plain]) is None
+ assert _extract_codec_video_paths([]) is None
+ # Tuple without the marker key is ignored.
+ assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None
+
+
+def test_extract_codec_video_paths_mixed_batch_returns_none():
+ # If any item in the batch lacks the marker, the whole batch is treated
+ # as non-codec (the backend does not mix codec and frame videos).
+ codec = prepare_codec_video_input("/x/1.mp4")
+ plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
+ assert _extract_codec_video_paths([codec, plain]) is None
+
+
+# ---------------------------------------------------------------------------
+# Media access controls (_validate_video_source)
+#
+# The codec backend keeps the raw path string alive past vLLM's
+# MultiModalDataParser and hands it to the trust-remote-code codec module,
+# which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the
+# codec backend is restricted to *local files* confined to
+# --allowed-local-media-path; remote http(s)/data URLs are rejected (they must
+# use the frame backend, which rides the connector). The validator also returns
+# the *resolved* path so the codec module opens exactly what was validated
+# (no symlink-retarget / validate-vs-open gap).
+# ---------------------------------------------------------------------------
+
+
+def test_validate_http_url_rejected():
+ # Codec backend is local-only: remote URLs bypass the connector's domain
+ # and redirect controls, so they are rejected regardless of allowlist.
+ with pytest.raises(ValueError):
+ _validate_video_source("http://example.com/v.mp4", _model_config())
+
+
+def test_validate_https_url_rejected_even_when_host_allowlisted():
+ with pytest.raises(ValueError):
+ _validate_video_source(
+ "https://good.com/v.mp4",
+ _model_config(domains=["good.com"]),
+ )
+
+
+def test_validate_data_url_rejected():
+ # data: URLs are a remote/inline source for the frame backend, not codec.
+ with pytest.raises(ValueError):
+ _validate_video_source("data:video/mp4;base64,AAAA", _model_config())
+
+
+def test_validate_local_file_blocked_without_allowed_path():
+ # Local file access is opt-in: without --allowed-local-media-path the
+ # bare path is rejected.
+ with pytest.raises(ValueError):
+ _validate_video_source("/data/v.mp4", _model_config())
+
+
+def test_validate_local_file_allowed_inside_allowed_dir(tmp_path):
+ f = tmp_path / "v.mp4"
+ f.touch()
+ assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f)
+
+
+def test_validate_local_file_traversal_blocked():
+ # Path traversal escaping the allowed root is rejected after resolution.
+ with pytest.raises(ValueError):
+ _validate_video_source(
+ "/tmp/ov2/../../etc/passwd",
+ _model_config(local="/tmp/ov2"),
+ )
+
+
+def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path):
+ f = tmp_path / "v.mp4"
+ f.touch()
+ assert _validate_video_source(
+ f.as_uri(), _model_config(local=str(tmp_path))
+ ) == str(f)
+
+
+def test_validate_file_scheme_percent_encoded_traversal_blocked():
+ # ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first
+ # (url2pathname) so the path cannot stay literally under the allowed root.
+ with pytest.raises(ValueError):
+ _validate_video_source(
+ "file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd",
+ _model_config(local="/tmp/ov2"),
+ )
+
+
+def test_validate_unsupported_scheme_blocked():
+ with pytest.raises(ValueError):
+ _validate_video_source("ftp://example.com/v.mp4", _model_config())
+
+
+def test_validate_returns_resolved_path_through_symlink(tmp_path):
+ # The validator resolves symlinks and returns the *real* path, so the codec
+ # module opens exactly what was validated (closes the validate-vs-open gap).
+ real = tmp_path / "real.mp4"
+ real.touch()
+ link = tmp_path / "link.mp4"
+ link.symlink_to(real)
+ assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str(
+ real
+ )
+
+
+def test_validate_relative_bare_path_blocked():
+ # Bare (scheme=="") paths come only from the codec backend and must be
+ # absolute: resolving a relative path against an ambiguous CWD before the
+ # confinement check is brittle/unsafe, so it is rejected outright.
+ with pytest.raises(ValueError):
+ _validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2"))
+
+
+# ---------------------------------------------------------------------------
+# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
+#
+# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
+# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
+# ``video_url`` inputs, or by the dummy-inputs builder during profiling
+# (``video_needs_metadata=True``). The helper materialises PIL frames and
+# per-frame timestamps (``frame_index / fps``), padding the frame count up to
+# the even temporal-merge boundary.
+# ---------------------------------------------------------------------------
+
+
+def test_frame_video_to_pil_and_timestamps_basic():
+ frames = np.zeros((4, 8, 8, 3), dtype=np.uint8)
+ metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]}
+ pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
+
+ assert len(pil_frames) == 4
+ assert all(f.size == (8, 8) for f in pil_frames)
+ # timestamps = frame_index / fps
+ assert timestamps == [0.0, 2.0, 4.0, 6.0]
+
+
+def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count():
+ # Odd frame count -> last frame repeated to satisfy temporal merge=2.
+ frames = np.zeros((3, 8, 8, 3), dtype=np.uint8)
+ metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]}
+ pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
+
+ assert len(pil_frames) == 4
+ assert len(timestamps) == 4
+ # The padded frame reuses the final index/timestamp.
+ assert timestamps == [0.0, 1.0, 2.0, 2.0]
+
+
+def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse():
+ # Missing frames_indices -> sequential range; missing/zero fps -> default.
+ frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
+ pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {}))
+
+ assert len(pil_frames) == 2
+ # Default fps is 1.0, indices fall back to range(T).
+ assert timestamps == [0.0, 1.0]
+
+
+def test_frame_video_to_pil_and_timestamps_rejects_non_tuple():
+ # Bare arrays (no metadata) must be rejected: the frame backend requires
+ # the (frames, metadata) tuple produced by the registered loader.
+ plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
+ with pytest.raises(ValueError):
+ _frame_video_to_pil_and_timestamps(plain)
+
+
+# ---------------------------------------------------------------------------
+# LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller
+# supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can
+# override the conservative defaults (fps=1.0, max_frames=32). Unset target
+# fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants.
+# ---------------------------------------------------------------------------
+
+
+def _src(total_frames: int, fps: float) -> VideoSourceMetadata:
+ return VideoSourceMetadata(
+ total_frames_num=total_frames,
+ original_fps=fps,
+ duration=total_frames / fps if fps > 0 else 0.0,
+ )
+
+
+def test_backend_defaults_cap_at_32_frames():
+ # 300 frames @ 1fps source, target unset -> capped at default max_frames=32.
+ src = _src(300, 1.0)
+ target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0)
+ idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
+
+ assert len(idx) == 32
+ assert idx[0] == 0
+ assert idx[-1] == 299
+ assert len(idx) % 2 == 0
+
+
+def test_backend_target_num_frames_overrides_default_cap():
+ # VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap.
+ src = _src(300, 1.0)
+ target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0)
+ idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
+
+ assert len(idx) == 128
+ assert idx[0] == 0
+ assert idx[-1] == 299
+
+
+def test_backend_target_fps_controls_sampling_when_below_cap():
+ # 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded).
+ src = _src(60, 30.0)
+ target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0)
+ idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
+
+ # fps-derived nframes (2) is below the 128 cap, so fps wins.
+ assert len(idx) <= 8
+ assert len(idx) % 2 == 0
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 87299d90e2..83848f97ce 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1048,6 +1048,17 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"LlavaNextVideoForConditionalGeneration": _HfExamplesInfo(
"llava-hf/LLaVA-NeXT-Video-7B-hf"
),
+ "LlavaOnevision2ForConditionalGeneration": _HfExamplesInfo(
+ "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct",
+ trust_remote_code=True,
+ # Keep the init/schema harness eager (these tests never run forward).
+ # NOTE: a separate H200-only hang exists in the core FA3 attention
+ # construction (after `FlashAttentionImpl.__init__` logs the FA3
+ # version, while building the Qwen3 backbone). enforce_eager does NOT
+ # gate that path, so this flag is hygiene, not the fix -- tracked with
+ # the maintainer.
+ enforce_eager=True,
+ ),
"LlavaOnevisionForConditionalGeneration": _HfExamplesInfo(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
),
diff --git a/vllm/model_executor/models/llava_onevision2.py b/vllm/model_executor/models/llava_onevision2.py
new file mode 100644
index 0000000000..552e2a9e8f
--- /dev/null
+++ b/vllm/model_executor/models/llava_onevision2.py
@@ -0,0 +1,2266 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Inference-only LLaVA-OneVision-2 (OV2) model for vLLM.
+
+Architecture notes:
+
+ * LLM backbone is plain Qwen3-8B with 1-D position_ids (no M-RoPE).
+ * Vision tower removes the CLS token (no class_embedding/class_pos_emb).
+ * Vision RoPE is 3-D (T:H:W) with a 4:6:6 head_dim split and uses
+ ``patch_positions`` instead of grid_thw to compute per-token freqs.
+ * ``rotate_half`` is *interleaved* (``(::2, 1::2)``) rather than
+ split-half.
+ * Vision attention uses windowed ``cu_seqlens`` (``frame_windows_size``
+ in T-dim); two backends implemented (SDPA + flash_attn varlen).
+ * ``patch_positions: [total_patches, 3]`` is plumbed as a first-class
+ MM kwarg alongside ``pixel_values`` / ``image_grid_thw``.
+ * Video frame-backend and codec-backend both alias to the image path
+ inside the HF processor, so the model implements a single visual
+ code path.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import importlib
+import json
+import os
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from functools import lru_cache
+from typing import (
+ Annotated,
+ Any,
+ Literal,
+)
+
+import numpy as np
+import regex as re
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from huggingface_hub import hf_hub_download
+from PIL import Image
+from transformers import AutoProcessor, AutoTokenizer, BatchFeature
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from transformers.models.qwen2_vl import Qwen2VLImageProcessor
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
+
+from vllm.compilation.decorators import (
+ should_torch_compile_mm_encoder,
+ support_torch_compile,
+)
+from vllm.config import VllmConfig
+from vllm.config.multimodal import BaseDummyOptions
+from vllm.distributed import parallel_state
+from vllm.distributed import utils as dist_utils
+from vllm.inputs import ModalityData, MultiModalDataDict
+from vllm.logger import init_logger
+from vllm.model_executor.layers.attention import MMEncoderAttention
+from vllm.model_executor.layers.linear import (
+ ColumnParallelLinear,
+ QKVParallelLinear,
+ RowParallelLinear,
+)
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.model_executor.models.interfaces import (
+ MultiModalEmbeddings,
+ SupportsMultiModal,
+ SupportsPP,
+)
+from vllm.model_executor.models.module_mapping import MultiModelKeys
+from vllm.model_executor.models.utils import (
+ AutoWeightsLoader,
+ WeightsMapper,
+ init_vllm_registered_model,
+ maybe_prefix,
+)
+from vllm.model_executor.models.utils import (
+ _merge_multimodal_embeddings as merge_multimodal_embeddings,
+)
+from vllm.model_executor.models.vision import get_vit_attn_backend
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import (
+ ImageItem,
+ MultiModalFieldConfig,
+ MultiModalKwargsItems,
+)
+from vllm.multimodal.parse import (
+ DictEmbeddingItems,
+ ImageSize,
+ ModalityDataItems,
+ MultiModalDataItems,
+ MultiModalDataParser,
+)
+from vllm.multimodal.processing import (
+ BaseMultiModalProcessor,
+ BaseProcessingInfo,
+ PromptReplacement,
+ PromptUpdate,
+ PromptUpdateDetails,
+)
+from vllm.multimodal.processing.dummy_inputs import BaseDummyInputsBuilder
+from vllm.multimodal.video import (
+ VIDEO_LOADER_REGISTRY,
+ VideoBackend,
+ VideoSourceMetadata,
+ VideoTargetMetadata,
+)
+from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.processor import _merge_mm_kwargs
+from vllm.transformers_utils.utils import convert_model_repo_to_path
+from vllm.utils.tensor_schema import TensorSchema, TensorShape
+
+logger = init_logger(__name__)
+
+
+@lru_cache
+def _load_ov2_processor(
+ model: str,
+ revision: str | None,
+ trust_remote_code: bool,
+ **kwargs: Any,
+):
+ # OV2's trust_remote_code processor is a bare class (not a ProcessorMixin),
+ # so the shared type-checked get_hf_processor rejects it. We also can't use
+ # AutoProcessor.from_pretrained: OV2's remote from_pretrained drops
+ # trust_remote_code before building its nested tokenizer, which makes that
+ # nested load fall back to an interactive stdin prompt that hangs in
+ # non-interactive CI. Instead, assemble the processor here with
+ # trust_remote_code threaded through every component explicitly.
+ path = convert_model_repo_to_path(model)
+ revision = revision or "main"
+
+ processor_cls = get_class_from_dynamic_module(
+ "processing_llava_onevision2.LlavaOnevision2Processor",
+ path,
+ revision=revision,
+ trust_remote_code=trust_remote_code,
+ )
+ video_processor_cls = get_class_from_dynamic_module(
+ "video_processing_llava_onevision2.LlavaOnevision2VideoProcessor",
+ path,
+ revision=revision,
+ trust_remote_code=trust_remote_code,
+ )
+
+ # Slow Qwen2VLImageProcessor mirrors the remote processor (the Fast variant
+ # has normalization rounding differences that change pixel_values).
+ image_processor = Qwen2VLImageProcessor.from_pretrained(
+ path, revision=revision, **kwargs
+ )
+ tokenizer = AutoTokenizer.from_pretrained(
+ path, revision=revision, trust_remote_code=trust_remote_code, **kwargs
+ )
+ video_processor = video_processor_cls(
+ image_processor=image_processor,
+ min_pixels=getattr(image_processor, "min_pixels", 256 * 28 * 28),
+ max_pixels=getattr(image_processor, "max_pixels", 1605632),
+ patch_size=getattr(image_processor, "patch_size", 14),
+ spatial_merge_size=getattr(image_processor, "merge_size", 2),
+ )
+
+ # Codec defaults live under the "codec" key of preprocessor_config.json,
+ # which Qwen2VLImageProcessor does not preserve; read them directly so the
+ # codec video backend keeps its configured defaults.
+ codec_config: dict = {}
+ try:
+ config_file = os.path.join(path, "preprocessor_config.json")
+ if not os.path.isfile(config_file):
+ config_file = hf_hub_download(
+ path, "preprocessor_config.json", revision=revision
+ )
+ with open(config_file, encoding="utf-8") as f:
+ codec_config = json.load(f).get("codec", {}) or {}
+ except Exception:
+ logger.debug("OV2: no codec defaults found in preprocessor_config.json")
+
+ return processor_cls(
+ image_processor=image_processor,
+ tokenizer=tokenizer,
+ video_processor=video_processor,
+ codec_config=codec_config,
+ )
+
+
+# Upper bound on frames used when profiling the worst-case video item, mirroring
+# Qwen2-VL. The real frame count is decided by the HF VideoProcessor at apply
+# time; this only sizes the memory-profiling estimate.
+_MAX_FRAMES_PER_VIDEO = 14
+
+
+def _pack_timestamps(per_video: list[list[float]]) -> torch.Tensor:
+ if not per_video:
+ return torch.empty((0, 0), dtype=torch.float32)
+ t_max = max((len(ts) for ts in per_video), default=0)
+ padded = torch.zeros((len(per_video), t_max), dtype=torch.float32)
+ for i, ts in enumerate(per_video):
+ padded[i, : len(ts)] = torch.tensor(ts, dtype=torch.float32)
+ return padded
+
+
+def _validate_video_source(path: str, model_config) -> str:
+ """Confine a codec video path to ``--allowed-local-media-path``.
+
+ The codec backend keeps the raw path string alive past vLLM's
+ ``MultiModalDataParser`` and hands it to the trust-remote-code codec
+ module, which opens it directly via ``cv2.VideoCapture`` / ffmpeg. That
+ bypasses both ``MediaConnector``'s access controls and its redirect
+ handling (``VLLM_MEDIA_URL_ALLOW_REDIRECTS``), so we restrict the codec
+ backend to **local files only**: remote ``http(s)`` / ``data`` URLs are
+ rejected here and must instead go through the frame backend (a registered
+ ``VIDEO_LOADER_REGISTRY`` loader), which rides vLLM's connector and its
+ domain/redirect gates.
+
+ Returns the *resolved* absolute path so the codec module opens exactly the
+ file that was validated, closing the validate-vs-open (symlink-retarget)
+ window. Mirrors the confinement in ``MediaConnector._load_file_url``.
+ """
+ from pathlib import Path
+ from urllib.request import url2pathname
+
+ from urllib3.util import parse_url
+
+ allowed_local = getattr(model_config, "allowed_local_media_path", "") or ""
+
+ parsed = parse_url(str(path))
+ scheme = (parsed.scheme or "").lower()
+
+ if scheme in ("http", "https", "data"):
+ raise ValueError(
+ f"The codec video backend does not support remote {scheme!r} URLs: "
+ f"its trust-remote-code decoder fetches them outside vLLM's domain "
+ f"and redirect controls. Use a local file path, or the frame "
+ f"backend for remote videos."
+ )
+ if scheme not in ("", "file"):
+ raise ValueError(
+ f"Unsupported codec video URL scheme {scheme!r}; only local file "
+ f"paths or file:// URLs are supported."
+ )
+
+ # Local file access is opt-in: require --allowed-local-media-path and
+ # confine the resolved path to that directory (connector.py:253-271).
+ if not allowed_local:
+ raise ValueError(
+ "Local video file access is disabled. Set "
+ "--allowed-local-media-path to enable reading local videos."
+ )
+ if scheme == "file":
+ # Decode percent-encoding (mirrors MediaConnector._load_file_url),
+ # including the netloc so file://host/path is handled identically.
+ local = Path(url2pathname((parsed.netloc or "") + (parsed.path or "")))
+ else:
+ local = Path(str(path))
+ # Require an absolute path: resolving a relative path against an ambiguous
+ # CWD before the confinement check is brittle/unsafe.
+ if not local.is_absolute():
+ raise ValueError(
+ f"Local video path {str(path)!r} must be absolute; "
+ f"relative paths are not supported."
+ )
+ allowed_root = Path(allowed_local).resolve()
+ resolved = local.resolve()
+ if resolved != allowed_root and allowed_root not in resolved.parents:
+ raise ValueError(
+ f"Video path {str(path)!r} is outside the allowed local media "
+ f"directory {allowed_local!r}."
+ )
+ return str(resolved)
+
+
+def _validate_video_sources(paths, model_config) -> list[str]:
+ # Return the resolved paths so the codec module opens exactly what was
+ # validated (no validate-vs-open / symlink-retarget differential).
+ return [_validate_video_source(path, model_config) for path in paths]
+
+
+# Design note: the two video backends take deliberately different paths.
+#
+# * frame backend: a normal vLLM VIDEO_LOADER_REGISTRY loader
+# (``LlavaOnevision2VideoBackend`` below). Decoding to RGB
+# ``(frames, metadata)`` is exactly what loaders are for, so frame sampling
+# participates in the standard decode-stage pipeline.
+#
+# * codec backend: NOT a loader. OV2's codec path needs the video path string
+# to survive into ``_call_hf_processor``, where the HF processor builds the
+# codec canvas + smart_resize + patchify
+# (pixel_values/image_grid_thw/patch_positions). That transform is
+# path-level and inseparable; it cannot be reconstructed from pre-decoded
+# RGB frames, so it must run at the processor stage rather than the decode
+# stage. The small marker/parser machinery below keeps the path alive for
+# codec; ``_validate_video_source`` then confines it to a local file (codec
+# does not support remote URLs -- see its docstring).
+_CODEC_VIDEO_MARKER = "ov2_codec_video"
+
+
+def prepare_codec_video_input(video_path: str) -> tuple:
+ """Wrap a video path for vLLM's MultiModalDataParser + OV2 codec backend.
+
+ Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
+ parser's 4-D shape check and the metadata carries the actual path to
+ our ``_call_hf_processor``. Use as::
+
+ multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
+
+ The dummy ndarray bytes encode a hash of ``video_path`` so distinct codec
+ videos get distinct mm_hashes: the parser drops the metadata dict before
+ hashing (only the ndarray reaches MultiModalHasher), so without this
+ variance every video after the first would collide and skip the encoder.
+ """
+ path_str = str(video_path)
+ digest = hashlib.blake2b(path_str.encode("utf-8"), digest_size=16).digest()
+ dummy = np.frombuffer(digest, dtype=np.uint8).reshape(1, 1, 16, 1)
+ dummy = np.broadcast_to(dummy, (1, 1, 16, 3)).copy()
+ return (dummy, {_CODEC_VIDEO_MARKER: str(video_path)})
+
+
+def _extract_codec_video_paths(videos: Any) -> list[str] | None:
+ # vLLM's parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
+ # We accept either that shape or a single raw tuple (pre-parser cases).
+ def _path_from(item):
+ if (
+ isinstance(item, tuple)
+ and len(item) == 2
+ and isinstance(item[1], dict)
+ and _CODEC_VIDEO_MARKER in item[1]
+ ):
+ return item[1][_CODEC_VIDEO_MARKER]
+ return None
+
+ if isinstance(videos, list):
+ paths: list[str] = []
+ for item in videos:
+ p = _path_from(item)
+ if p is None:
+ return None
+ paths.append(p)
+ return paths if paths else None
+ p = _path_from(videos)
+ return [p] if p is not None else None
+
+
+_CODEC_FPS_CACHE: dict[str, float] = {}
+
+
+def _codec_fps_for(video_path: str, hf_processor) -> float:
+ if video_path in _CODEC_FPS_CACHE:
+ return _CODEC_FPS_CACHE[video_path]
+ # The codec module is shipped inside the HF transformers_modules package
+ # for OV2, so an absolute import does not resolve. Locate it relative to
+ # the processor module (which lives in the same package).
+ proc_module_name = type(hf_processor).__module__
+ pkg = proc_module_name.rsplit(".", 1)[0] if "." in proc_module_name else ""
+ codec_mod = importlib.import_module(
+ f"{pkg}.codec_video_processing_llava_onevision2"
+ if pkg
+ else "codec_video_processing_llava_onevision2"
+ )
+ CodecConfig = codec_mod.CodecConfig
+ process_codec_video = codec_mod.process_codec_video
+ cfg_defaults = dict(getattr(hf_processor, "_codec_config_defaults", {}))
+ cfg = CodecConfig(**cfg_defaults)
+ payload = process_codec_video(video_path, cfg)
+ fps = float(payload["fps"])
+ _CODEC_FPS_CACHE[video_path] = fps
+ return fps
+
+
+def _codec_timestamp_runs(
+ patch_positions: torch.Tensor,
+ fps: float,
+ spatial_merge_size: int,
+) -> list[tuple[float, int]]:
+ # Mirrors HF's _timestamp_runs (codec_video_processing_llava_onevision2.py)
+ # exactly: same column, same merge factor, same negative-t skip, same
+ # zero-token-count skip. Keeping the logic local avoids importing the
+ # private helper from transformers_modules.
+ t_values = patch_positions[:, 0]
+ unique_t, counts = torch.unique_consecutive(t_values, return_counts=True)
+ merge_factor = int(spatial_merge_size) ** 2
+ runs: list[tuple[float, int]] = []
+ for t_val, count in zip(unique_t.tolist(), counts.tolist()):
+ if int(t_val) < 0:
+ continue
+ token_count = int(count) // merge_factor
+ if token_count <= 0:
+ continue
+ runs.append((float(t_val) / float(fps), token_count))
+ return runs
+
+
+def _create_field_factory(
+ spatial_merge_size: int,
+) -> Callable[[Mapping[str, torch.Tensor]], Mapping[str, MultiModalFieldConfig]]:
+ """Build the per-batch field-config callback.
+
+ OV2-specific: also exposes ``patch_positions`` as a flat-from-sizes
+ field, sized by the total per-image patch count (T*H*W). The merger and
+ the 3-D RoPE both consume it.
+ """
+
+ def _field_config(hf_inputs: Mapping[str, torch.Tensor]):
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
+ image_pixel_grid_sizes = image_grid_thw.prod(-1)
+ image_embed_grid_sizes = (
+ image_pixel_grid_sizes // spatial_merge_size // spatial_merge_size
+ )
+
+ video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
+ # OV2 emits one grid_thw row per frame, so vLLM's per-video sharding
+ # requires explicit frame counts. video_patch_sizes sums H*W over the
+ # frames that belong to each video; video_grid_thw uses the frame
+ # count directly (one row per frame).
+ video_num_frames = hf_inputs.get(
+ "video_num_frames", torch.empty((0,), dtype=torch.long)
+ )
+ if video_num_frames.numel() > 0:
+ per_row_patches = video_grid_thw.prod(-1)
+ offsets = torch.cumsum(
+ torch.cat([torch.zeros(1, dtype=torch.long), video_num_frames[:-1]]), 0
+ ).tolist()
+ video_patch_sizes = torch.tensor(
+ [
+ int(per_row_patches[int(s) : int(s) + int(n)].sum())
+ for s, n in zip(offsets, video_num_frames.tolist())
+ ],
+ dtype=torch.long,
+ )
+ else:
+ video_patch_sizes = torch.empty((0,), dtype=torch.long)
+
+ return dict(
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
+ "image", image_pixel_grid_sizes
+ ),
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
+ "image", image_embed_grid_sizes
+ ),
+ image_grid_thw=MultiModalFieldConfig.batched("image"),
+ # OV2 first-class MM kwarg: per-patch (t,h,w)
+ # positions required by the 3-D vision RoPE.
+ patch_positions=MultiModalFieldConfig.flat_from_sizes(
+ "image", image_pixel_grid_sizes
+ ),
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
+ "video", video_patch_sizes
+ ),
+ video_grid_thw=MultiModalFieldConfig.flat_from_sizes(
+ "video", video_num_frames
+ ),
+ patch_positions_videos=MultiModalFieldConfig.flat_from_sizes(
+ "video", video_patch_sizes
+ ),
+ video_num_frames=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
+ frame_timestamps=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
+ # Per-video flag: 0 = frame-sampling backend, 1 = codec backend.
+ # Drives codec-aware ``\n`` insertion in ``get_video_replacement``.
+ video_is_codec=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
+ # Codec backend: per-video source-frame fps. Needed at
+ # replacement time to convert patch_positions t-indices into
+ # the timestamp tags HF writes (````).
+ codec_fps=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
+ )
+
+ return _field_config
+
+
+# ---------------------------------------------------------------------------
+# Frame backend helpers
+# ---------------------------------------------------------------------------
+# The default video pathway materialises each video as a series of PIL frames
+# (decoded + sampled by the registered ``LlavaOnevision2VideoBackend``) and
+# feeds them to the HF processor through the *image* branch (per-frame timestamp
+# marker + ``<|image_pad|>``). This mirrors the validated lmms-eval
+# ``vllm_hf_chat`` adapter, and empirically scores higher on Video-MME than
+# OV2's native VideoProcessor frame extractor.
+_DEFAULT_TIMESTAMP_DECIMALS = 1
+_DEFAULT_FPS = 1.0
+_DEFAULT_MAX_FRAMES = 32
+# OV2 vision tower has spatial_merge_size=2 -> temporal frame count must be
+# even. The hf-chat reference pads by repeating the last frame; same here.
+_TEMPORAL_MERGE_SIZE = 2
+
+# Token sequence emitted by the OV2 vLLM dummy inputs builder for each video
+# item. The frame backend expands each marker into per-frame image markers
+# (timestamp + image_pad block).
+_VIDEO_MARKER = "<|vision_start|><|video_pad|><|vision_end|>"
+_IMAGE_MARKER = "<|vision_start|><|image_pad|><|vision_end|>"
+
+
+def _frame_video_to_pil_and_timestamps(
+ item: Any,
+) -> tuple[list[Image.Image], list[float]]:
+ """Convert a ``(frames_ndarray, metadata)`` video item into
+ ``(pil_frames, timestamps_seconds)``.
+
+ Both real ``video_url`` inputs (decoded + sampled by the registered
+ ``LlavaOnevision2VideoBackend``) and dummy profiling videos arrive here as a
+ ``(frames, metadata)`` tuple because the data parser runs with
+ ``video_needs_metadata=True``. ``frames`` is a ``(T, H, W, C)`` uint8 array;
+ ``metadata`` carries ``frames_indices`` and the source ``fps``.
+
+ Timestamps follow the qwen_vl_utils policy: ``frame_index / original_fps``.
+ The frame count is padded up to ``_TEMPORAL_MERGE_SIZE`` (repeating the last
+ frame) because OV2's vision tower merges frames temporally in pairs.
+ """
+ if not (isinstance(item, (tuple, list)) and len(item) == 2):
+ raise ValueError(
+ "LlavaOnevision2 frame backend expects each video as a "
+ f"(frames_ndarray, metadata) tuple; got {type(item).__name__}. "
+ "Pass videos via `video_url` so the registered backend can decode "
+ "and sample them."
+ )
+ frames, metadata = item
+ if isinstance(frames, torch.Tensor):
+ frames_np = frames.cpu().numpy()
+ else:
+ frames_np = np.asarray(frames)
+
+ pil_frames = [Image.fromarray(f.astype(np.uint8)) for f in frames_np]
+
+ indices = metadata.get("frames_indices") if isinstance(metadata, Mapping) else None
+ if indices is None:
+ indices = list(range(len(pil_frames)))
+ elif not isinstance(indices, list):
+ indices = list(indices)
+ # Keep indices aligned with the actual frame count.
+ if len(indices) != len(pil_frames):
+ if len(indices) > len(pil_frames):
+ indices = indices[: len(pil_frames)]
+ else:
+ indices = list(indices) + [indices[-1] if indices else 0] * (
+ len(pil_frames) - len(indices)
+ )
+
+ fps = _DEFAULT_FPS
+ if isinstance(metadata, Mapping) and metadata.get("fps"):
+ fps = float(metadata["fps"])
+ if fps <= 0:
+ fps = _DEFAULT_FPS
+
+ # OV2 vision tower: temporal merge=2 -> frame count must be even.
+ if len(pil_frames) % _TEMPORAL_MERGE_SIZE != 0:
+ pad = _TEMPORAL_MERGE_SIZE - len(pil_frames) % _TEMPORAL_MERGE_SIZE
+ pil_frames = pil_frames + [pil_frames[-1]] * pad
+ indices = indices + [indices[-1]] * pad
+
+ timestamps = [idx / fps for idx in indices]
+ return pil_frames, timestamps
+
+
+def _expand_video_markers_in_prompt(
+ prompt: str,
+ per_video_timestamps: list[list[float]],
+ *,
+ timestamp_decimals: int,
+) -> str:
+ """Replace each ``<|vision_start|><|video_pad|><|vision_end|>`` with a
+ sequence of ``<{t:.Nf} seconds><|vision_start|><|image_pad|><|vision_end|>``
+ blocks -- one per frame -- matching ``vllm_hf_chat._build_prompt``.
+
+ Replacement is positional: the *i*-th marker consumes
+ ``per_video_timestamps[i]``.
+ """
+ parts: list[str] = []
+ cursor = 0
+ idx = 0
+ pattern = re.escape(_VIDEO_MARKER)
+ for m in re.finditer(pattern, prompt):
+ parts.append(prompt[cursor : m.start()])
+ if idx >= len(per_video_timestamps):
+ raise ValueError(
+ f"Prompt has more video markers than supplied timestamp "
+ f"groups ({len(per_video_timestamps)})"
+ )
+ timestamps = per_video_timestamps[idx]
+ expanded = "".join(
+ f"<{t:.{timestamp_decimals}f} seconds>{_IMAGE_MARKER}" for t in timestamps
+ )
+ parts.append(expanded)
+ cursor = m.end()
+ idx += 1
+ parts.append(prompt[cursor:])
+ if idx != len(per_video_timestamps):
+ raise ValueError(
+ f"Prompt has {idx} video markers but {len(per_video_timestamps)} "
+ f"timestamp groups were supplied"
+ )
+ return "".join(parts)
+
+
+# ---------------------------------------------------------------------------
+# Registered video loader backend (frame sampling)
+# ---------------------------------------------------------------------------
+# OV2 frame-sampling policy, expressed as a vLLM ``VideoBackend`` so videos
+# entering through the standard ``video_url`` -> MediaConnector -> VideoMediaIO
+# path are decoded + sampled inside vLLM (no qwen_vl_utils dependency, and the
+# connector's SSRF / local-file gates apply automatically).
+#
+# Parity note: ``compute_frames_index_to_sample`` replicates
+# ``qwen_vl_utils.smart_nframes`` (the policy the OV2 hf-chat recipe validated)
+# -- frame count and index selection are byte-identical to qwen. The one
+# residual difference is the source frame *count*: qwen decodes via decord
+# whereas vLLM uses OpenCV/PyAV, whose ``CAP_PROP_FRAME_COUNT`` (a duration x
+# fps estimate) can differ by +/-1 frame on some containers, shifting sampled
+# indices by one frame on those files. Downstream metrics stay within noise.
+_OV2_FRAME_FACTOR = 2
+_OV2_FPS_MIN_FRAMES = 4
+
+
+def _round_by_factor(n: float, factor: int) -> int:
+ return round(n / factor) * factor
+
+
+def _ceil_by_factor(n: float, factor: int) -> int:
+ import math as _math
+
+ return _math.ceil(n / factor) * factor
+
+
+def _floor_by_factor(n: float, factor: int) -> int:
+ import math as _math
+
+ return _math.floor(n / factor) * factor
+
+
+def _ov2_smart_nframes(
+ total_frames: int,
+ video_fps: float,
+ *,
+ fps: float,
+ min_frames: int,
+ max_frames: int,
+) -> int:
+ """Replicate ``qwen_vl_utils.smart_nframes`` (fps branch).
+
+ Returns an even frame count in ``[min_frames, min(max_frames, total)]``.
+ """
+ min_frames = _ceil_by_factor(min_frames, _OV2_FRAME_FACTOR)
+ max_frames = _floor_by_factor(max_frames, _OV2_FRAME_FACTOR)
+ nframes = total_frames / video_fps * fps if video_fps > 0 else total_frames
+ nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+ nframes = _floor_by_factor(nframes, _OV2_FRAME_FACTOR)
+ return max(int(nframes), _OV2_FRAME_FACTOR)
+
+
+@VIDEO_LOADER_REGISTRY.register(
+ "llava_onevision2",
+ video_processor="LlavaOnevision2VideoProcessor",
+)
+class LlavaOnevision2VideoBackend(VideoBackend):
+ """Frame-sampling backend for LLaVA-OneVision-2.
+
+ Selected automatically for OV2 via the ``video_processor`` binding
+ (``video_processor_type == "LlavaOnevision2VideoProcessor"`` in the model's
+ ``video_preprocessor_config.json``). Decoding uses the inherited OpenCV /
+ PyAV codecs; only the sampling index policy is overridden to match qwen.
+ """
+
+ _sampling_suffix = "_llava_onevision2"
+
+ # OV2 hf-chat reference sampling constants (mirror the validated adapter).
+ _FPS = _DEFAULT_FPS
+ _MAX_FRAMES = _DEFAULT_MAX_FRAMES
+ _MIN_FRAMES = _OV2_FPS_MIN_FRAMES
+
+ @classmethod
+ def compute_frames_index_to_sample(
+ cls,
+ source: VideoSourceMetadata,
+ target: VideoTargetMetadata,
+ **kwargs,
+ ) -> list[int]:
+ total = int(source.total_frames_num)
+ if total <= 0:
+ return []
+ video_fps = float(source.original_fps)
+ # Honor caller-provided sampling targets (via ``--media-io-kwargs`` →
+ # ``VideoTargetMetadata``) so benchmarks can override the conservative
+ # defaults (e.g. VSI-Bench needs max_frames=128). Fall back to the OV2
+ # hf-chat reference constants when the target leaves a field unset
+ # (sentinel ``<= 0``).
+ target_fps = float(target.fps) if target.fps > 0 else cls._FPS
+ target_max_frames = (
+ int(target.num_frames) if target.num_frames > 0 else cls._MAX_FRAMES
+ )
+ n = _ov2_smart_nframes(
+ total,
+ video_fps,
+ fps=target_fps,
+ min_frames=cls._MIN_FRAMES,
+ max_frames=target_max_frames,
+ )
+ # qwen uses linspace().round() (NOT the floor cast used by the base
+ # uniform backend), so replicate the rounding exactly.
+ idx = np.linspace(0, total - 1, n).round().astype(int).tolist()
+ # smart_nframes floors to FRAME_FACTOR so ``n`` is even; guard anyway
+ # since OV2's vision tower (temporal merge = 2) requires even counts.
+ if len(idx) % _OV2_FRAME_FACTOR != 0:
+ idx.append(idx[-1])
+ return idx
+
+
+class LlavaOnevision2ImagePixelInputs(TensorSchema):
+ type: Literal["pixel_values"]
+
+ pixel_values: Annotated[torch.Tensor, TensorShape("np", "cps")]
+ image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
+ patch_positions: Annotated[torch.Tensor, TensorShape("np", 3)]
+
+
+class LlavaOnevision2ImageEmbeddingInputs(TensorSchema):
+ type: Literal["image_embeds"]
+
+ image_embeds: Annotated[torch.Tensor, TensorShape("nf", "hs")]
+ image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
+
+
+class LlavaOnevision2VideoPixelInputs(TensorSchema):
+ type: Literal["pixel_values_videos"]
+
+ pixel_values_videos: Annotated[torch.Tensor, TensorShape("np", "cps")]
+ video_grid_thw: Annotated[torch.Tensor, TensorShape("nf", 3)]
+ patch_positions_videos: Annotated[torch.Tensor, TensorShape("np", 3)]
+ video_num_frames: Annotated[torch.Tensor, TensorShape("nv")]
+
+
+LlavaOnevision2ImageInputs = (
+ LlavaOnevision2ImagePixelInputs | LlavaOnevision2ImageEmbeddingInputs
+)
+
+
+class LlavaOnevision2VisionRotaryEmbedding(nn.Module):
+ """3-D rotary frequency constructor with 4:6:6 (T:H:W) split.
+
+ Mirrors ``VisionRotaryEmbedding`` in the HF reference
+ (``modeling_llava_onevision2.py`` L79-L210). The three ``inv_freq_*``
+ buffers are non-persistent — they are *not* in the checkpoint and must
+ be reconstructed at module init time (which we do here).
+
+ Public entry points used by the vision tower:
+ * ``forward_from_positions(patch_positions)`` — per-patch (t,h,w)
+ positions → per-token freqs [N, half].
+ """
+
+ def __init__(self, head_dim: int, theta: float = 10000.0) -> None:
+ super().__init__()
+ assert head_dim % 2 == 0, "head_dim must be even"
+ assert head_dim % 16 == 0, "head_dim must be divisible by 16 (4:6:6)"
+ half = head_dim // 2
+ assert half % 16 == 0, "head_dim//2 must be divisible by 16"
+
+ self.head_dim = head_dim
+ self.half = half
+ self.base = float(theta)
+
+ unit = half // 16
+ self.t_size = 4 * unit
+ self.h_size = 6 * unit
+ self.w_size = 6 * unit
+ assert self.t_size + self.h_size + self.w_size == half
+
+ self.register_buffer(
+ "inv_freq_t",
+ 1.0
+ / (
+ self.base
+ ** (torch.arange(self.t_size, dtype=torch.float32) / self.t_size)
+ ),
+ persistent=False,
+ )
+ self.register_buffer(
+ "inv_freq_h",
+ 1.0
+ / (
+ self.base
+ ** (torch.arange(self.h_size, dtype=torch.float32) / self.h_size)
+ ),
+ persistent=False,
+ )
+ self.register_buffer(
+ "inv_freq_w",
+ 1.0
+ / (
+ self.base
+ ** (torch.arange(self.w_size, dtype=torch.float32) / self.w_size)
+ ),
+ persistent=False,
+ )
+
+ def forward_from_positions(self, patch_positions: torch.Tensor) -> torch.Tensor:
+ """[N, 3] (t,h,w) int → [N, half] float frequencies."""
+ device = patch_positions.device
+ inv_t = self.inv_freq_t.to(device=device)
+ inv_h = self.inv_freq_h.to(device=device)
+ inv_w = self.inv_freq_w.to(device=device)
+
+ t_pos = patch_positions[:, 0].float()
+ h_pos = patch_positions[:, 1].float()
+ w_pos = patch_positions[:, 2].float()
+
+ ft = torch.outer(t_pos, inv_t)
+ fh = torch.outer(h_pos, inv_h)
+ fw = torch.outer(w_pos, inv_w)
+ return torch.cat([ft, fh, fw], dim=-1)
+
+
+class LlavaOnevision2VisionEmbeddings(nn.Module):
+ def __init__(
+ self, patch_size: int = 14, in_channels: int = 3, embed_dim: int = 1024
+ ) -> None:
+ super().__init__()
+ self.patch_size = patch_size
+ self.in_channels = in_channels
+ self.embed_dim = embed_dim
+ self.patch_embedding = nn.Conv2d(
+ in_channels,
+ embed_dim,
+ kernel_size=(patch_size, patch_size),
+ stride=(patch_size, patch_size),
+ bias=False,
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x = x.view(-1, self.in_channels, self.patch_size, self.patch_size)
+ x = self.patch_embedding(x).view(-1, self.embed_dim)
+ return x
+
+
+class LlavaOnevision2VisionMLP(nn.Module):
+ def __init__(
+ self,
+ in_features: int,
+ hidden_features: int,
+ bias: bool = True,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ if quant_config is not None:
+ raise RuntimeError("LLaVAOneVision2 does not support quantization")
+ self.fc1 = ColumnParallelLinear(
+ in_features,
+ hidden_features,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.fc1",
+ disable_tp=use_data_parallel,
+ )
+ self.fc2 = RowParallelLinear(
+ hidden_features,
+ in_features,
+ bias=bias,
+ quant_config=quant_config,
+ prefix=f"{prefix}.fc2",
+ disable_tp=use_data_parallel,
+ )
+ self.act_fn = F.gelu
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ x, _ = self.fc1(x)
+ x = self.act_fn(x)
+ x, _ = self.fc2(x)
+ return x
+
+
+class LlavaOnevision2VisionAttn(nn.Module):
+ """Vision self-attention with windowed cu_seqlens.
+
+ The HF checkpoint ships a *fused* qkv linear (``self_attn.qkv``), so
+ we load directly into ``QKVParallelLinear`` with no stacked_params
+ mapping. (Compare OV1.5, whose checkpoint had separate q/k/v.)
+ """
+
+ def __init__(
+ self,
+ embed_dim: int,
+ num_heads: int,
+ projection_size: int,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ if quant_config is not None:
+ raise RuntimeError("LLaVAOneVision2 does not support quantization")
+
+ self.tp_size = (
+ 1
+ if use_data_parallel
+ else parallel_state.get_tensor_model_parallel_world_size()
+ )
+ self.tp_rank = parallel_state.get_tensor_model_parallel_rank()
+ self.num_heads = num_heads
+ self.hidden_size_per_attn_head = dist_utils.divide(projection_size, num_heads)
+ self.num_attn_heads_per_partition = dist_utils.divide(num_heads, self.tp_size)
+
+ self.qkv = QKVParallelLinear(
+ hidden_size=embed_dim,
+ head_size=self.hidden_size_per_attn_head,
+ total_num_heads=num_heads,
+ total_num_kv_heads=num_heads,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.qkv",
+ disable_tp=use_data_parallel,
+ )
+
+ self.proj = RowParallelLinear(
+ input_size=projection_size,
+ output_size=embed_dim,
+ quant_config=quant_config,
+ prefix=f"{prefix}.proj",
+ disable_tp=use_data_parallel,
+ )
+
+ self.attn = MMEncoderAttention(
+ num_heads=self.num_attn_heads_per_partition,
+ head_size=self.hidden_size_per_attn_head,
+ scale=self.hidden_size_per_attn_head**-0.5,
+ prefix=f"{prefix}.attn",
+ )
+
+ @staticmethod
+ def _rotate_half_interleaved(x: torch.Tensor) -> torch.Tensor:
+ """OV2-specific interleaved rotate_half.
+
+ Pairs adjacent dims: (x[::2], x[1::2]) -> (-x[1::2], x[::2]).
+ NOT compatible with the split-half rotate used in OV1.5/LLaMA.
+ """
+ x_even = x[..., 0::2]
+ x_odd = x[..., 1::2]
+ out = torch.stack((-x_odd, x_even), dim=-1)
+ return out.flatten(-2)
+
+ def _apply_rotary_pos_embed(
+ self, t: torch.Tensor, freqs: torch.Tensor
+ ) -> torch.Tensor:
+ # freqs is [seq_len, half]; cat([f,f],-1) pair-repeat layout
+ # matches the interleaved rotate above.
+ orig_dtype = t.dtype
+ t = t.float()
+ emb = torch.cat((freqs, freqs), dim=-1)
+ cos = emb.cos().unsqueeze(-2).float()
+ sin = emb.sin().unsqueeze(-2).float()
+ t = (t * cos) + (self._rotate_half_interleaved(t) * sin)
+ return t.to(orig_dtype)
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ cu_seqlens: torch.Tensor,
+ rotary_pos_emb: torch.Tensor,
+ max_seqlen: torch.Tensor | None = None,
+ sequence_lengths: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ x, _ = self.qkv(x)
+ seq_len = x.shape[0]
+ # QKVParallelLinear packs q/k/v along the last dim as
+ # [q_heads, k_heads, v_heads]; view splits the three sections, each
+ # holding this partition's heads (no all-gather: MMEncoderAttention
+ # runs per-partition and ``proj`` reduces across TP ranks).
+ qkv = x.view(
+ seq_len,
+ 3,
+ self.num_attn_heads_per_partition,
+ self.hidden_size_per_attn_head,
+ )
+ q, k, v = qkv.unbind(1)
+
+ if rotary_pos_emb is not None:
+ # OV2 uses interleaved RoPE on the raw freqs, applied outside
+ # MMEncoderAttention (which is rotary-agnostic).
+ q = self._apply_rotary_pos_embed(q, rotary_pos_emb)
+ k = self._apply_rotary_pos_embed(k, rotary_pos_emb)
+
+ # Add a leading batch dim (b=1) for MMEncoderAttention, which expects
+ # (batch, seq_len, num_heads, head_size) and windows via cu_seqlens.
+ output = self.attn(
+ query=q.unsqueeze(0),
+ key=k.unsqueeze(0),
+ value=v.unsqueeze(0),
+ cu_seqlens=cu_seqlens,
+ max_seqlen=max_seqlen,
+ sequence_lengths=sequence_lengths,
+ )
+ output = output.reshape(seq_len, -1)
+
+ output, _ = self.proj(output)
+ return output
+
+
+@support_torch_compile(
+ dynamic_arg_dims={
+ "x": 0,
+ "cu_seqlens": 0,
+ "rotary_pos_emb": 0,
+ "sequence_lengths": 0,
+ },
+ enable_if=should_torch_compile_mm_encoder,
+ is_encoder=True,
+)
+class LlavaOnevision2VisionTowerBlock(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ num_heads: int,
+ mlp_hidden_dim: int,
+ norm_eps: float = 1e-6,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ self.layer_norm1 = nn.LayerNorm(dim, eps=norm_eps)
+ self.layer_norm2 = nn.LayerNorm(dim, eps=norm_eps)
+ self.self_attn = LlavaOnevision2VisionAttn(
+ embed_dim=dim,
+ num_heads=num_heads,
+ projection_size=dim,
+ quant_config=quant_config,
+ prefix=f"{prefix}.self_attn",
+ use_data_parallel=use_data_parallel,
+ )
+ self.mlp = LlavaOnevision2VisionMLP(
+ dim,
+ mlp_hidden_dim,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.mlp",
+ use_data_parallel=use_data_parallel,
+ )
+
+ def forward(
+ self,
+ x: torch.Tensor,
+ cu_seqlens: torch.Tensor,
+ rotary_pos_emb: torch.Tensor,
+ max_seqlen: torch.Tensor | None = None,
+ sequence_lengths: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ x = x + self.self_attn(
+ self.layer_norm1(x),
+ cu_seqlens=cu_seqlens,
+ rotary_pos_emb=rotary_pos_emb,
+ max_seqlen=max_seqlen,
+ sequence_lengths=sequence_lengths,
+ )
+ x = x + self.mlp(self.layer_norm2(x))
+ return x
+
+
+class LlavaOnevision2PatchMerger(nn.Module):
+ def __init__(
+ self,
+ d_model: int,
+ context_dim: int,
+ spatial_merge_size: int = 2,
+ norm_eps: float = 1e-6,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ self.hidden_size = context_dim * (spatial_merge_size**2)
+ self.ln_q = nn.LayerNorm(context_dim, eps=norm_eps)
+ self.mlp = nn.ModuleList(
+ [
+ ColumnParallelLinear(
+ self.hidden_size,
+ self.hidden_size,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.mlp.0",
+ disable_tp=use_data_parallel,
+ ),
+ nn.GELU(),
+ RowParallelLinear(
+ self.hidden_size,
+ d_model,
+ bias=True,
+ quant_config=quant_config,
+ prefix=f"{prefix}.mlp.2",
+ disable_tp=use_data_parallel,
+ ),
+ ]
+ )
+
+ def forward(
+ self, x: torch.Tensor, patch_positions: torch.Tensor | None = None
+ ) -> torch.Tensor:
+ # patch_positions accepted for API symmetry with the HF impl,
+ # but unused: pixel_values already arrive in spatial-merge block
+ # order (Qwen2VLImageProcessor handles that across image / video-
+ # frames / video-codec backends). See codec_video_processing for
+ # the codec backend's ``codec_positions_for_processor`` call.
+ del patch_positions
+ x = self.ln_q(x)
+ x = x.view(-1, self.hidden_size)
+ fc1, act, fc2 = self.mlp
+ x, _ = fc1(x)
+ x = act(x)
+ x, _ = fc2(x)
+ return x
+
+
+class LlavaOnevision2VisionTower(nn.Module):
+ """OV2 vision tower (no CLS token, 3-D RoPE, windowed attention).
+
+ Module attribute names mirror HF checkpoint names verbatim so the
+ WeightsMapper only needs prefix rewrites (no substring rules, which would
+ otherwise collide with the Qwen3 text-path ``self_attn`` modules):
+ visual.embeddings.patch_embedding
+ visual.layernorm_pre
+ visual.encoder.layers.{i}.self_attn.{qkv,proj}
+ visual.encoder.layers.{i}.layer_norm{1,2}
+ visual.encoder.layers.{i}.mlp.fc{1,2}
+ visual.merger.{ln_q, mlp.{0,2}}
+ visual.rotary_pos_emb (non-persistent inv_freq buffers)
+ """
+
+ def __init__(
+ self,
+ vision_config,
+ text_hidden_size: int,
+ norm_eps: float = 1e-6,
+ quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
+ use_data_parallel: bool = False,
+ ) -> None:
+ super().__init__()
+ if quant_config is not None:
+ raise RuntimeError("LLaVAOneVision2 does not support quantization")
+
+ patch_size = vision_config.patch_size
+ spatial_merge_size = vision_config.spatial_merge_size
+ in_channels = getattr(vision_config, "num_channels", 3)
+ hidden_size = vision_config.hidden_size
+ embed_dim = hidden_size
+ depth = vision_config.num_hidden_layers
+ num_heads = vision_config.num_attention_heads
+ mlp_hidden_dim = vision_config.intermediate_size
+ frame_windows_size = getattr(vision_config, "frame_windows_size", 4)
+ rope_theta = getattr(vision_config, "rope_theta", 10000.0)
+
+ self.spatial_merge_size = spatial_merge_size
+ self.frame_windows_size = int(frame_windows_size)
+ self.num_heads = num_heads
+ self.embed_dim = embed_dim
+ self.head_dim = embed_dim // num_heads
+ self.use_data_parallel = use_data_parallel
+ self.tp_size = (
+ 1
+ if use_data_parallel
+ else parallel_state.get_tensor_model_parallel_world_size()
+ )
+
+ self.embeddings = LlavaOnevision2VisionEmbeddings(
+ patch_size=patch_size, in_channels=in_channels, embed_dim=embed_dim
+ )
+ self.layernorm_pre = nn.LayerNorm(embed_dim, eps=norm_eps)
+
+ self.rotary_pos_emb = LlavaOnevision2VisionRotaryEmbedding(
+ self.head_dim, theta=rope_theta
+ )
+
+ self.encoder = nn.Module()
+ self.encoder.layers = nn.ModuleList(
+ [
+ LlavaOnevision2VisionTowerBlock(
+ dim=embed_dim,
+ num_heads=num_heads,
+ mlp_hidden_dim=mlp_hidden_dim,
+ norm_eps=norm_eps,
+ quant_config=quant_config,
+ prefix=f"{prefix}.encoder.layers.{i}",
+ use_data_parallel=use_data_parallel,
+ )
+ for i in range(depth)
+ ]
+ )
+
+ self.merger = LlavaOnevision2PatchMerger(
+ d_model=text_hidden_size,
+ context_dim=embed_dim,
+ spatial_merge_size=spatial_merge_size,
+ norm_eps=norm_eps,
+ quant_config=quant_config,
+ prefix=f"{prefix}.merger",
+ use_data_parallel=use_data_parallel,
+ )
+
+ # Vision attention backend; mirrors the one MMEncoderAttention picks
+ # internally so the cu_seqlens / max_seqlen metadata computed below
+ # matches the kernel actually used.
+ self.attn_backend = get_vit_attn_backend(
+ head_size=self.head_dim, dtype=torch.get_default_dtype()
+ )
+
+ @property
+ def dtype(self) -> torch.dtype:
+ return self.embeddings.patch_embedding.weight.dtype
+
+ @property
+ def device(self) -> torch.device:
+ return self.embeddings.patch_embedding.weight.device
+
+ def _build_window_cu_seqlens(self, grid_thw: torch.Tensor) -> np.ndarray:
+ """Build cu_seqlens that chunk each sample's T-axis into windows of
+ ``frame_windows_size`` frames.
+
+ Returns an int32 ``np.ndarray`` of shape [num_windows+1] (the
+ canonical prefix-sum format). Backend-specific transforms are applied
+ afterwards via ``MMEncoderAttention.maybe_recompute_cu_seqlens``.
+ """
+ win = self.frame_windows_size
+ chunk_lengths: list[int] = []
+ for row in grid_thw.tolist():
+ t, h, w = int(row[0]), int(row[1]), int(row[2])
+ per_frame = h * w
+ t_remaining = t
+ while t_remaining > 0:
+ this_t = min(win, t_remaining)
+ chunk_lengths.append(this_t * per_frame)
+ t_remaining -= this_t
+ cu = np.concatenate(
+ [
+ np.zeros(1, dtype=np.int32),
+ np.array(chunk_lengths, dtype=np.int32).cumsum(dtype=np.int32),
+ ]
+ )
+ return cu
+
+ def forward(
+ self,
+ pixel_values: torch.Tensor,
+ grid_thw: torch.Tensor,
+ patch_positions: torch.Tensor,
+ ) -> torch.Tensor:
+ x = pixel_values.to(device=self.device, dtype=self.dtype)
+ x = self.embeddings(x)
+ x = self.layernorm_pre(x)
+
+ rotary_pos_emb = self.rotary_pos_emb.forward_from_positions(
+ patch_positions.to(self.device)
+ )
+
+ # Build window cu_seqlens, then derive backend-specific attention
+ # metadata (passthrough for FA/SDPA; transformed for FlashInfer).
+ cu_seqlens_np = self._build_window_cu_seqlens(grid_thw)
+ sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
+ self.attn_backend, cu_seqlens_np, self.device
+ )
+ max_seqlen = torch.tensor(
+ MMEncoderAttention.compute_max_seqlen(self.attn_backend, cu_seqlens_np),
+ dtype=torch.int32,
+ )
+ cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
+ self.attn_backend,
+ cu_seqlens_np,
+ self.embed_dim,
+ self.tp_size,
+ self.device,
+ )
+
+ for blk in self.encoder.layers:
+ x = blk(
+ x,
+ cu_seqlens=cu_seqlens,
+ rotary_pos_emb=rotary_pos_emb,
+ max_seqlen=max_seqlen,
+ sequence_lengths=sequence_lengths,
+ )
+
+ return self.merger(x, patch_positions=patch_positions)
+
+
+class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
+ def get_hf_config(self):
+ return self.ctx.get_hf_config()
+
+ def get_data_parser(self) -> MultiModalDataParser:
+ # ``video_needs_metadata=True`` makes the parser preserve both the
+ # ``(frames, metadata)`` tuples from the frame backend and the
+ # ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
+ # both are dispatched by metadata content in ``_call_hf_processor``.
+ return LlavaOnevision2MultiModalDataParser(
+ self.get_hf_config().vision_config.spatial_merge_size,
+ video_needs_metadata=True,
+ )
+
+ def get_hf_processor(self, **kwargs: object):
+ # OV2's trust_remote_code processor is a bare class (not a
+ # ProcessorMixin), so the shared get_hf_processor cannot load it; load
+ # via AutoProcessor here (as other trust_remote_code models do).
+ model_config = self.ctx.model_config
+ # ``_merge_mm_kwargs`` restricts caller ``mm_processor_kwargs`` to known
+ # processor args and wraps values as hashable for the lru_cache.
+ merged = _merge_mm_kwargs(model_config, AutoProcessor, **kwargs)
+ merged.setdefault("use_fast", True)
+ return _load_ov2_processor(
+ model_config.model,
+ model_config.revision,
+ model_config.trust_remote_code,
+ **merged,
+ )
+
+ def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessor:
+ return self.get_hf_processor(**kwargs).image_processor
+
+ def get_mm_max_tokens_per_item(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ ) -> Mapping[str, int]:
+ return {
+ "image": self.get_max_image_tokens(),
+ "video": self.get_max_video_tokens(seq_len, mm_counts),
+ }
+
+ def _get_vision_info(
+ self,
+ *,
+ image_width: int,
+ image_height: int,
+ num_frames: int = 1,
+ do_resize: bool = True,
+ image_processor: Qwen2VLImageProcessor | None,
+ ) -> tuple[ImageSize, int]:
+ if image_processor is None:
+ image_processor = self.get_image_processor()
+ hf_config = self.get_hf_config()
+ vision_config = hf_config.vision_config
+ patch_size = vision_config.patch_size
+ merge_size = vision_config.spatial_merge_size
+ temporal_patch_size = getattr(vision_config, "temporal_patch_size", 1)
+ if do_resize:
+ min_pixels = getattr(image_processor, "min_pixels", None)
+ max_pixels = getattr(image_processor, "max_pixels", None)
+ if min_pixels is None or max_pixels is None:
+ size = image_processor.size
+ min_pixels = (
+ getattr(size, "shortest_edge", None) or size["shortest_edge"]
+ )
+ max_pixels = getattr(size, "longest_edge", None) or size["longest_edge"]
+ rh, rw = smart_resize(
+ height=image_height,
+ width=image_width,
+ factor=patch_size * merge_size,
+ min_pixels=min_pixels,
+ max_pixels=max_pixels,
+ )
+ preprocessed = ImageSize(width=rw, height=rh)
+ else:
+ preprocessed = ImageSize(width=image_width, height=image_height)
+ padded_frames = num_frames + num_frames % temporal_patch_size
+ grid_t = max(padded_frames // temporal_patch_size, 1)
+ grid_h = preprocessed.height // patch_size
+ grid_w = preprocessed.width // patch_size
+ num_patches = grid_t * grid_h * grid_w
+ return preprocessed, num_patches // (merge_size**2)
+
+ def get_num_image_tokens(
+ self,
+ *,
+ image_width: int,
+ image_height: int,
+ image_processor: Qwen2VLImageProcessor | None,
+ ) -> int:
+ _, n = self._get_vision_info(
+ image_width=image_width,
+ image_height=image_height,
+ image_processor=image_processor,
+ )
+ return n
+
+ def get_image_size_with_most_features(self) -> ImageSize:
+ sz, _ = self._get_vision_info(
+ image_width=1800, image_height=1800, image_processor=None
+ )
+ return sz
+
+ def get_max_image_tokens(self) -> int:
+ w, h = self.get_image_size_with_most_features()
+ return self.get_num_image_tokens(
+ image_width=w, image_height=h, image_processor=None
+ )
+
+ def get_num_video_tokens(
+ self,
+ *,
+ image_width: int,
+ image_height: int,
+ num_frames: int,
+ image_processor: Qwen2VLImageProcessor | None = None,
+ ) -> int:
+ _, n = self._get_vision_info(
+ image_width=image_width,
+ image_height=image_height,
+ num_frames=num_frames,
+ image_processor=image_processor,
+ )
+ return n
+
+ def _get_max_video_frames(self, max_tokens: int, start_num_frames: int = 1) -> int:
+ w, h = self.get_image_size_with_most_features()
+ num_frames = start_num_frames
+ while True:
+ next_num_frames = num_frames + 1
+ next_max_tokens = self.get_num_video_tokens(
+ image_width=w,
+ image_height=h,
+ num_frames=next_num_frames,
+ )
+ if next_max_tokens > max_tokens:
+ break
+ num_frames = next_num_frames
+ return num_frames
+
+ def get_num_frames_with_most_features(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO,
+ ) -> int:
+ max_videos = mm_counts.get("video", 0)
+ max_total_frames = self._get_max_video_frames(seq_len)
+ max_frames_per_video = min(
+ max_total_frames // max(max_videos, 1), max_frames_per_video
+ )
+ return max(max_frames_per_video, 1)
+
+ def get_max_video_tokens(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ ) -> int:
+ w, h = self.get_image_size_with_most_features()
+ return self.get_num_video_tokens(
+ image_width=w,
+ image_height=h,
+ num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts),
+ )
+
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+ return {"image": None, "video": None}
+
+
+class LlavaOnevision2DummyInputsBuilder(
+ BaseDummyInputsBuilder[LlavaOnevision2ProcessingInfo]
+):
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+ n_img = mm_counts.get("image", 0)
+ n_vid = mm_counts.get("video", 0)
+ return (
+ "<|vision_start|><|image_pad|><|vision_end|>" * n_img
+ + "<|vision_start|><|video_pad|><|vision_end|>" * n_vid
+ )
+
+ def get_dummy_mm_data(
+ self,
+ seq_len: int,
+ mm_counts: Mapping[str, int],
+ mm_options: Mapping[str, BaseDummyOptions] | None = None,
+ ) -> MultiModalDataDict:
+ n_img = mm_counts.get("image", 0)
+ n_vid = mm_counts.get("video", 0)
+ w, h = self.info.get_image_size_with_most_features()
+ out: MultiModalDataDict = {}
+ if n_img:
+ out["image"] = self._get_dummy_images(width=w, height=h, num_images=n_img)
+ if n_vid:
+ # 4 frames per dummy video keeps profiling cheap; the real frame
+ # count is decided by the HF VideoProcessor at apply time.
+ out["video"] = self._get_dummy_videos(
+ width=w, height=h, num_frames=4, num_videos=n_vid
+ )
+ return out
+
+ def _get_dummy_videos(
+ self,
+ *,
+ width: int,
+ height: int,
+ num_frames: int,
+ num_videos: int,
+ overrides=None,
+ ):
+ # ``video_needs_metadata=True`` (see ProcessingInfo.get_data_parser)
+ # makes the parser require a metadata dict on every video item, so the
+ # dummy profiling videos must carry one too. ``do_sample_frames=False``
+ # plus ``frames_indices=range(T)`` tells the frame path to consume the
+ # pre-built frames verbatim (no resampling) -- mirrors GLM-4V.
+ # OV2's vision tower (temporal merge = 2) needs an even frame count.
+ num_frames = max(num_frames, _OV2_FRAME_FACTOR)
+ if num_frames % _OV2_FRAME_FACTOR != 0:
+ num_frames += 1
+ videos = super()._get_dummy_videos(
+ width=width,
+ height=height,
+ num_frames=num_frames,
+ num_videos=num_videos,
+ overrides=overrides,
+ )
+ video_items = []
+ for video in videos:
+ t = video.shape[0]
+ metadata = {
+ "fps": 1.0,
+ "duration": float(t),
+ "total_num_frames": int(t),
+ "frames_indices": list(range(t)),
+ "video_backend": "llava_onevision2",
+ "do_sample_frames": False,
+ }
+ video_items.append((video, metadata))
+ return video_items
+
+
+class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
+ def __init__(self, spatial_merge_size: int, *args, **kwargs):
+ self._spatial_merge_size = spatial_merge_size
+ super().__init__(*args, **kwargs)
+
+ def _parse_image_data(
+ self,
+ data: dict[str, torch.Tensor] | ModalityData[ImageItem],
+ ) -> ModalityDataItems[Any, Any] | None:
+ if isinstance(data, dict):
+ return DictEmbeddingItems(
+ data,
+ modality="image",
+ required_fields={"image_embeds", "image_grid_thw"},
+ fields_factory=_create_field_factory(self._spatial_merge_size),
+ )
+ return super()._parse_image_data(data)
+
+
+class LlavaOnevision2MultiModalProcessor(
+ BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
+):
+ def _get_data_parser(self) -> MultiModalDataParser:
+ # Retained for symmetry; vLLM actually fetches the parser via
+ # info.get_data_parser() (see ProcessingInfo override above).
+ return LlavaOnevision2MultiModalDataParser(
+ self.info.get_hf_config().vision_config.spatial_merge_size
+ )
+
+ def _call_hf_processor(
+ self,
+ prompt: str,
+ mm_data: Mapping[str, object],
+ mm_kwargs: Mapping[str, object],
+ tok_kwargs: Mapping[str, object],
+ ) -> BatchFeature:
+ # The wrapped OV2 processor is a bare custom class without the standard
+ # ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
+ # fails; overriding this method routes the base class to call us
+ # directly.
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
+ merged_kwargs = self.info.ctx.get_merged_mm_kwargs(
+ dict(**mm_kwargs, **tok_kwargs)
+ )
+ merged_kwargs.setdefault("return_tensors", "pt")
+ call_kwargs = {
+ k: v
+ for k, v in merged_kwargs.items()
+ if k
+ in {
+ "return_tensors",
+ "padding",
+ "num_frames",
+ "max_frames",
+ "target_fps",
+ "video_backend",
+ "max_pixels",
+ "codec_config",
+ }
+ }
+ mm_data = dict(mm_data)
+ # Explicit None + length checks: ``mm_data[...]`` may be a list, numpy
+ # array, or tensor, and ``and `` would raise on the ambiguous
+ # truth value of a multi-element array.
+ _videos = mm_data.get("videos")
+ videos_present = _videos is not None and len(_videos) > 0
+
+ codec_video_paths = (
+ _extract_codec_video_paths(mm_data["videos"]) if videos_present else None
+ )
+ is_codec_marker = codec_video_paths is not None
+ # Fallback: caller passed video_backend=codec via mm_processor_kwargs
+ # without wrapping paths through prepare_codec_video_input (e.g.
+ # lmms-eval's chat/vllm.py ov2_path_video=True). Recover the path
+ # strings directly from mm_data["videos"] so the codec rename
+ # branch still fires and video-modality fields get populated.
+ is_codec_kwarg = (
+ not is_codec_marker
+ and videos_present
+ and call_kwargs.get("video_backend") == "codec"
+ )
+ if is_codec_kwarg:
+ raw = mm_data["videos"]
+ if isinstance(raw, str):
+ codec_video_paths = [raw]
+ elif isinstance(raw, (list, tuple)) and all(
+ isinstance(x, str) for x in raw
+ ):
+ codec_video_paths = list(raw)
+ else:
+ # Non-string payload (PIL/ndarray/etc.) - codec backend
+ # cannot consume pre-decoded frames; fall through to frame
+ # path.
+ is_codec_kwarg = False
+ is_codec = is_codec_marker or is_codec_kwarg
+
+ if is_codec:
+ # Confine codec paths to --allowed-local-media-path (local-only;
+ # SSRF / local-file-read protection) and use the *resolved* paths
+ # downstream so the codec module opens exactly the file that was
+ # validated (no symlink-retarget / validate-vs-open gap).
+ codec_video_paths = _validate_video_sources(
+ codec_video_paths, self.info.ctx.model_config
+ )
+ # Codec backend: HF processor consumes the path string directly and
+ # performs decode + canvas-packing internally. The dummy ndarray
+ # we attached during prepare_codec_video_input is discarded here.
+ mm_data["videos"] = (
+ codec_video_paths
+ if len(codec_video_paths) > 1
+ else codec_video_paths[0]
+ )
+ # Route through the base ``_call_hf_processor`` so float-tensor
+ # dtype postprocessing is applied automatically; inject
+ # ``video_backend="codec"`` via mm_kwargs so the wrapped processor
+ # dispatches to its codec branch.
+ output = super()._call_hf_processor(
+ prompt=prompt,
+ mm_data=mm_data,
+ mm_kwargs={**mm_kwargs, "video_backend": "codec"},
+ tok_kwargs=tok_kwargs,
+ )
+ data = dict(output)
+ return BatchFeature(
+ self._rename_codec_outputs_to_video(
+ data, codec_video_paths, hf_processor
+ )
+ )
+
+ # ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
+ # Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
+ # tuple (``video_needs_metadata=True``): the connector decoded + sampled
+ # it through ``LlavaOnevision2VideoBackend`` for real ``video_url``
+ # inputs, or the dummy-inputs builder attached synthetic metadata during
+ # profiling. We materialise the frames as PIL images + per-frame
+ # timestamp markers and feed them through the HF processor's *image*
+ # branch (smart_resize + patchify), then re-tag the image-series outputs
+ # as video-series so vLLM's ``<|video_pad|>`` placeholder replacement
+ # finds them. Sampling parity with the original qwen_vl_utils policy is
+ # provided by the backend's ``compute_frames_index_to_sample``; SSRF /
+ # local-file gating is enforced by the connector before decoding.
+ if videos_present:
+ timestamp_decimals = int(
+ mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
+ )
+
+ per_video_frames: list[list[Image.Image]] = []
+ per_video_timestamps: list[list[float]] = []
+ for item in mm_data["videos"]:
+ pil_frames, timestamps = _frame_video_to_pil_and_timestamps(item)
+ per_video_frames.append(pil_frames)
+ per_video_timestamps.append(timestamps)
+
+ # Rewrite the prompt so each video marker becomes a sequence of
+ # ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
+ # blocks (matches the OV2 hf-chat reference exactly).
+ new_prompt = _expand_video_markers_in_prompt(
+ prompt,
+ per_video_timestamps,
+ timestamp_decimals=timestamp_decimals,
+ )
+
+ # Build the merged ``images`` list the wrapped HF processor will
+ # consume. The processor binds the merged list *positionally* to the
+ # ``<|image_pad|>`` slots in prompt order, so it must follow the
+ # interleaved marker order of the prompt (not a fixed "videos first"
+ # order) -- otherwise mixed image+video requests bind frames to the
+ # wrong placeholder. ``row_is_video`` labels each grid row so outputs
+ # can be split back into per-modality keys below.
+ merged_mm_data = dict(mm_data)
+ existing_images = merged_mm_data.pop("images", None)
+ caller_images: list[Image.Image] = []
+ if existing_images:
+ if isinstance(existing_images, list):
+ caller_images.extend(existing_images)
+ else:
+ caller_images.append(existing_images)
+
+ marker_pattern = re.compile(
+ "(?P" + re.escape(_IMAGE_MARKER) + ")"
+ "|(?P