feat: Add video support for the Transformers backend (#57441)
pre-commit / pre-run-check (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s

Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harshal Janjani
2026-10-05 00:51:15 -07:00
committed by GitHub
co-authored by Harry Mellor Claude Opus 5
parent 042ab0305c
commit 34051ad714
9 changed files with 775 additions and 115 deletions
+13 -6
View File
@@ -587,12 +587,14 @@ class MarkingStub(SupportsMultiModal, nn.Module):
_pre_trained_model_classes = Base._pre_trained_model_classes
def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
def build_marked_model(
image_limit: int, video_limit: int = 0, skip_tokenizer_init: bool = False
):
"""Build the HF model inside the marking context and return it."""
model_config = ModelConfig(
model=MULTIMODAL_MODEL,
model_impl="transformers",
limit_mm_per_prompt={"image": image_limit},
limit_mm_per_prompt={"image": image_limit, "video": video_limit},
)
# Set after construction: building the config itself needs the tokenizer
model_config.skip_tokenizer_init = skip_tokenizer_init
@@ -606,10 +608,15 @@ def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
return stub.model
@pytest.mark.parametrize(("image_limit", "skipped"), [(0, True), (4, False)])
def test_tower_weights_skipped_when_modality_disabled(image_limit, skipped):
"""`--limit-mm-per-prompt image=0` should drop the vision tower's weights."""
vision_tower = build_marked_model(image_limit).vision_tower
@pytest.mark.parametrize(
("image_limit", "video_limit", "skipped"),
[(0, 0, True), (4, 0, False), (0, 4, False)],
)
def test_tower_weights_skipped_when_modality_disabled(
image_limit, video_limit, skipped
):
"""Only `--limit-mm-per-prompt image=0,video=0` drops the tower they share."""
vision_tower = build_marked_model(image_limit, video_limit).vision_tower
assert isinstance(vision_tower, StageMissingLayer) is skipped