feat: Add video support for the Transformers backend (#57441)
Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Harry Mellor
Claude Opus 5
parent
042ab0305c
commit
34051ad714
@@ -587,12 +587,14 @@ class MarkingStub(SupportsMultiModal, nn.Module):
|
||||
_pre_trained_model_classes = Base._pre_trained_model_classes
|
||||
|
||||
|
||||
def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
|
||||
def build_marked_model(
|
||||
image_limit: int, video_limit: int = 0, skip_tokenizer_init: bool = False
|
||||
):
|
||||
"""Build the HF model inside the marking context and return it."""
|
||||
model_config = ModelConfig(
|
||||
model=MULTIMODAL_MODEL,
|
||||
model_impl="transformers",
|
||||
limit_mm_per_prompt={"image": image_limit},
|
||||
limit_mm_per_prompt={"image": image_limit, "video": video_limit},
|
||||
)
|
||||
# Set after construction: building the config itself needs the tokenizer
|
||||
model_config.skip_tokenizer_init = skip_tokenizer_init
|
||||
@@ -606,10 +608,15 @@ def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
|
||||
return stub.model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("image_limit", "skipped"), [(0, True), (4, False)])
|
||||
def test_tower_weights_skipped_when_modality_disabled(image_limit, skipped):
|
||||
"""`--limit-mm-per-prompt image=0` should drop the vision tower's weights."""
|
||||
vision_tower = build_marked_model(image_limit).vision_tower
|
||||
@pytest.mark.parametrize(
|
||||
("image_limit", "video_limit", "skipped"),
|
||||
[(0, 0, True), (4, 0, False), (0, 4, False)],
|
||||
)
|
||||
def test_tower_weights_skipped_when_modality_disabled(
|
||||
image_limit, video_limit, skipped
|
||||
):
|
||||
"""Only `--limit-mm-per-prompt image=0,video=0` drops the tower they share."""
|
||||
vision_tower = build_marked_model(image_limit, video_limit).vision_tower
|
||||
assert isinstance(vision_tower, StageMissingLayer) is skipped
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user