[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -286,7 +286,7 @@ def _validate_video_sources(paths, model_config) -> list[str]:
|
||||
# participates in the standard decode-stage pipeline.
|
||||
#
|
||||
# * codec backend: NOT a loader. OV2's codec path needs the video path string
|
||||
# to survive into ``_call_hf_processor``, where the HF processor builds the
|
||||
# to survive into ``_apply_hf_processor_main``, where the HF processor builds the
|
||||
# codec canvas + smart_resize + patchify
|
||||
# (pixel_values/image_grid_thw/patch_positions). That transform is
|
||||
# path-level and inseparable; it cannot be reconstructed from pre-decoded
|
||||
@@ -302,7 +302,7 @@ def prepare_codec_video_input(video_path: str) -> tuple:
|
||||
|
||||
Returns ``(dummy_ndarray, metadata)`` where the ndarray satisfies the
|
||||
parser's 4-D shape check and the metadata carries the actual path to
|
||||
our ``_call_hf_processor``. Use as::
|
||||
our ``_apply_hf_processor_main``. Use as::
|
||||
|
||||
multi_modal_data = {"video": prepare_codec_video_input("foo.mp4")}
|
||||
|
||||
@@ -1283,7 +1283,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
||||
# ``video_needs_metadata=True`` makes the parser preserve both the
|
||||
# ``(frames, metadata)`` tuples from the frame backend and the
|
||||
# ``(dummy, {marker: path})`` tuples from prepare_codec_video_input;
|
||||
# both are dispatched by metadata content in ``_call_hf_processor``.
|
||||
# both are dispatched by metadata content in ``_apply_hf_processor_main``.
|
||||
return LlavaOnevision2MultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
@@ -1547,18 +1547,23 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
class LlavaOnevision2MultiModalProcessor(
|
||||
BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]
|
||||
):
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# The wrapped OV2 processor is a bare custom class without the standard
|
||||
# ProcessorMixin ``_merge_kwargs`` machinery, so vLLM's default path
|
||||
# fails; overriding this method routes the base class to call us
|
||||
# directly.
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if not mm_data:
|
||||
return BatchFeature(dict(passthrough_data))
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
merged_kwargs = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
merged_kwargs.setdefault("return_tensors", "pt")
|
||||
call_kwargs = {
|
||||
k: v
|
||||
@@ -1627,21 +1632,25 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
if len(codec_video_paths) > 1
|
||||
else codec_video_paths[0]
|
||||
)
|
||||
# Route through the base ``_call_hf_processor`` so float-tensor
|
||||
# Route through ``ctx.call_hf_processor`` so float-tensor
|
||||
# dtype postprocessing is applied automatically; inject
|
||||
# ``video_backend="codec"`` via mm_kwargs so the wrapped processor
|
||||
# dispatches to its codec branch.
|
||||
output = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs={**mm_kwargs, "video_backend": "codec"},
|
||||
output = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(
|
||||
**{**hf_processor_mm_kwargs, "video_backend": "codec"}
|
||||
),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
{**hf_processor_mm_kwargs, "video_backend": "codec"},
|
||||
)
|
||||
data = dict(output)
|
||||
return BatchFeature(
|
||||
processed_data = BatchFeature(
|
||||
self._rename_codec_outputs_to_video(
|
||||
data, codec_video_paths, hf_processor
|
||||
)
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
# ---- Frame backend (registered LlavaOnevision2VideoBackend) ------
|
||||
# Every non-codec video reaches here as a ``(frames_ndarray, metadata)``
|
||||
@@ -1657,7 +1666,9 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# local-file gating is enforced by the connector before decoding.
|
||||
if videos_present:
|
||||
timestamp_decimals = int(
|
||||
mm_kwargs.get("timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS)
|
||||
hf_processor_mm_kwargs.get(
|
||||
"timestamp_decimals", _DEFAULT_TIMESTAMP_DECIMALS
|
||||
)
|
||||
)
|
||||
|
||||
per_video_frames: list[list[Image.Image]] = []
|
||||
@@ -1671,7 +1682,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
# ``<{t} seconds><|vision_start|><|image_pad|><|vision_end|>``
|
||||
# blocks (matches the OV2 hf-chat reference exactly).
|
||||
new_prompt = _expand_video_markers_in_prompt(
|
||||
prompt,
|
||||
prompt_text,
|
||||
per_video_timestamps,
|
||||
timestamp_decimals=timestamp_decimals,
|
||||
)
|
||||
@@ -1700,7 +1711,7 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
row_is_video: list[bool] = []
|
||||
vid_idx = 0
|
||||
img_idx = 0
|
||||
for marker in marker_pattern.finditer(prompt):
|
||||
for marker in marker_pattern.finditer(prompt_text):
|
||||
if marker.lastgroup == "video":
|
||||
frames = per_video_frames[vid_idx]
|
||||
vid_idx += 1
|
||||
@@ -1723,15 +1734,15 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
merged_mm_data.pop("videos", None)
|
||||
merged_mm_data["images"] = flat_frames
|
||||
|
||||
# Route through the base ``_call_hf_processor`` (applies float-tensor
|
||||
# Route through ``ctx.call_hf_processor`` (applies float-tensor
|
||||
# dtype postprocessing automatically). The wrapped processor's image
|
||||
# branch ignores video/codec-only kwargs and does not forward extra
|
||||
# **kwargs to the image processor, so passing the full merged kwarg
|
||||
# set here is a no-op beyond return_tensors/padding.
|
||||
output = super()._call_hf_processor(
|
||||
prompt=new_prompt,
|
||||
mm_data=merged_mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
output = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=new_prompt, **merged_mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
data = dict(output)
|
||||
|
||||
@@ -1788,17 +1799,21 @@ class LlavaOnevision2MultiModalProcessor(
|
||||
(len(per_video_timestamps),), dtype=torch.long
|
||||
)
|
||||
|
||||
return BatchFeature(data)
|
||||
processed_data = BatchFeature(data)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
# ---- Image-only / text-only call --------------------------------
|
||||
# No videos present: delegate to the base ``_call_hf_processor``, which
|
||||
# No videos present: delegate to ``ctx.call_hf_processor``, which
|
||||
# runs the wrapped processor over the (possibly empty) image set and
|
||||
# applies float-tensor dtype postprocessing automatically.
|
||||
return super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
def _rename_codec_outputs_to_video(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user