[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
+2 -26
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import TypeVar
import torch
import torch.nn as nn
@@ -35,8 +34,6 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
)
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
def get_vision_encoder_info(
@@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
return PixtralHFEncoderInfo(self.get_hf_config())
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _postprocess_hf_mm_data(
@@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
if not mm_data:
return processed_data
input_ids = processed_data.get("input_ids")
if input_ids is not None:
processor = self.info.get_hf_processor()
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
break_id = vocab.get(processor.image_break_token)
end_id = vocab.get(processor.image_end_token)
# create mask to remove break/end tokens
keep_mask = ~torch.isin(
input_ids,
torch.tensor([break_id, end_id]),
)
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
if "attention_mask" in processed_data:
processed_data["attention_mask"] = processed_data["attention_mask"][
keep_mask
].unsqueeze(0)
# un-pad pixel_values per-image so caches remain independent.
pixel_values = processed_data.get("pixel_values")
if pixel_values is not None: