[MM] Further cleanup _apply_hf_processor_main (#53610)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -35,8 +34,6 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
|
||||
_I = TypeVar("_I", bound=Mistral3ProcessingInfo)
|
||||
|
||||
|
||||
class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
|
||||
def get_vision_encoder_info(
|
||||
@@ -49,8 +46,8 @@ class LightOnOCRProcessingInfo(Mistral3ProcessingInfo):
|
||||
return PixtralHFEncoderInfo(self.get_hf_config())
|
||||
|
||||
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
@@ -62,27 +59,6 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingIn
|
||||
if not mm_data:
|
||||
return processed_data
|
||||
|
||||
input_ids = processed_data.get("input_ids")
|
||||
if input_ids is not None:
|
||||
processor = self.info.get_hf_processor()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
break_id = vocab.get(processor.image_break_token)
|
||||
end_id = vocab.get(processor.image_end_token)
|
||||
|
||||
# create mask to remove break/end tokens
|
||||
keep_mask = ~torch.isin(
|
||||
input_ids,
|
||||
torch.tensor([break_id, end_id]),
|
||||
)
|
||||
|
||||
processed_data["input_ids"] = input_ids[keep_mask].unsqueeze(0)
|
||||
if "attention_mask" in processed_data:
|
||||
processed_data["attention_mask"] = processed_data["attention_mask"][
|
||||
keep_mask
|
||||
].unsqueeze(0)
|
||||
|
||||
# un-pad pixel_values per-image so caches remain independent.
|
||||
pixel_values = processed_data.get("pixel_values")
|
||||
if pixel_values is not None:
|
||||
|
||||
Reference in New Issue
Block a user