[Model] Refactor Step3-VL processor to HF style (#37579)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -39,7 +39,11 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.step3_vl import Step3VisionEncoderConfig
|
||||
from vllm.transformers_utils.processors.step3_vl import Step3VLProcessor
|
||||
from vllm.transformers_utils.processors.step3_vl import (
|
||||
MAX_IMAGE_SIZE,
|
||||
Step3VLImageProcessor,
|
||||
Step3VLProcessor,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
@@ -86,21 +90,30 @@ Step3VLImageInputs: TypeAlias = Step3VLImagePixelInputs | Step3VLImageEmbeddingI
|
||||
|
||||
|
||||
class Step3VLProcessingInfo(BaseProcessingInfo):
|
||||
def get_image_processor(self, **kwargs):
|
||||
config = self.get_hf_config()
|
||||
|
||||
kwargs.setdefault(
|
||||
"enable_patch",
|
||||
getattr(config.vision_config, "enable_patch", True),
|
||||
)
|
||||
|
||||
return Step3VLImageProcessor(**kwargs)
|
||||
|
||||
def get_hf_processor(self) -> Step3VLProcessor:
|
||||
return Step3VLProcessor(
|
||||
self.get_hf_config(),
|
||||
self.get_tokenizer(),
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=self.get_image_processor(),
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
hf_processor = self.get_hf_processor()
|
||||
return hf_processor.get_num_image_tokens(
|
||||
self.get_image_size_with_most_features().width,
|
||||
self.get_image_size_with_most_features().height,
|
||||
)
|
||||
image_processor = self.get_image_processor()
|
||||
target_width, target_height = self.get_image_size_with_most_features()
|
||||
|
||||
return image_processor.get_num_image_tokens(target_width, target_height)
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
@@ -110,20 +123,7 @@ class Step3VLProcessingInfo(BaseProcessingInfo):
|
||||
return {"image": self.get_max_image_tokens()}
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
return ImageSize(3024, 3024)
|
||||
|
||||
def get_num_mm_tokens(self, mm_data: MultiModalDataDict) -> int:
|
||||
if len(mm_data) != 1 or "image" not in mm_data:
|
||||
raise ValueError("mm_data could only contain one key 'image' for steo1o")
|
||||
|
||||
image_data = mm_data["image"]
|
||||
if not isinstance(image_data, (list, tuple)):
|
||||
image_data = [image_data]
|
||||
|
||||
return sum(
|
||||
self.get_hf_processor().get_num_image_tokens(img.width, img.height)
|
||||
for img in image_data
|
||||
)
|
||||
return ImageSize(MAX_IMAGE_SIZE, MAX_IMAGE_SIZE)
|
||||
|
||||
|
||||
class Step3VLDummyInputsBuilder(BaseDummyInputsBuilder[Step3VLProcessingInfo]):
|
||||
@@ -165,13 +165,11 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo])
|
||||
def get_replacement_step1o(item_idx: int):
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
num_patches = int(out_item["num_patches"].data)
|
||||
if num_patches > 0:
|
||||
patch_newline_mask = out_item["patch_newline_mask"].data
|
||||
image_repl_ids = hf_processor._get_image_repl_features(
|
||||
1, num_patches, patch_newline_mask.tolist()
|
||||
)[1]
|
||||
else:
|
||||
image_repl_ids = hf_processor._get_image_repl_features(1, 0, None)[1]
|
||||
patch_newline_mask = out_item["patch_newline_mask"].data
|
||||
image_repl_ids = hf_processor.get_image_repl_feature_ids(
|
||||
1, num_patches, patch_newline_mask.tolist()
|
||||
)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
seq=image_repl_ids,
|
||||
embed_token_id=image_placeholder_token_id,
|
||||
|
||||
Reference in New Issue
Block a user