[Mypy Fix] Mypy fix for "vllm/model_executor/models/[eE][fF]" (#53381)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -1076,7 +1076,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
def _pixel_values_norm(
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
mm_kwargs: object,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> torch.Tensor:
|
||||
hf_config = self.info.get_hf_config()
|
||||
vision_config = hf_config.vision_config
|
||||
@@ -1125,6 +1125,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
|
||||
mm_data = dict(mm_data)
|
||||
if "images" not in mm_data:
|
||||
mm_data["images"] = []
|
||||
if "videos" not in mm_data:
|
||||
@@ -1136,15 +1137,15 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
|
||||
hf_processor, "supports_video_metadata", False
|
||||
)
|
||||
|
||||
if mm_data["videos"] and not supports_video_metadata:
|
||||
videos = mm_data["videos"]
|
||||
assert isinstance(videos, Sequence)
|
||||
if videos and not supports_video_metadata:
|
||||
# Old HF processor, unwrap tuple to pure frames
|
||||
logger.warning_once(
|
||||
"HF processor doesn't support video metadata. "
|
||||
"Timestamps will NOT be rendered. Please upgrade the model."
|
||||
)
|
||||
mm_data["videos"] = [
|
||||
v[0] if isinstance(v, tuple) else v for v in mm_data["videos"]
|
||||
]
|
||||
mm_data["videos"] = [v[0] if isinstance(v, tuple) else v for v in videos]
|
||||
|
||||
processor_output = self.info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
@@ -1286,6 +1287,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
video_overrides = mm_options.get("video")
|
||||
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
@@ -1541,10 +1543,14 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
|
||||
offset = mm_feature.mm_position.offset
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
grid_thw = mm_feature.data["image_grid_thw"].data
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
t, h, w = grid_thw.tolist()
|
||||
yield offset, t, h // spatial_conv_size, w // spatial_conv_size
|
||||
elif mm_feature.modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
grid_thw = mm_feature.data["video_grid_thw"].data
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
t, h, w = grid_thw.tolist()
|
||||
yield (
|
||||
offset,
|
||||
t // temporal_conv_size,
|
||||
@@ -1563,12 +1569,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
if pixel_values is None:
|
||||
return None
|
||||
|
||||
if pixel_values is not None:
|
||||
return Ernie4_5_VLImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
return Ernie4_5_VLImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
@@ -1579,12 +1584,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
if pixel_values_videos is None:
|
||||
return None
|
||||
|
||||
if pixel_values_videos is not None:
|
||||
return Ernie4_5_VLVideoPixelInputs(
|
||||
type="pixel_values_videos",
|
||||
pixel_values_videos=pixel_values_videos,
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
return Ernie4_5_VLVideoPixelInputs(
|
||||
type="pixel_values_videos",
|
||||
pixel_values_videos=pixel_values_videos,
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
|
||||
# -- SupportsEncoderCudaGraph protocol methods --
|
||||
#
|
||||
@@ -1748,6 +1752,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
# the actual batch grid_thw, then scatter the post-merge embeddings.
|
||||
# Ernie only uses the single "default" encoder path.
|
||||
output = outputs["default"]
|
||||
assert batch_mm_kwargs is not None
|
||||
grid_thw_cpu = batch_mm_kwargs["image_grid_thw"]
|
||||
grid_thw = grid_thw_cpu.to(output.device, non_blocking=True)
|
||||
# The valid token count slices the graph output for the eager
|
||||
@@ -1799,7 +1804,9 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
return video_embeds.split(sizes.tolist())
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
modalities: dict[
|
||||
str, Ernie4_5_VLImageInputs | Ernie4_5_VLVideoInputs | None
|
||||
] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
|
||||
Reference in New Issue
Block a user