[Refactor] Remove dead code in multiple files (#49745)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -257,25 +257,6 @@ class FunAudioChatAudioEncoder(nn.Module):
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.conv1.weight.dtype
|
||||
|
||||
def _prepare_attention_mask(
|
||||
self, inputs_tensor: torch.Tensor, cu_seqlens: torch.Tensor
|
||||
) -> torch.Tensor | None:
|
||||
if getattr(self.config, "_attn_implementation", "eager") == "flash_attention_2":
|
||||
return None
|
||||
|
||||
seq_length = inputs_tensor.shape[0]
|
||||
attention_mask = torch.full(
|
||||
(1, 1, seq_length, seq_length),
|
||||
torch.finfo(inputs_tensor.dtype).min,
|
||||
device=inputs_tensor.device,
|
||||
dtype=inputs_tensor.dtype,
|
||||
)
|
||||
for i in range(1, len(cu_seqlens)):
|
||||
start = int(cu_seqlens[i - 1].item())
|
||||
end = int(cu_seqlens[i].item())
|
||||
attention_mask[..., start:end, start:end] = 0
|
||||
return attention_mask
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_features: torch.Tensor,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# copied from : https://github.com/huggingface/transformers
|
||||
import ast
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
@@ -891,37 +890,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
for i in range(len(feats_per_video))
|
||||
)
|
||||
|
||||
def _prepare_multimodal_kwargs(self, **kwargs: object):
|
||||
output = defaultdict(list)
|
||||
for k, v in kwargs.items():
|
||||
if len(v) < 1 or len(v[0]) < 1:
|
||||
continue # if empty batch of empty sample
|
||||
|
||||
new_k, is_video = k, False
|
||||
if not k.endswith("_images") and not k.endswith("_videos"):
|
||||
pass
|
||||
else:
|
||||
new_k, is_video = k.split("_")[:-1], k.split("_")[-1]
|
||||
new_k = "_".join(new_k)
|
||||
is_video = is_video == "videos"
|
||||
|
||||
for _sample_idx, _v in enumerate(v): # batch -> sample
|
||||
if new_k not in ["pixel_values"]:
|
||||
if len(output[new_k]) < _sample_idx + 1:
|
||||
output[new_k].append(list())
|
||||
_v = _v.detach().cpu().numpy().tolist()
|
||||
output[new_k][_sample_idx] += _v
|
||||
elif isinstance(_v, torch.Tensor):
|
||||
if len(output[new_k]) < _sample_idx + 1:
|
||||
output[new_k].append(list())
|
||||
output["is_videos"].append(list())
|
||||
_v = list(torch.unbind(_v, dim=0))
|
||||
output[new_k][_sample_idx] += _v
|
||||
output["is_videos"][_sample_idx] += [
|
||||
is_video,
|
||||
] * len(_v)
|
||||
return dict(output)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -138,30 +138,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
return height, width
|
||||
|
||||
def _get_resize_output_image_size(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
resolution_max_side: int,
|
||||
) -> tuple[int, int]:
|
||||
hf_processor = self.get_hf_processor()
|
||||
image_processor: Idefics3ImageProcessor = hf_processor.image_processor
|
||||
max_image_size = image_processor.size["longest_edge"]
|
||||
if resolution_max_side > max_image_size:
|
||||
raise ValueError(
|
||||
"`resolution_max_side` cannot be larger than `max_image_size`"
|
||||
)
|
||||
|
||||
height, width = image_height, image_width
|
||||
|
||||
# Find the output size, when rescaling the longest edge to max_len and
|
||||
# preserving the aspect ratio
|
||||
height, width = self._resize_output_size(
|
||||
height=height, width=width, max_len=resolution_max_side
|
||||
)
|
||||
return height, width
|
||||
|
||||
def _get_image_feature_grid_size(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -603,10 +603,6 @@ _OV2_FRAME_FACTOR = 2
|
||||
_OV2_FPS_MIN_FRAMES = 4
|
||||
|
||||
|
||||
def _round_by_factor(n: float, factor: int) -> int:
|
||||
return round(n / factor) * factor
|
||||
|
||||
|
||||
def _ceil_by_factor(n: float, factor: int) -> int:
|
||||
import math as _math
|
||||
|
||||
|
||||
@@ -1530,52 +1530,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
result_placeholders["audio"] = audio_placeholders
|
||||
return result_placeholders
|
||||
|
||||
def _get_raw_input_ids(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
use_audio_in_video: bool = False,
|
||||
) -> list[int]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vision_bos_token = tokenizer.encode(tokenizer.vision_bos_token)[0]
|
||||
vision_eos_token = tokenizer.encode(tokenizer.vision_eos_token)[0]
|
||||
audio_bos_token = tokenizer.encode(tokenizer.audio_bos_token)[0]
|
||||
audio_eos_token = tokenizer.encode(tokenizer.audio_eos_token)[0]
|
||||
audio_token = tokenizer.encode("<|audio_pad|>")[0]
|
||||
image_token = tokenizer.encode("<|image_pad|>")[0]
|
||||
video_token = tokenizer.encode("<|video_pad|>")[0]
|
||||
|
||||
result = token_ids[:]
|
||||
if use_audio_in_video:
|
||||
while True:
|
||||
start = None
|
||||
for i in range(len(result) - 1):
|
||||
if result[i : i + 2] == [vision_bos_token, audio_bos_token]:
|
||||
start = i
|
||||
break
|
||||
if start is not None:
|
||||
end = None
|
||||
for i in range(start + 2, len(result) - 1):
|
||||
if result[i : i + 2] == [audio_eos_token, vision_eos_token]:
|
||||
end = i
|
||||
break
|
||||
if end is not None:
|
||||
result = (
|
||||
result[:start]
|
||||
+ [vision_bos_token, video_token, vision_eos_token]
|
||||
+ result[end + 2 :]
|
||||
)
|
||||
else:
|
||||
break
|
||||
|
||||
for mm_token in [audio_token, image_token, video_token]:
|
||||
compressed = []
|
||||
for x in result:
|
||||
if x != mm_token or (not compressed or compressed[-1] != mm_token):
|
||||
compressed.append(x)
|
||||
result = compressed
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixin):
|
||||
def _process_audio_input(
|
||||
|
||||
@@ -331,54 +331,6 @@ class ViTPatchGenerator(nn.Module):
|
||||
def num_skip(self):
|
||||
return self.num_cls_tokens + self.num_registers
|
||||
|
||||
def _load_embed(self, src_embed: torch.Tensor, targ_embed: nn.Parameter):
|
||||
if src_embed.shape != targ_embed.shape:
|
||||
src_size = int(math.sqrt(src_embed.shape[1]))
|
||||
|
||||
assert src_size**2 == src_embed.shape[1], (
|
||||
"Unable to interpolate non-square embedding"
|
||||
)
|
||||
|
||||
src_embed = rearrange(
|
||||
src_embed, "b (h w) c -> b c h w", h=src_size, w=src_size
|
||||
)
|
||||
src_embed = F.interpolate(
|
||||
src_embed,
|
||||
size=(self.num_rows, self.num_cols),
|
||||
mode="bicubic",
|
||||
align_corners=True,
|
||||
antialias=False,
|
||||
)
|
||||
src_embed = rearrange(src_embed, "b c h w -> b (h w) c")
|
||||
targ_embed.data.copy_(src_embed)
|
||||
|
||||
def _load_projection(
|
||||
self, src_proj_weight: torch.Tensor, targ_proj_weight: torch.Tensor
|
||||
):
|
||||
if src_proj_weight.shape != targ_proj_weight.shape:
|
||||
src_patch_size = int(math.sqrt(src_proj_weight.shape[1] // 3))
|
||||
|
||||
assert (src_patch_size**2) * 3 == src_proj_weight.shape[1], (
|
||||
"Unable to interpolate non-square patch size"
|
||||
)
|
||||
|
||||
src_proj_weight = rearrange(
|
||||
src_proj_weight,
|
||||
"b (c h w) -> b c h w",
|
||||
c=3,
|
||||
h=src_patch_size,
|
||||
w=src_patch_size,
|
||||
)
|
||||
src_proj_weight = F.interpolate(
|
||||
src_proj_weight,
|
||||
size=(self.patch_size, self.patch_size),
|
||||
mode="bicubic",
|
||||
align_corners=True,
|
||||
antialias=False,
|
||||
)
|
||||
src_proj_weight = rearrange(src_proj_weight, "b c h w -> b (c h w)")
|
||||
targ_proj_weight.data.copy_(src_proj_weight)
|
||||
|
||||
def embed_patches(self, x: torch.Tensor) -> torch.Tensor:
|
||||
patches = self.im_to_patches(x)
|
||||
patches = self.embedder(patches)
|
||||
|
||||
Reference in New Issue
Block a user