[Refactor] Remove dead code in multiple files (#49745)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-07-27 15:58:26 -04:00
committed by GitHub
parent b2f9e4caa4
commit b5bcb3ce88
22 changed files with 0 additions and 519 deletions
@@ -194,7 +194,6 @@ class HunYuanVLTextConfig(PretrainedConfig):
self.use_cache = use_cache
self.rope_theta = rope_theta
self.rope_scaling = rope_scaling
# self._rope_scaling_validation() # TODO: Need validation?
self.attention_bias = attention_bias
self.attention_dropout = attention_dropout
@@ -206,46 +205,6 @@ class HunYuanVLTextConfig(PretrainedConfig):
**kwargs,
)
def _rope_scaling_validation(self):
"""
Validate the `rope_scaling` configuration.
"""
if self.rope_scaling is None:
return
if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
raise ValueError(
"`rope_scaling` must be a dictionary with with two fields, `type` and "
f"`factor` or `type` and `alpha`, got {self.rope_scaling}"
)
rope_scaling_type = self.rope_scaling.get("type", None)
rope_scaling_factor = self.rope_scaling.get("factor", None)
rope_scaling_alpha = self.rope_scaling.get("alpha", None)
if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
raise ValueError(
"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], "
f"got {rope_scaling_type}"
)
if rope_scaling_factor is None and rope_scaling_alpha is None:
raise ValueError(
"`rope_scaling`'s factor or alpha field must be have one, "
"got both of none"
)
if rope_scaling_factor is not None and (
not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0
):
raise ValueError(
"`rope_scaling`'s factor field must be a float > 1.0, "
f"got {rope_scaling_factor}"
)
if rope_scaling_alpha is not None and (
not isinstance(rope_scaling_alpha, float) or rope_scaling_alpha <= 1.0
):
raise ValueError(
"`rope_scaling`'s alpha field must be a float > 1.0, "
f"got {rope_scaling_alpha}"
)
class HunYuanVLConfig(PretrainedConfig):
model_type = "hunyuan_vl"
@@ -72,19 +72,6 @@ class MiniCPMOProcessor(ProcessorMixin):
self.version = getattr(image_processor, "version", None)
self.pool_step = pool_step
def _safe_get_token_id(self, attr_name, default_token_str):
"""Get token ID safely, with fallback to default."""
val = getattr(self.tokenizer, attr_name, None)
if val is None:
val = self.tokenizer.convert_tokens_to_ids(default_token_str)
if val is None:
return -1
return val
def _safe_get_token_str(self, attr_name, default_token_str):
"""Get token string safely, with fallback to default."""
return getattr(self.tokenizer, attr_name, default_token_str)
def __call__(
self,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
@@ -15,7 +15,6 @@ feeds decoded frames to the processor.
import math
import regex as re
import torch
from torchvision.transforms import InterpolationMode
from transformers import AutoTokenizer, BatchFeature
@@ -554,85 +553,6 @@ class MiniMaxVLProcessor(ProcessorMixin):
self.VISION_END_TOKEN
)
def _prune_video_tokens(
self,
input_text: str,
video_segments: list[int],
video_token: str,
) -> str:
"""Prune video tokens by temporal_patch_size (e.g., 2:1).
Expects the prompt to carry exactly sum(video_segments) video tokens
— i.e. one token per *sampled* frame — then drops tokens.
"""
# If no videos or temporal_patch_size <= 1, no pruning needed
if not video_segments or self.video_processor.temporal_patch_size <= 1:
return input_text
# Split while keeping delimiters
special_tokens = [video_token]
pattern = "|".join(map(re.escape, special_tokens))
parts = re.split(f"({pattern})", input_text)
def is_timestamp(text: str) -> bool:
"""Check if text ends with timestamp format like ']<]0.0 seconds[>['"""
return (
text.endswith("seconds[>[")
or text.endswith("seconds[>[ ")
or text.endswith("seconds [>[")
or text.endswith("seconds [>[ ")
)
def extract_timestamp(text: str) -> str:
"""Extract timestamp text from the end, starting from ']<]'"""
start_index = text.rfind("]<]")
if start_index == -1:
raise ValueError(f"Failed to extract timestamp: {text}")
return text[start_index:]
# Build new text with pruned video tokens
final_parts = []
current_seg_idx = 0 # Which video segment we're in
frame_in_seg = 0 # Frame index within current segment
last_timestamp_len = 0 # Length of timestamp to potentially remove
for part in parts:
if part == video_token:
if current_seg_idx < len(video_segments):
if frame_in_seg % self.video_processor.temporal_patch_size == 0:
# Keep this video token
final_parts.append(part)
frame_in_seg += 1
if frame_in_seg >= video_segments[current_seg_idx]:
current_seg_idx += 1
frame_in_seg = 0
last_timestamp_len = 0
else:
# Skip this video token
frame_in_seg += 1
if frame_in_seg >= video_segments[current_seg_idx]:
current_seg_idx += 1
frame_in_seg = 0
# Remove the timestamp that was already appended
if last_timestamp_len > 0:
assert len(final_parts) > 0
final_parts[-1] = final_parts[-1][:-last_timestamp_len]
last_timestamp_len = 0
else:
# No more video segments, keep as is
final_parts.append(part)
last_timestamp_len = 0
else:
# Text part
final_parts.append(part)
# Check if this text ends with a timestamp
if is_timestamp(part):
last_timestamp_len = len(extract_timestamp(part))
else:
last_timestamp_len = 0
return "".join(final_parts)
def __call__(
self,
images=None,