[Refactor] Remove dead code in multiple files (#49745)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
@@ -307,7 +307,3 @@ class LLBf16Dotprod:
|
||||
)
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
|
||||
def make_host_bf16(k_val: int, bs: int = 128):
|
||||
return LLBf16Dotprod(k=k_val, bs=bs)
|
||||
|
||||
@@ -160,11 +160,6 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
return topk_ids
|
||||
return self.global_to_physical[topk_ids]
|
||||
|
||||
def _map_local_to_global_ids(self, expert_topk_ids: torch.Tensor) -> torch.Tensor:
|
||||
if self.local_expert_global_ids is None:
|
||||
return expert_topk_ids
|
||||
return self.local_expert_global_ids[expert_topk_ids]
|
||||
|
||||
def _do_quant(
|
||||
self,
|
||||
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
|
||||
@@ -254,16 +254,6 @@ class CopyNumelCounter(TorchDispatchMode):
|
||||
return out
|
||||
|
||||
|
||||
def _copy_missing_attrs(old: torch.Tensor, new: torch.Tensor) -> None:
|
||||
"""Copies any attrs present in `old` but not in `new` to `new`"""
|
||||
new_attrs = set(dir(new))
|
||||
attrs_to_set = {}
|
||||
for attr in dir(old):
|
||||
if attr not in new_attrs:
|
||||
attrs_to_set[attr] = getattr(old, attr)
|
||||
set_weight_attrs(new, attrs_to_set)
|
||||
|
||||
|
||||
class Fp8LinearMethod(LinearMethodBase):
|
||||
"""Linear method for FP8.
|
||||
Supports loading FP8 checkpoints with static weight scale and
|
||||
|
||||
@@ -58,9 +58,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
process_fp8_weight_channel_strategy,
|
||||
@@ -2014,90 +2011,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
f"Expected {name} dtype {expected_dtype}, got {actual}."
|
||||
)
|
||||
|
||||
def _shuffle_weights_for_trtllm(self, layer: torch.nn.Module) -> None:
|
||||
"""Shuffle weights and scales into FlashInfer TRTLLM MXFP8 layout."""
|
||||
from flashinfer import (
|
||||
reorder_rows_for_gated_act_gemm,
|
||||
shuffle_matrix_a,
|
||||
shuffle_matrix_sf_a,
|
||||
)
|
||||
|
||||
epilogue_tile_m = 128
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
is_gated = self.moe.is_act_and_mul
|
||||
intermediate_size_factor = 2 if is_gated else 1
|
||||
|
||||
w13_weight = layer.w13_weight.data
|
||||
w13_scale = layer.w13_weight_scale.data
|
||||
if is_gated:
|
||||
# FI TRTLLM gated kernels use W31 ordering. Model checkpoints store
|
||||
# gated projection as W13, so convert once before shuffling.
|
||||
w13_weight = swap_w13_to_w31(w13_weight)
|
||||
w13_scale = swap_w13_to_w31(w13_scale)
|
||||
|
||||
w13_weight_shuffled = []
|
||||
w2_weight_shuffled = []
|
||||
w13_scale_shuffled = []
|
||||
w2_scale_shuffled = []
|
||||
for i in range(num_experts):
|
||||
w13_i = w13_weight[i].reshape(
|
||||
intermediate_size_factor * layer.intermediate_size_per_partition, -1
|
||||
)
|
||||
w13_sf_i = w13_scale[i].reshape(
|
||||
intermediate_size_factor * layer.intermediate_size_per_partition, -1
|
||||
)
|
||||
if is_gated:
|
||||
# Reorder rows for gated activation layout expected by TRTLLM.
|
||||
w13_i = reorder_rows_for_gated_act_gemm(w13_i.clone())
|
||||
w13_sf_i = reorder_rows_for_gated_act_gemm(w13_sf_i.clone())
|
||||
|
||||
w13_shuffled_i = shuffle_matrix_a(w13_i.view(torch.uint8), epilogue_tile_m)
|
||||
w2_shuffled_i = shuffle_matrix_a(
|
||||
layer.w2_weight.data[i].view(torch.uint8), epilogue_tile_m
|
||||
)
|
||||
w13_weight_shuffled.append(
|
||||
w13_shuffled_i.contiguous().view(MXFP8_VALUE_DTYPE)
|
||||
)
|
||||
w2_weight_shuffled.append(
|
||||
w2_shuffled_i.contiguous().view(MXFP8_VALUE_DTYPE)
|
||||
)
|
||||
w13_sf_shuffled_i = shuffle_matrix_sf_a(
|
||||
w13_sf_i.view(torch.uint8).reshape(
|
||||
intermediate_size_factor * layer.intermediate_size_per_partition,
|
||||
-1,
|
||||
),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
w2_sf_shuffled_i = shuffle_matrix_sf_a(
|
||||
layer.w2_weight_scale.data[i]
|
||||
.view(torch.uint8)
|
||||
.reshape(layer.hidden_size, -1),
|
||||
epilogue_tile_m,
|
||||
)
|
||||
w13_scale_shuffled.append(
|
||||
w13_sf_shuffled_i.contiguous().view(MXFP8_SCALE_DTYPE)
|
||||
)
|
||||
w2_scale_shuffled.append(
|
||||
w2_sf_shuffled_i.contiguous().view(MXFP8_SCALE_DTYPE)
|
||||
)
|
||||
|
||||
replace_parameter(
|
||||
layer, "w13_weight", torch.stack(w13_weight_shuffled).contiguous()
|
||||
)
|
||||
replace_parameter(
|
||||
layer, "w2_weight", torch.stack(w2_weight_shuffled).contiguous()
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
"w13_weight_scale",
|
||||
torch.stack(w13_scale_shuffled).contiguous(),
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
"w2_weight_scale",
|
||||
torch.stack(w2_scale_shuffled).contiguous(),
|
||||
)
|
||||
|
||||
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
|
||||
"""One-time MXFP8->BF16 weight dequant for the emulation path.
|
||||
|
||||
|
||||
@@ -116,19 +116,6 @@ def _dequantize_nvfp4_kernel(
|
||||
tl.store(output_ptr + out_indices, result, mask=block_mask[:, None])
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _e2m1_lookup(magnitude):
|
||||
"""Lookup E2M1 float value from 3-bit magnitude."""
|
||||
result = tl.where(magnitude == 1, 0.5, 0.0)
|
||||
result = tl.where(magnitude == 2, 1.0, result)
|
||||
result = tl.where(magnitude == 3, 1.5, result)
|
||||
result = tl.where(magnitude == 4, 2.0, result)
|
||||
result = tl.where(magnitude == 5, 3.0, result)
|
||||
result = tl.where(magnitude == 6, 4.0, result)
|
||||
result = tl.where(magnitude == 7, 6.0, result)
|
||||
return result
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _round_to_fp4(x):
|
||||
"""Round float values to the nearest E2M1 representable value.
|
||||
|
||||
@@ -257,25 +257,6 @@ class FunAudioChatAudioEncoder(nn.Module):
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.conv1.weight.dtype
|
||||
|
||||
def _prepare_attention_mask(
|
||||
self, inputs_tensor: torch.Tensor, cu_seqlens: torch.Tensor
|
||||
) -> torch.Tensor | None:
|
||||
if getattr(self.config, "_attn_implementation", "eager") == "flash_attention_2":
|
||||
return None
|
||||
|
||||
seq_length = inputs_tensor.shape[0]
|
||||
attention_mask = torch.full(
|
||||
(1, 1, seq_length, seq_length),
|
||||
torch.finfo(inputs_tensor.dtype).min,
|
||||
device=inputs_tensor.device,
|
||||
dtype=inputs_tensor.dtype,
|
||||
)
|
||||
for i in range(1, len(cu_seqlens)):
|
||||
start = int(cu_seqlens[i - 1].item())
|
||||
end = int(cu_seqlens[i].item())
|
||||
attention_mask[..., start:end, start:end] = 0
|
||||
return attention_mask
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_features: torch.Tensor,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# copied from : https://github.com/huggingface/transformers
|
||||
import ast
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from itertools import accumulate
|
||||
@@ -891,37 +890,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
for i in range(len(feats_per_video))
|
||||
)
|
||||
|
||||
def _prepare_multimodal_kwargs(self, **kwargs: object):
|
||||
output = defaultdict(list)
|
||||
for k, v in kwargs.items():
|
||||
if len(v) < 1 or len(v[0]) < 1:
|
||||
continue # if empty batch of empty sample
|
||||
|
||||
new_k, is_video = k, False
|
||||
if not k.endswith("_images") and not k.endswith("_videos"):
|
||||
pass
|
||||
else:
|
||||
new_k, is_video = k.split("_")[:-1], k.split("_")[-1]
|
||||
new_k = "_".join(new_k)
|
||||
is_video = is_video == "videos"
|
||||
|
||||
for _sample_idx, _v in enumerate(v): # batch -> sample
|
||||
if new_k not in ["pixel_values"]:
|
||||
if len(output[new_k]) < _sample_idx + 1:
|
||||
output[new_k].append(list())
|
||||
_v = _v.detach().cpu().numpy().tolist()
|
||||
output[new_k][_sample_idx] += _v
|
||||
elif isinstance(_v, torch.Tensor):
|
||||
if len(output[new_k]) < _sample_idx + 1:
|
||||
output[new_k].append(list())
|
||||
output["is_videos"].append(list())
|
||||
_v = list(torch.unbind(_v, dim=0))
|
||||
output[new_k][_sample_idx] += _v
|
||||
output["is_videos"][_sample_idx] += [
|
||||
is_video,
|
||||
] * len(_v)
|
||||
return dict(output)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -138,30 +138,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
return height, width
|
||||
|
||||
def _get_resize_output_image_size(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
resolution_max_side: int,
|
||||
) -> tuple[int, int]:
|
||||
hf_processor = self.get_hf_processor()
|
||||
image_processor: Idefics3ImageProcessor = hf_processor.image_processor
|
||||
max_image_size = image_processor.size["longest_edge"]
|
||||
if resolution_max_side > max_image_size:
|
||||
raise ValueError(
|
||||
"`resolution_max_side` cannot be larger than `max_image_size`"
|
||||
)
|
||||
|
||||
height, width = image_height, image_width
|
||||
|
||||
# Find the output size, when rescaling the longest edge to max_len and
|
||||
# preserving the aspect ratio
|
||||
height, width = self._resize_output_size(
|
||||
height=height, width=width, max_len=resolution_max_side
|
||||
)
|
||||
return height, width
|
||||
|
||||
def _get_image_feature_grid_size(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -603,10 +603,6 @@ _OV2_FRAME_FACTOR = 2
|
||||
_OV2_FPS_MIN_FRAMES = 4
|
||||
|
||||
|
||||
def _round_by_factor(n: float, factor: int) -> int:
|
||||
return round(n / factor) * factor
|
||||
|
||||
|
||||
def _ceil_by_factor(n: float, factor: int) -> int:
|
||||
import math as _math
|
||||
|
||||
|
||||
@@ -1530,52 +1530,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
result_placeholders["audio"] = audio_placeholders
|
||||
return result_placeholders
|
||||
|
||||
def _get_raw_input_ids(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
use_audio_in_video: bool = False,
|
||||
) -> list[int]:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vision_bos_token = tokenizer.encode(tokenizer.vision_bos_token)[0]
|
||||
vision_eos_token = tokenizer.encode(tokenizer.vision_eos_token)[0]
|
||||
audio_bos_token = tokenizer.encode(tokenizer.audio_bos_token)[0]
|
||||
audio_eos_token = tokenizer.encode(tokenizer.audio_eos_token)[0]
|
||||
audio_token = tokenizer.encode("<|audio_pad|>")[0]
|
||||
image_token = tokenizer.encode("<|image_pad|>")[0]
|
||||
video_token = tokenizer.encode("<|video_pad|>")[0]
|
||||
|
||||
result = token_ids[:]
|
||||
if use_audio_in_video:
|
||||
while True:
|
||||
start = None
|
||||
for i in range(len(result) - 1):
|
||||
if result[i : i + 2] == [vision_bos_token, audio_bos_token]:
|
||||
start = i
|
||||
break
|
||||
if start is not None:
|
||||
end = None
|
||||
for i in range(start + 2, len(result) - 1):
|
||||
if result[i : i + 2] == [audio_eos_token, vision_eos_token]:
|
||||
end = i
|
||||
break
|
||||
if end is not None:
|
||||
result = (
|
||||
result[:start]
|
||||
+ [vision_bos_token, video_token, vision_eos_token]
|
||||
+ result[end + 2 :]
|
||||
)
|
||||
else:
|
||||
break
|
||||
|
||||
for mm_token in [audio_token, image_token, video_token]:
|
||||
compressed = []
|
||||
for x in result:
|
||||
if x != mm_token or (not compressed or compressed[-1] != mm_token):
|
||||
compressed.append(x)
|
||||
result = compressed
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixin):
|
||||
def _process_audio_input(
|
||||
|
||||
@@ -331,54 +331,6 @@ class ViTPatchGenerator(nn.Module):
|
||||
def num_skip(self):
|
||||
return self.num_cls_tokens + self.num_registers
|
||||
|
||||
def _load_embed(self, src_embed: torch.Tensor, targ_embed: nn.Parameter):
|
||||
if src_embed.shape != targ_embed.shape:
|
||||
src_size = int(math.sqrt(src_embed.shape[1]))
|
||||
|
||||
assert src_size**2 == src_embed.shape[1], (
|
||||
"Unable to interpolate non-square embedding"
|
||||
)
|
||||
|
||||
src_embed = rearrange(
|
||||
src_embed, "b (h w) c -> b c h w", h=src_size, w=src_size
|
||||
)
|
||||
src_embed = F.interpolate(
|
||||
src_embed,
|
||||
size=(self.num_rows, self.num_cols),
|
||||
mode="bicubic",
|
||||
align_corners=True,
|
||||
antialias=False,
|
||||
)
|
||||
src_embed = rearrange(src_embed, "b c h w -> b (h w) c")
|
||||
targ_embed.data.copy_(src_embed)
|
||||
|
||||
def _load_projection(
|
||||
self, src_proj_weight: torch.Tensor, targ_proj_weight: torch.Tensor
|
||||
):
|
||||
if src_proj_weight.shape != targ_proj_weight.shape:
|
||||
src_patch_size = int(math.sqrt(src_proj_weight.shape[1] // 3))
|
||||
|
||||
assert (src_patch_size**2) * 3 == src_proj_weight.shape[1], (
|
||||
"Unable to interpolate non-square patch size"
|
||||
)
|
||||
|
||||
src_proj_weight = rearrange(
|
||||
src_proj_weight,
|
||||
"b (c h w) -> b c h w",
|
||||
c=3,
|
||||
h=src_patch_size,
|
||||
w=src_patch_size,
|
||||
)
|
||||
src_proj_weight = F.interpolate(
|
||||
src_proj_weight,
|
||||
size=(self.patch_size, self.patch_size),
|
||||
mode="bicubic",
|
||||
align_corners=True,
|
||||
antialias=False,
|
||||
)
|
||||
src_proj_weight = rearrange(src_proj_weight, "b c h w -> b (c h w)")
|
||||
targ_proj_weight.data.copy_(src_proj_weight)
|
||||
|
||||
def embed_patches(self, x: torch.Tensor) -> torch.Tensor:
|
||||
patches = self.im_to_patches(x)
|
||||
patches = self.embedder(patches)
|
||||
|
||||
@@ -130,18 +130,6 @@ class HunyuanA13BReasoningParser(ReasoningParser):
|
||||
|
||||
return None, model_output
|
||||
|
||||
def _is_strict_increasing_subsequence(
|
||||
self, subsequence: Sequence[int], sequence: Sequence[int]
|
||||
) -> bool:
|
||||
if not subsequence:
|
||||
return False
|
||||
|
||||
sub_idx = 0
|
||||
for num in sequence:
|
||||
if sub_idx < len(subsequence) and num == subsequence[sub_idx]:
|
||||
sub_idx += 1
|
||||
return sub_idx == len(subsequence)
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
|
||||
@@ -45,20 +45,6 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _pop_unallowed_keys_and_warn(
|
||||
dictionary: dict[str, Any], allowed_keys: set[str], err_dict_name: str
|
||||
):
|
||||
keys = list(dictionary.keys())
|
||||
for key in keys:
|
||||
if key not in allowed_keys:
|
||||
dictionary.pop(key)
|
||||
logger.warning_once(
|
||||
f"'{key=}' is not supported by mistral-common "
|
||||
f"for {err_dict_name}. It has been popped from the "
|
||||
"object."
|
||||
)
|
||||
|
||||
|
||||
def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):
|
||||
# SEE: https://github.com/vllm-project/vllm/pull/9951
|
||||
# Credits go to: @gcalmettes
|
||||
|
||||
@@ -37,15 +37,8 @@ do not need a transformers dependency for output parsing.
|
||||
|
||||
import regex as re
|
||||
|
||||
# Tool call delimiter tokens as they appear in decoded text.
|
||||
# Standard format: <|tool_call>call:name{args}<tool_call|>
|
||||
_TOOL_CALL_START_TAG = "<|tool_call>"
|
||||
_TOOL_CALL_END_TAG = "<tool_call|>"
|
||||
_TOOL_RESPONSE_START_TAG = "<|tool_response>"
|
||||
|
||||
# Gemma4 escape token as it appears in decoded text.
|
||||
_ESCAPE_TOKEN = '<|"|>'
|
||||
|
||||
|
||||
def _parse_tool_arguments(args_str: str) -> dict[str, str]:
|
||||
"""Parse tool call arguments from the Gemma4 compact format.
|
||||
|
||||
@@ -47,12 +47,6 @@ def _bracket_level_state(
|
||||
return level, in_string, escaped
|
||||
|
||||
|
||||
def _bracket_level(s: str, opening: str = "{", closing: str = "}") -> int:
|
||||
"""Calculate the current level of nested brackets in a string."""
|
||||
level, _, _ = _bracket_level_state(s, opening, closing)
|
||||
return level
|
||||
|
||||
|
||||
def filter_delta_text(
|
||||
delta_text: str,
|
||||
previous_text: str,
|
||||
|
||||
@@ -194,7 +194,6 @@ class HunYuanVLTextConfig(PretrainedConfig):
|
||||
self.use_cache = use_cache
|
||||
self.rope_theta = rope_theta
|
||||
self.rope_scaling = rope_scaling
|
||||
# self._rope_scaling_validation() # TODO: Need validation?
|
||||
self.attention_bias = attention_bias
|
||||
self.attention_dropout = attention_dropout
|
||||
|
||||
@@ -206,46 +205,6 @@ class HunYuanVLTextConfig(PretrainedConfig):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _rope_scaling_validation(self):
|
||||
"""
|
||||
Validate the `rope_scaling` configuration.
|
||||
"""
|
||||
if self.rope_scaling is None:
|
||||
return
|
||||
|
||||
if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
|
||||
raise ValueError(
|
||||
"`rope_scaling` must be a dictionary with with two fields, `type` and "
|
||||
f"`factor` or `type` and `alpha`, got {self.rope_scaling}"
|
||||
)
|
||||
rope_scaling_type = self.rope_scaling.get("type", None)
|
||||
rope_scaling_factor = self.rope_scaling.get("factor", None)
|
||||
rope_scaling_alpha = self.rope_scaling.get("alpha", None)
|
||||
if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
|
||||
raise ValueError(
|
||||
"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], "
|
||||
f"got {rope_scaling_type}"
|
||||
)
|
||||
if rope_scaling_factor is None and rope_scaling_alpha is None:
|
||||
raise ValueError(
|
||||
"`rope_scaling`'s factor or alpha field must be have one, "
|
||||
"got both of none"
|
||||
)
|
||||
if rope_scaling_factor is not None and (
|
||||
not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0
|
||||
):
|
||||
raise ValueError(
|
||||
"`rope_scaling`'s factor field must be a float > 1.0, "
|
||||
f"got {rope_scaling_factor}"
|
||||
)
|
||||
if rope_scaling_alpha is not None and (
|
||||
not isinstance(rope_scaling_alpha, float) or rope_scaling_alpha <= 1.0
|
||||
):
|
||||
raise ValueError(
|
||||
"`rope_scaling`'s alpha field must be a float > 1.0, "
|
||||
f"got {rope_scaling_alpha}"
|
||||
)
|
||||
|
||||
|
||||
class HunYuanVLConfig(PretrainedConfig):
|
||||
model_type = "hunyuan_vl"
|
||||
|
||||
@@ -72,19 +72,6 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
self.version = getattr(image_processor, "version", None)
|
||||
self.pool_step = pool_step
|
||||
|
||||
def _safe_get_token_id(self, attr_name, default_token_str):
|
||||
"""Get token ID safely, with fallback to default."""
|
||||
val = getattr(self.tokenizer, attr_name, None)
|
||||
if val is None:
|
||||
val = self.tokenizer.convert_tokens_to_ids(default_token_str)
|
||||
if val is None:
|
||||
return -1
|
||||
return val
|
||||
|
||||
def _safe_get_token_str(self, attr_name, default_token_str):
|
||||
"""Get token string safely, with fallback to default."""
|
||||
return getattr(self.tokenizer, attr_name, default_token_str)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
|
||||
|
||||
@@ -15,7 +15,6 @@ feeds decoded frames to the processor.
|
||||
|
||||
import math
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torchvision.transforms import InterpolationMode
|
||||
from transformers import AutoTokenizer, BatchFeature
|
||||
@@ -554,85 +553,6 @@ class MiniMaxVLProcessor(ProcessorMixin):
|
||||
self.VISION_END_TOKEN
|
||||
)
|
||||
|
||||
def _prune_video_tokens(
|
||||
self,
|
||||
input_text: str,
|
||||
video_segments: list[int],
|
||||
video_token: str,
|
||||
) -> str:
|
||||
"""Prune video tokens by temporal_patch_size (e.g., 2:1).
|
||||
|
||||
Expects the prompt to carry exactly sum(video_segments) video tokens
|
||||
— i.e. one token per *sampled* frame — then drops tokens.
|
||||
"""
|
||||
# If no videos or temporal_patch_size <= 1, no pruning needed
|
||||
if not video_segments or self.video_processor.temporal_patch_size <= 1:
|
||||
return input_text
|
||||
|
||||
# Split while keeping delimiters
|
||||
special_tokens = [video_token]
|
||||
pattern = "|".join(map(re.escape, special_tokens))
|
||||
parts = re.split(f"({pattern})", input_text)
|
||||
|
||||
def is_timestamp(text: str) -> bool:
|
||||
"""Check if text ends with timestamp format like ']<]0.0 seconds[>['"""
|
||||
return (
|
||||
text.endswith("seconds[>[")
|
||||
or text.endswith("seconds[>[ ")
|
||||
or text.endswith("seconds [>[")
|
||||
or text.endswith("seconds [>[ ")
|
||||
)
|
||||
|
||||
def extract_timestamp(text: str) -> str:
|
||||
"""Extract timestamp text from the end, starting from ']<]'"""
|
||||
start_index = text.rfind("]<]")
|
||||
if start_index == -1:
|
||||
raise ValueError(f"Failed to extract timestamp: {text}")
|
||||
return text[start_index:]
|
||||
|
||||
# Build new text with pruned video tokens
|
||||
final_parts = []
|
||||
current_seg_idx = 0 # Which video segment we're in
|
||||
frame_in_seg = 0 # Frame index within current segment
|
||||
last_timestamp_len = 0 # Length of timestamp to potentially remove
|
||||
|
||||
for part in parts:
|
||||
if part == video_token:
|
||||
if current_seg_idx < len(video_segments):
|
||||
if frame_in_seg % self.video_processor.temporal_patch_size == 0:
|
||||
# Keep this video token
|
||||
final_parts.append(part)
|
||||
frame_in_seg += 1
|
||||
if frame_in_seg >= video_segments[current_seg_idx]:
|
||||
current_seg_idx += 1
|
||||
frame_in_seg = 0
|
||||
last_timestamp_len = 0
|
||||
else:
|
||||
# Skip this video token
|
||||
frame_in_seg += 1
|
||||
if frame_in_seg >= video_segments[current_seg_idx]:
|
||||
current_seg_idx += 1
|
||||
frame_in_seg = 0
|
||||
# Remove the timestamp that was already appended
|
||||
if last_timestamp_len > 0:
|
||||
assert len(final_parts) > 0
|
||||
final_parts[-1] = final_parts[-1][:-last_timestamp_len]
|
||||
last_timestamp_len = 0
|
||||
else:
|
||||
# No more video segments, keep as is
|
||||
final_parts.append(part)
|
||||
last_timestamp_len = 0
|
||||
else:
|
||||
# Text part
|
||||
final_parts.append(part)
|
||||
# Check if this text ends with a timestamp
|
||||
if is_timestamp(part):
|
||||
last_timestamp_len = len(extract_timestamp(part))
|
||||
else:
|
||||
last_timestamp_len = 0
|
||||
|
||||
return "".join(final_parts)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images=None,
|
||||
|
||||
@@ -231,9 +231,6 @@ class PytHooks:
|
||||
super().__init__()
|
||||
self.module_to_name_map = {}
|
||||
|
||||
def _process_layer_params(self, module_obj):
|
||||
return process_layer_params(module_obj)
|
||||
|
||||
def module_fwd_hook(self, module_obj, in_tensor, out_tensor):
|
||||
"""Callback function that ends the NVTX marker.
|
||||
Records the module name and tensor information.
|
||||
|
||||
@@ -592,37 +592,6 @@ def rocm_fp8_mqa_logits(
|
||||
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
|
||||
|
||||
|
||||
def _topk_indices_torch(
|
||||
logits: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
row_starts: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
k = min(topk_tokens, logits.shape[-1])
|
||||
values, indices = torch.topk(logits, k=k, dim=-1)
|
||||
indices = indices.to(torch.int32)
|
||||
indices = torch.where(
|
||||
values == float("-inf"),
|
||||
torch.full_like(indices, -1, dtype=torch.int32),
|
||||
indices,
|
||||
)
|
||||
if row_starts is not None:
|
||||
# Match the CUDA top_k_per_row_prefill contract: indices are local to
|
||||
# each row's valid [row_start, row_end) range, not columns in the
|
||||
# concatenated chunk logits matrix.
|
||||
starts = row_starts.to(dtype=torch.int32).view(-1, 1)
|
||||
indices = torch.where(indices < 0, indices, indices - starts)
|
||||
if k == topk_tokens:
|
||||
return indices
|
||||
padded = torch.full(
|
||||
(logits.shape[0], topk_tokens),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=logits.device,
|
||||
)
|
||||
padded[:, :k] = indices
|
||||
return padded
|
||||
|
||||
|
||||
def rocm_aiter_sparse_attn_indexer_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
k_cache_prefix: LayerNameType,
|
||||
|
||||
@@ -731,10 +731,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.reset_encoder_cache()
|
||||
|
||||
def _get_num_input_tokens(self, num_scheduled_tokens: int) -> int:
|
||||
# SP is not supported yet.
|
||||
return num_scheduled_tokens
|
||||
|
||||
def profile_cudagraph_memory(self) -> int:
|
||||
# NOTE(woosuk): It is TBD whether we keep this API or not.
|
||||
return 0
|
||||
|
||||
@@ -495,22 +495,6 @@ class InputBatch:
|
||||
del self._req_ids[self.num_reqs :]
|
||||
del self.req_output_token_ids[self.num_reqs :]
|
||||
|
||||
def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
|
||||
max_prompt_len = self.num_prompt_tokens[: self.num_reqs].max()
|
||||
prompt_token_ids_cpu_tensor = torch.empty(
|
||||
(self.num_reqs, max_prompt_len),
|
||||
device="cpu",
|
||||
dtype=torch.int64,
|
||||
pin_memory=self.pin_memory,
|
||||
)
|
||||
prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
|
||||
prompt_token_ids[:] = self.token_ids_cpu[: self.num_reqs, :max_prompt_len]
|
||||
# Use the value of vocab_size as a pad since we don't have a
|
||||
# token_id of this value.
|
||||
for i in range(self.num_reqs):
|
||||
prompt_token_ids[i, self.num_prompt_tokens[i] :] = self.vocab_size
|
||||
return prompt_token_ids_cpu_tensor.to(device=self.device, non_blocking=True)
|
||||
|
||||
def make_lora_inputs(
|
||||
self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]:
|
||||
|
||||
Reference in New Issue
Block a user