[Refactor] Remove dead code in multiple files (#49745)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-07-27 15:58:26 -04:00
committed by GitHub
parent b2f9e4caa4
commit b5bcb3ce88
22 changed files with 0 additions and 519 deletions
@@ -307,7 +307,3 @@ class LLBf16Dotprod:
) )
if const_expr(self.use_pdl): if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents() cute.arch.griddepcontrol_launch_dependents()
def make_host_bf16(k_val: int, bs: int = 128):
return LLBf16Dotprod(k=k_val, bs=bs)
@@ -160,11 +160,6 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
return topk_ids return topk_ids
return self.global_to_physical[topk_ids] return self.global_to_physical[topk_ids]
def _map_local_to_global_ids(self, expert_topk_ids: torch.Tensor) -> torch.Tensor:
if self.local_expert_global_ids is None:
return expert_topk_ids
return self.local_expert_global_ids[expert_topk_ids]
def _do_quant( def _do_quant(
self, self,
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor], x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
@@ -254,16 +254,6 @@ class CopyNumelCounter(TorchDispatchMode):
return out return out
def _copy_missing_attrs(old: torch.Tensor, new: torch.Tensor) -> None:
"""Copies any attrs present in `old` but not in `new` to `new`"""
new_attrs = set(dir(new))
attrs_to_set = {}
for attr in dir(old):
if attr not in new_attrs:
attrs_to_set[attr] = getattr(old, attr)
set_weight_attrs(new, attrs_to_set)
class Fp8LinearMethod(LinearMethodBase): class Fp8LinearMethod(LinearMethodBase):
"""Linear method for FP8. """Linear method for FP8.
Supports loading FP8 checkpoints with static weight scale and Supports loading FP8 checkpoints with static weight scale and
@@ -58,9 +58,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase, QuantizeMethodBase,
) )
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import ( from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_input_tensor_strategy_moe, process_fp8_input_tensor_strategy_moe,
process_fp8_weight_channel_strategy, process_fp8_weight_channel_strategy,
@@ -2014,90 +2011,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
f"Expected {name} dtype {expected_dtype}, got {actual}." f"Expected {name} dtype {expected_dtype}, got {actual}."
) )
def _shuffle_weights_for_trtllm(self, layer: torch.nn.Module) -> None:
"""Shuffle weights and scales into FlashInfer TRTLLM MXFP8 layout."""
from flashinfer import (
reorder_rows_for_gated_act_gemm,
shuffle_matrix_a,
shuffle_matrix_sf_a,
)
epilogue_tile_m = 128
num_experts = layer.w13_weight.shape[0]
is_gated = self.moe.is_act_and_mul
intermediate_size_factor = 2 if is_gated else 1
w13_weight = layer.w13_weight.data
w13_scale = layer.w13_weight_scale.data
if is_gated:
# FI TRTLLM gated kernels use W31 ordering. Model checkpoints store
# gated projection as W13, so convert once before shuffling.
w13_weight = swap_w13_to_w31(w13_weight)
w13_scale = swap_w13_to_w31(w13_scale)
w13_weight_shuffled = []
w2_weight_shuffled = []
w13_scale_shuffled = []
w2_scale_shuffled = []
for i in range(num_experts):
w13_i = w13_weight[i].reshape(
intermediate_size_factor * layer.intermediate_size_per_partition, -1
)
w13_sf_i = w13_scale[i].reshape(
intermediate_size_factor * layer.intermediate_size_per_partition, -1
)
if is_gated:
# Reorder rows for gated activation layout expected by TRTLLM.
w13_i = reorder_rows_for_gated_act_gemm(w13_i.clone())
w13_sf_i = reorder_rows_for_gated_act_gemm(w13_sf_i.clone())
w13_shuffled_i = shuffle_matrix_a(w13_i.view(torch.uint8), epilogue_tile_m)
w2_shuffled_i = shuffle_matrix_a(
layer.w2_weight.data[i].view(torch.uint8), epilogue_tile_m
)
w13_weight_shuffled.append(
w13_shuffled_i.contiguous().view(MXFP8_VALUE_DTYPE)
)
w2_weight_shuffled.append(
w2_shuffled_i.contiguous().view(MXFP8_VALUE_DTYPE)
)
w13_sf_shuffled_i = shuffle_matrix_sf_a(
w13_sf_i.view(torch.uint8).reshape(
intermediate_size_factor * layer.intermediate_size_per_partition,
-1,
),
epilogue_tile_m,
)
w2_sf_shuffled_i = shuffle_matrix_sf_a(
layer.w2_weight_scale.data[i]
.view(torch.uint8)
.reshape(layer.hidden_size, -1),
epilogue_tile_m,
)
w13_scale_shuffled.append(
w13_sf_shuffled_i.contiguous().view(MXFP8_SCALE_DTYPE)
)
w2_scale_shuffled.append(
w2_sf_shuffled_i.contiguous().view(MXFP8_SCALE_DTYPE)
)
replace_parameter(
layer, "w13_weight", torch.stack(w13_weight_shuffled).contiguous()
)
replace_parameter(
layer, "w2_weight", torch.stack(w2_weight_shuffled).contiguous()
)
replace_parameter(
layer,
"w13_weight_scale",
torch.stack(w13_scale_shuffled).contiguous(),
)
replace_parameter(
layer,
"w2_weight_scale",
torch.stack(w2_scale_shuffled).contiguous(),
)
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None: def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
"""One-time MXFP8->BF16 weight dequant for the emulation path. """One-time MXFP8->BF16 weight dequant for the emulation path.
@@ -116,19 +116,6 @@ def _dequantize_nvfp4_kernel(
tl.store(output_ptr + out_indices, result, mask=block_mask[:, None]) tl.store(output_ptr + out_indices, result, mask=block_mask[:, None])
@triton.jit
def _e2m1_lookup(magnitude):
"""Lookup E2M1 float value from 3-bit magnitude."""
result = tl.where(magnitude == 1, 0.5, 0.0)
result = tl.where(magnitude == 2, 1.0, result)
result = tl.where(magnitude == 3, 1.5, result)
result = tl.where(magnitude == 4, 2.0, result)
result = tl.where(magnitude == 5, 3.0, result)
result = tl.where(magnitude == 6, 4.0, result)
result = tl.where(magnitude == 7, 6.0, result)
return result
@triton.jit @triton.jit
def _round_to_fp4(x): def _round_to_fp4(x):
"""Round float values to the nearest E2M1 representable value. """Round float values to the nearest E2M1 representable value.
@@ -257,25 +257,6 @@ class FunAudioChatAudioEncoder(nn.Module):
def dtype(self) -> torch.dtype: def dtype(self) -> torch.dtype:
return self.conv1.weight.dtype return self.conv1.weight.dtype
def _prepare_attention_mask(
self, inputs_tensor: torch.Tensor, cu_seqlens: torch.Tensor
) -> torch.Tensor | None:
if getattr(self.config, "_attn_implementation", "eager") == "flash_attention_2":
return None
seq_length = inputs_tensor.shape[0]
attention_mask = torch.full(
(1, 1, seq_length, seq_length),
torch.finfo(inputs_tensor.dtype).min,
device=inputs_tensor.device,
dtype=inputs_tensor.dtype,
)
for i in range(1, len(cu_seqlens)):
start = int(cu_seqlens[i - 1].item())
end = int(cu_seqlens[i].item())
attention_mask[..., start:end, start:end] = 0
return attention_mask
def forward( def forward(
self, self,
input_features: torch.Tensor, input_features: torch.Tensor,
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# copied from : https://github.com/huggingface/transformers # copied from : https://github.com/huggingface/transformers
import ast import ast
from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from functools import partial from functools import partial
from itertools import accumulate from itertools import accumulate
@@ -891,37 +890,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
for i in range(len(feats_per_video)) for i in range(len(feats_per_video))
) )
def _prepare_multimodal_kwargs(self, **kwargs: object):
output = defaultdict(list)
for k, v in kwargs.items():
if len(v) < 1 or len(v[0]) < 1:
continue # if empty batch of empty sample
new_k, is_video = k, False
if not k.endswith("_images") and not k.endswith("_videos"):
pass
else:
new_k, is_video = k.split("_")[:-1], k.split("_")[-1]
new_k = "_".join(new_k)
is_video = is_video == "videos"
for _sample_idx, _v in enumerate(v): # batch -> sample
if new_k not in ["pixel_values"]:
if len(output[new_k]) < _sample_idx + 1:
output[new_k].append(list())
_v = _v.detach().cpu().numpy().tolist()
output[new_k][_sample_idx] += _v
elif isinstance(_v, torch.Tensor):
if len(output[new_k]) < _sample_idx + 1:
output[new_k].append(list())
output["is_videos"].append(list())
_v = list(torch.unbind(_v, dim=0))
output[new_k][_sample_idx] += _v
output["is_videos"][_sample_idx] += [
is_video,
] * len(_v)
return dict(output)
def compute_logits( def compute_logits(
self, self,
hidden_states: torch.Tensor, hidden_states: torch.Tensor,
-24
View File
@@ -138,30 +138,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
return height, width return height, width
def _get_resize_output_image_size(
self,
*,
image_width: int,
image_height: int,
resolution_max_side: int,
) -> tuple[int, int]:
hf_processor = self.get_hf_processor()
image_processor: Idefics3ImageProcessor = hf_processor.image_processor
max_image_size = image_processor.size["longest_edge"]
if resolution_max_side > max_image_size:
raise ValueError(
"`resolution_max_side` cannot be larger than `max_image_size`"
)
height, width = image_height, image_width
# Find the output size, when rescaling the longest edge to max_len and
# preserving the aspect ratio
height, width = self._resize_output_size(
height=height, width=width, max_len=resolution_max_side
)
return height, width
def _get_image_feature_grid_size( def _get_image_feature_grid_size(
self, self,
*, *,
@@ -603,10 +603,6 @@ _OV2_FRAME_FACTOR = 2
_OV2_FPS_MIN_FRAMES = 4 _OV2_FPS_MIN_FRAMES = 4
def _round_by_factor(n: float, factor: int) -> int:
return round(n / factor) * factor
def _ceil_by_factor(n: float, factor: int) -> int: def _ceil_by_factor(n: float, factor: int) -> int:
import math as _math import math as _math
@@ -1530,52 +1530,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
result_placeholders["audio"] = audio_placeholders result_placeholders["audio"] = audio_placeholders
return result_placeholders return result_placeholders
def _get_raw_input_ids(
self,
token_ids: list[int],
use_audio_in_video: bool = False,
) -> list[int]:
tokenizer = self.info.get_tokenizer()
vision_bos_token = tokenizer.encode(tokenizer.vision_bos_token)[0]
vision_eos_token = tokenizer.encode(tokenizer.vision_eos_token)[0]
audio_bos_token = tokenizer.encode(tokenizer.audio_bos_token)[0]
audio_eos_token = tokenizer.encode(tokenizer.audio_eos_token)[0]
audio_token = tokenizer.encode("<|audio_pad|>")[0]
image_token = tokenizer.encode("<|image_pad|>")[0]
video_token = tokenizer.encode("<|video_pad|>")[0]
result = token_ids[:]
if use_audio_in_video:
while True:
start = None
for i in range(len(result) - 1):
if result[i : i + 2] == [vision_bos_token, audio_bos_token]:
start = i
break
if start is not None:
end = None
for i in range(start + 2, len(result) - 1):
if result[i : i + 2] == [audio_eos_token, vision_eos_token]:
end = i
break
if end is not None:
result = (
result[:start]
+ [vision_bos_token, video_token, vision_eos_token]
+ result[end + 2 :]
)
else:
break
for mm_token in [audio_token, image_token, video_token]:
compressed = []
for x in result:
if x != mm_token or (not compressed or compressed[-1] != mm_token):
compressed.append(x)
result = compressed
return result
class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixin): class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixin):
def _process_audio_input( def _process_audio_input(
-48
View File
@@ -331,54 +331,6 @@ class ViTPatchGenerator(nn.Module):
def num_skip(self): def num_skip(self):
return self.num_cls_tokens + self.num_registers return self.num_cls_tokens + self.num_registers
def _load_embed(self, src_embed: torch.Tensor, targ_embed: nn.Parameter):
if src_embed.shape != targ_embed.shape:
src_size = int(math.sqrt(src_embed.shape[1]))
assert src_size**2 == src_embed.shape[1], (
"Unable to interpolate non-square embedding"
)
src_embed = rearrange(
src_embed, "b (h w) c -> b c h w", h=src_size, w=src_size
)
src_embed = F.interpolate(
src_embed,
size=(self.num_rows, self.num_cols),
mode="bicubic",
align_corners=True,
antialias=False,
)
src_embed = rearrange(src_embed, "b c h w -> b (h w) c")
targ_embed.data.copy_(src_embed)
def _load_projection(
self, src_proj_weight: torch.Tensor, targ_proj_weight: torch.Tensor
):
if src_proj_weight.shape != targ_proj_weight.shape:
src_patch_size = int(math.sqrt(src_proj_weight.shape[1] // 3))
assert (src_patch_size**2) * 3 == src_proj_weight.shape[1], (
"Unable to interpolate non-square patch size"
)
src_proj_weight = rearrange(
src_proj_weight,
"b (c h w) -> b c h w",
c=3,
h=src_patch_size,
w=src_patch_size,
)
src_proj_weight = F.interpolate(
src_proj_weight,
size=(self.patch_size, self.patch_size),
mode="bicubic",
align_corners=True,
antialias=False,
)
src_proj_weight = rearrange(src_proj_weight, "b c h w -> b (c h w)")
targ_proj_weight.data.copy_(src_proj_weight)
def embed_patches(self, x: torch.Tensor) -> torch.Tensor: def embed_patches(self, x: torch.Tensor) -> torch.Tensor:
patches = self.im_to_patches(x) patches = self.im_to_patches(x)
patches = self.embedder(patches) patches = self.embedder(patches)
@@ -130,18 +130,6 @@ class HunyuanA13BReasoningParser(ReasoningParser):
return None, model_output return None, model_output
def _is_strict_increasing_subsequence(
self, subsequence: Sequence[int], sequence: Sequence[int]
) -> bool:
if not subsequence:
return False
sub_idx = 0
for num in sequence:
if sub_idx < len(subsequence) and num == subsequence[sub_idx]:
sub_idx += 1
return sub_idx == len(subsequence)
def extract_reasoning_streaming( def extract_reasoning_streaming(
self, self,
previous_text: str, previous_text: str,
-14
View File
@@ -45,20 +45,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__) logger = init_logger(__name__)
def _pop_unallowed_keys_and_warn(
dictionary: dict[str, Any], allowed_keys: set[str], err_dict_name: str
):
keys = list(dictionary.keys())
for key in keys:
if key not in allowed_keys:
dictionary.pop(key)
logger.warning_once(
f"'{key=}' is not supported by mistral-common "
f"for {err_dict_name}. It has been popped from the "
"object."
)
def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"): def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):
# SEE: https://github.com/vllm-project/vllm/pull/9951 # SEE: https://github.com/vllm-project/vllm/pull/9951
# Credits go to: @gcalmettes # Credits go to: @gcalmettes
-7
View File
@@ -37,15 +37,8 @@ do not need a transformers dependency for output parsing.
import regex as re import regex as re
# Tool call delimiter tokens as they appear in decoded text.
# Standard format: <|tool_call>call:name{args}<tool_call|>
_TOOL_CALL_START_TAG = "<|tool_call>"
_TOOL_CALL_END_TAG = "<tool_call|>"
_TOOL_RESPONSE_START_TAG = "<|tool_response>" _TOOL_RESPONSE_START_TAG = "<|tool_response>"
# Gemma4 escape token as it appears in decoded text.
_ESCAPE_TOKEN = '<|"|>'
def _parse_tool_arguments(args_str: str) -> dict[str, str]: def _parse_tool_arguments(args_str: str) -> dict[str, str]:
"""Parse tool call arguments from the Gemma4 compact format. """Parse tool call arguments from the Gemma4 compact format.
-6
View File
@@ -47,12 +47,6 @@ def _bracket_level_state(
return level, in_string, escaped return level, in_string, escaped
def _bracket_level(s: str, opening: str = "{", closing: str = "}") -> int:
"""Calculate the current level of nested brackets in a string."""
level, _, _ = _bracket_level_state(s, opening, closing)
return level
def filter_delta_text( def filter_delta_text(
delta_text: str, delta_text: str,
previous_text: str, previous_text: str,
@@ -194,7 +194,6 @@ class HunYuanVLTextConfig(PretrainedConfig):
self.use_cache = use_cache self.use_cache = use_cache
self.rope_theta = rope_theta self.rope_theta = rope_theta
self.rope_scaling = rope_scaling self.rope_scaling = rope_scaling
# self._rope_scaling_validation() # TODO: Need validation?
self.attention_bias = attention_bias self.attention_bias = attention_bias
self.attention_dropout = attention_dropout self.attention_dropout = attention_dropout
@@ -206,46 +205,6 @@ class HunYuanVLTextConfig(PretrainedConfig):
**kwargs, **kwargs,
) )
def _rope_scaling_validation(self):
"""
Validate the `rope_scaling` configuration.
"""
if self.rope_scaling is None:
return
if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
raise ValueError(
"`rope_scaling` must be a dictionary with with two fields, `type` and "
f"`factor` or `type` and `alpha`, got {self.rope_scaling}"
)
rope_scaling_type = self.rope_scaling.get("type", None)
rope_scaling_factor = self.rope_scaling.get("factor", None)
rope_scaling_alpha = self.rope_scaling.get("alpha", None)
if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
raise ValueError(
"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], "
f"got {rope_scaling_type}"
)
if rope_scaling_factor is None and rope_scaling_alpha is None:
raise ValueError(
"`rope_scaling`'s factor or alpha field must be have one, "
"got both of none"
)
if rope_scaling_factor is not None and (
not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0
):
raise ValueError(
"`rope_scaling`'s factor field must be a float > 1.0, "
f"got {rope_scaling_factor}"
)
if rope_scaling_alpha is not None and (
not isinstance(rope_scaling_alpha, float) or rope_scaling_alpha <= 1.0
):
raise ValueError(
"`rope_scaling`'s alpha field must be a float > 1.0, "
f"got {rope_scaling_alpha}"
)
class HunYuanVLConfig(PretrainedConfig): class HunYuanVLConfig(PretrainedConfig):
model_type = "hunyuan_vl" model_type = "hunyuan_vl"
@@ -72,19 +72,6 @@ class MiniCPMOProcessor(ProcessorMixin):
self.version = getattr(image_processor, "version", None) self.version = getattr(image_processor, "version", None)
self.pool_step = pool_step self.pool_step = pool_step
def _safe_get_token_id(self, attr_name, default_token_str):
"""Get token ID safely, with fallback to default."""
val = getattr(self.tokenizer, attr_name, None)
if val is None:
val = self.tokenizer.convert_tokens_to_ids(default_token_str)
if val is None:
return -1
return val
def _safe_get_token_str(self, attr_name, default_token_str):
"""Get token string safely, with fallback to default."""
return getattr(self.tokenizer, attr_name, default_token_str)
def __call__( def __call__(
self, self,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput], text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
@@ -15,7 +15,6 @@ feeds decoded frames to the processor.
import math import math
import regex as re
import torch import torch
from torchvision.transforms import InterpolationMode from torchvision.transforms import InterpolationMode
from transformers import AutoTokenizer, BatchFeature from transformers import AutoTokenizer, BatchFeature
@@ -554,85 +553,6 @@ class MiniMaxVLProcessor(ProcessorMixin):
self.VISION_END_TOKEN self.VISION_END_TOKEN
) )
def _prune_video_tokens(
self,
input_text: str,
video_segments: list[int],
video_token: str,
) -> str:
"""Prune video tokens by temporal_patch_size (e.g., 2:1).
Expects the prompt to carry exactly sum(video_segments) video tokens
— i.e. one token per *sampled* frame — then drops tokens.
"""
# If no videos or temporal_patch_size <= 1, no pruning needed
if not video_segments or self.video_processor.temporal_patch_size <= 1:
return input_text
# Split while keeping delimiters
special_tokens = [video_token]
pattern = "|".join(map(re.escape, special_tokens))
parts = re.split(f"({pattern})", input_text)
def is_timestamp(text: str) -> bool:
"""Check if text ends with timestamp format like ']<]0.0 seconds[>['"""
return (
text.endswith("seconds[>[")
or text.endswith("seconds[>[ ")
or text.endswith("seconds [>[")
or text.endswith("seconds [>[ ")
)
def extract_timestamp(text: str) -> str:
"""Extract timestamp text from the end, starting from ']<]'"""
start_index = text.rfind("]<]")
if start_index == -1:
raise ValueError(f"Failed to extract timestamp: {text}")
return text[start_index:]
# Build new text with pruned video tokens
final_parts = []
current_seg_idx = 0 # Which video segment we're in
frame_in_seg = 0 # Frame index within current segment
last_timestamp_len = 0 # Length of timestamp to potentially remove
for part in parts:
if part == video_token:
if current_seg_idx < len(video_segments):
if frame_in_seg % self.video_processor.temporal_patch_size == 0:
# Keep this video token
final_parts.append(part)
frame_in_seg += 1
if frame_in_seg >= video_segments[current_seg_idx]:
current_seg_idx += 1
frame_in_seg = 0
last_timestamp_len = 0
else:
# Skip this video token
frame_in_seg += 1
if frame_in_seg >= video_segments[current_seg_idx]:
current_seg_idx += 1
frame_in_seg = 0
# Remove the timestamp that was already appended
if last_timestamp_len > 0:
assert len(final_parts) > 0
final_parts[-1] = final_parts[-1][:-last_timestamp_len]
last_timestamp_len = 0
else:
# No more video segments, keep as is
final_parts.append(part)
last_timestamp_len = 0
else:
# Text part
final_parts.append(part)
# Check if this text ends with a timestamp
if is_timestamp(part):
last_timestamp_len = len(extract_timestamp(part))
else:
last_timestamp_len = 0
return "".join(final_parts)
def __call__( def __call__(
self, self,
images=None, images=None,
-3
View File
@@ -231,9 +231,6 @@ class PytHooks:
super().__init__() super().__init__()
self.module_to_name_map = {} self.module_to_name_map = {}
def _process_layer_params(self, module_obj):
return process_layer_params(module_obj)
def module_fwd_hook(self, module_obj, in_tensor, out_tensor): def module_fwd_hook(self, module_obj, in_tensor, out_tensor):
"""Callback function that ends the NVTX marker. """Callback function that ends the NVTX marker.
Records the module name and tensor information. Records the module name and tensor information.
@@ -592,37 +592,6 @@ def rocm_fp8_mqa_logits(
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke) return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
def _topk_indices_torch(
logits: torch.Tensor,
topk_tokens: int,
row_starts: torch.Tensor | None = None,
) -> torch.Tensor:
k = min(topk_tokens, logits.shape[-1])
values, indices = torch.topk(logits, k=k, dim=-1)
indices = indices.to(torch.int32)
indices = torch.where(
values == float("-inf"),
torch.full_like(indices, -1, dtype=torch.int32),
indices,
)
if row_starts is not None:
# Match the CUDA top_k_per_row_prefill contract: indices are local to
# each row's valid [row_start, row_end) range, not columns in the
# concatenated chunk logits matrix.
starts = row_starts.to(dtype=torch.int32).view(-1, 1)
indices = torch.where(indices < 0, indices, indices - starts)
if k == topk_tokens:
return indices
padded = torch.full(
(logits.shape[0], topk_tokens),
-1,
dtype=torch.int32,
device=logits.device,
)
padded[:, :k] = indices
return padded
def rocm_aiter_sparse_attn_indexer_fake( def rocm_aiter_sparse_attn_indexer_fake(
hidden_states: torch.Tensor, hidden_states: torch.Tensor,
k_cache_prefix: LayerNameType, k_cache_prefix: LayerNameType,
-4
View File
@@ -731,10 +731,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.encoder_cache is not None: if self.encoder_cache is not None:
self.encoder_cache.reset_encoder_cache() self.encoder_cache.reset_encoder_cache()
def _get_num_input_tokens(self, num_scheduled_tokens: int) -> int:
# SP is not supported yet.
return num_scheduled_tokens
def profile_cudagraph_memory(self) -> int: def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not. # NOTE(woosuk): It is TBD whether we keep this API or not.
return 0 return 0
-16
View File
@@ -495,22 +495,6 @@ class InputBatch:
del self._req_ids[self.num_reqs :] del self._req_ids[self.num_reqs :]
del self.req_output_token_ids[self.num_reqs :] del self.req_output_token_ids[self.num_reqs :]
def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
max_prompt_len = self.num_prompt_tokens[: self.num_reqs].max()
prompt_token_ids_cpu_tensor = torch.empty(
(self.num_reqs, max_prompt_len),
device="cpu",
dtype=torch.int64,
pin_memory=self.pin_memory,
)
prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
prompt_token_ids[:] = self.token_ids_cpu[: self.num_reqs, :max_prompt_len]
# Use the value of vocab_size as a pad since we don't have a
# token_id of this value.
for i in range(self.num_reqs):
prompt_token_ids[i, self.num_prompt_tokens[i] :] = self.vocab_size
return prompt_token_ids_cpu_tensor.to(device=self.device, non_blocking=True)
def make_lora_inputs( def make_lora_inputs(
self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray
) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]: ) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]: