[Refactor] Remove dead code multiple places (#58002)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -453,35 +453,6 @@ def normalize_batched_scales_shape(
|
||||
return scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_topk_ids_weights_kernel(
|
||||
topk_ids_ptr,
|
||||
topk_weights_ptr,
|
||||
output_ptr,
|
||||
n_elements,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr, # triton metadata
|
||||
):
|
||||
pid = tl.program_id(axis=0)
|
||||
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < n_elements
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.extra.cuda.gdc_wait()
|
||||
expert_id = tl.load(topk_ids_ptr + offsets, mask=mask, other=0).to(tl.int32)
|
||||
expert_id_shifted = expert_id << 16
|
||||
|
||||
weight = tl.load(topk_weights_ptr + offsets, mask=mask, other=0.0)
|
||||
weight_bf16 = weight.to(tl.bfloat16)
|
||||
weight_int16 = weight_bf16.to(tl.int16, bitcast=True)
|
||||
|
||||
weight_int32 = weight_int16.to(tl.int32) & 0xFFFF
|
||||
|
||||
packed = expert_id_shifted | weight_int32
|
||||
tl.store(output_ptr + offsets, packed, mask=mask)
|
||||
|
||||
|
||||
def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
|
||||
"""Estimate FlashInfer's MoE autotuning maximum token count.
|
||||
|
||||
|
||||
@@ -59,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -1961,74 +1960,3 @@ direct_register_custom_op(
|
||||
op_func=qwen_gdn_attention_core_fused_norm_packed,
|
||||
mutates_args=["core_attn_out"],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_gdn_gating_kernel(
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
BLK_HEADS: tl.constexpr,
|
||||
):
|
||||
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
|
||||
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
|
||||
mask = head_off < NUM_HEADS
|
||||
blk_A_log = tl.load(A_log + head_off, mask=mask)
|
||||
blk_a = tl.load(a + off, mask=mask)
|
||||
blk_b = tl.load(b + off, mask=mask)
|
||||
blk_bias = tl.load(dt_bias + head_off, mask=mask)
|
||||
# If the model is loaded in fp16, without the .float() here, A might be -inf
|
||||
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
|
||||
softplus_x = tl.where(
|
||||
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
|
||||
)
|
||||
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
|
||||
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
|
||||
# compute beta_output = sigmoid(b)
|
||||
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
|
||||
tl.store(
|
||||
beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
def fused_gdn_gating(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused computation of g and beta for Gated Delta Net.
|
||||
g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
||||
beta_output = b.sigmoid()
|
||||
TODO maybe use torch.compile to replace this triton kernel
|
||||
"""
|
||||
batch, num_heads = a.shape
|
||||
seq_len = 1
|
||||
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
|
||||
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
|
||||
beta_output = torch.empty(1, batch, num_heads, dtype=b.dtype, device=b.device)
|
||||
fused_gdn_gating_kernel[grid](
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
num_heads,
|
||||
beta,
|
||||
threshold,
|
||||
8,
|
||||
num_warps=1,
|
||||
)
|
||||
return g, beta_output
|
||||
|
||||
@@ -12,7 +12,6 @@ import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
@@ -23,7 +22,7 @@ import huggingface_hub.constants
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import torch
|
||||
from safetensors.torch import load, load_file, safe_open, save_file
|
||||
from safetensors.torch import load, load_file, safe_open
|
||||
from tqdm.auto import tqdm
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
@@ -186,56 +185,6 @@ def maybe_download_from_modelscope(
|
||||
return None
|
||||
|
||||
|
||||
def _shared_pointers(tensors):
|
||||
ptrs = defaultdict(list)
|
||||
for k, v in tensors.items():
|
||||
ptrs[v.data_ptr()].append(k)
|
||||
failing = []
|
||||
for _, names in ptrs.items():
|
||||
if len(names) > 1:
|
||||
failing.append(names)
|
||||
return failing
|
||||
|
||||
|
||||
def convert_bin_to_safetensor_file(
|
||||
pt_filename: str,
|
||||
sf_filename: str,
|
||||
) -> None:
|
||||
loaded = torch.load(pt_filename, map_location="cpu", weights_only=True)
|
||||
if "state_dict" in loaded:
|
||||
loaded = loaded["state_dict"]
|
||||
shared = _shared_pointers(loaded)
|
||||
for shared_weights in shared:
|
||||
for name in shared_weights[1:]:
|
||||
loaded.pop(name)
|
||||
|
||||
# For tensors to be contiguous
|
||||
loaded = {k: v.contiguous() for k, v in loaded.items()}
|
||||
|
||||
dirname = os.path.dirname(sf_filename)
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
save_file(loaded, sf_filename, metadata={"format": "pt"})
|
||||
|
||||
# check file size
|
||||
sf_size = os.stat(sf_filename).st_size
|
||||
pt_size = os.stat(pt_filename).st_size
|
||||
if (sf_size - pt_size) / pt_size > 0.01:
|
||||
raise RuntimeError(
|
||||
f"""The file size different is more than 1%:
|
||||
- {sf_filename}: {sf_size}
|
||||
- {pt_filename}: {pt_size}
|
||||
"""
|
||||
)
|
||||
|
||||
# check if the tensors are the same
|
||||
reloaded = load_file(sf_filename)
|
||||
for k in loaded:
|
||||
pt_tensor = loaded[k]
|
||||
sf_tensor = reloaded[k]
|
||||
if not torch.equal(pt_tensor, sf_tensor):
|
||||
raise RuntimeError(f"The output tensors do not match for key {k}")
|
||||
|
||||
|
||||
# TODO(woosuk): Move this to other place.
|
||||
def get_quant_config(
|
||||
model_config: ModelConfig, load_config: LoadConfig
|
||||
@@ -1313,29 +1262,6 @@ def composed_weight_loader(
|
||||
return composed_loader
|
||||
|
||||
|
||||
def initialize_dummy_weights(
|
||||
model: torch.nn.Module,
|
||||
model_config: ModelConfig,
|
||||
low: float = -1e-3,
|
||||
high: float = 1e-3,
|
||||
seed: int = 1234,
|
||||
) -> None:
|
||||
"""Initialize model weights with random values.
|
||||
|
||||
The model weights must be randomly initialized for accurate performance
|
||||
measurements. Additionally, the model weights should not cause NaNs in the
|
||||
forward pass. We empirically found that initializing the weights with
|
||||
values between -1e-3 and 1e-3 works well for most models.
|
||||
|
||||
We use per-parameter random seed, so that dummy weights are consistent,
|
||||
even if the model is partitioned across multiple devices. When the seed
|
||||
is fixed, the random values generated by this function only depends on
|
||||
the parameter's number of elements and its data type.
|
||||
"""
|
||||
for param in model.state_dict().values():
|
||||
initialize_single_dummy_weight(param, low, high, seed)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def initialize_single_dummy_weight(
|
||||
param: torch.Tensor,
|
||||
|
||||
@@ -114,34 +114,6 @@ class InternVisionEmbeddings(nn.Module):
|
||||
return embeddings
|
||||
|
||||
|
||||
class InternVisionPatchModel(nn.Module):
|
||||
def __init__(self, config: PretrainedConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embeddings = InternVisionEmbeddings(config)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embeddings
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor | None = None,
|
||||
pixel_embeds: torch.Tensor | None = None,
|
||||
) -> torch.FloatTensor:
|
||||
if pixel_values is None and pixel_embeds is None:
|
||||
raise ValueError("You have to specify pixel_values or pixel_embeds")
|
||||
|
||||
if pixel_embeds is not None:
|
||||
hidden_states = pixel_embeds
|
||||
elif pixel_values is not None:
|
||||
if pixel_values.ndim == 4:
|
||||
hidden_states = self.embeddings(pixel_values)
|
||||
else:
|
||||
raise ValueError(f"wrong pixel_values size: {pixel_values.shape}")
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class InternParallelAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
# limitations under the License.
|
||||
import math
|
||||
from collections.abc import Hashable, Iterable, Mapping
|
||||
from itertools import tee
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import torch
|
||||
@@ -1041,25 +1040,6 @@ class Llama4ForConditionalGeneration(
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def separate_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
prefix: str,
|
||||
) -> tuple[Iterable[tuple[str, torch.Tensor]], Iterable[tuple[str, torch.Tensor]]]:
|
||||
weights1, weights2 = tee(weights, 2)
|
||||
|
||||
def get_prefix_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, data in weights1:
|
||||
if name.startswith(prefix):
|
||||
yield (name, data)
|
||||
|
||||
def get_other_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, data in weights2:
|
||||
if not name.startswith(prefix):
|
||||
yield (name, data)
|
||||
|
||||
return get_prefix_weights(), get_other_weights()
|
||||
|
||||
def _consolidate_qkv_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
|
||||
@@ -23,47 +23,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.llama import (
|
||||
LlamaDecoderLayer,
|
||||
LlamaForCausalLM,
|
||||
LlamaModel,
|
||||
)
|
||||
|
||||
|
||||
class TeleFLMModel(LlamaModel):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
layer_type: type[nn.Module] = LlamaDecoderLayer,
|
||||
):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type)
|
||||
"""
|
||||
This implementation is based on the µScaling paper presented at
|
||||
the ICLR 2025 Workshop:
|
||||
NanoLM: An Affordable LLM Study Benchmark \
|
||||
via Accurate Loss Prediction across Scales
|
||||
by Yiqun Yao et al.
|
||||
Available at: https://openreview.net/forum?id=IwaPYg1SCA
|
||||
arXiv preprint: https://arxiv.org/abs/2304.06875
|
||||
"""
|
||||
self.use_mup = self.config.use_mup
|
||||
if self.use_mup:
|
||||
self.input_mult = self.config.input_mult
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
embedding = self.embed_tokens(input_ids)
|
||||
if self.use_mup:
|
||||
embedding = embedding * self.input_mult
|
||||
return embedding
|
||||
|
||||
|
||||
class TeleFLMForCausalLM(LlamaForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
Reference in New Issue
Block a user