[Refactor] Remove dead code multiple places (#58002)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -6,10 +6,8 @@
|
||||
import json
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import regex as re
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
model_serializer,
|
||||
)
|
||||
@@ -21,7 +19,6 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -261,53 +258,6 @@ class FunctionDefinition(OpenAIBaseModel):
|
||||
return data
|
||||
|
||||
|
||||
# extra="forbid" is a workaround to have kwargs as a field,
|
||||
# see https://github.com/pydantic/pydantic/issues/3125
|
||||
class LogitsProcessorConstructor(BaseModel):
|
||||
qualname: str
|
||||
args: list[Any] | None = None
|
||||
kwargs: dict[str, Any] | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
LogitsProcessors = list[str | LogitsProcessorConstructor]
|
||||
|
||||
|
||||
def get_logits_processors(
|
||||
processors: LogitsProcessors | None, pattern: str | None
|
||||
) -> list[Any] | None:
|
||||
if processors and pattern:
|
||||
logits_processors = []
|
||||
for processor in processors:
|
||||
qualname = processor if isinstance(processor, str) else processor.qualname
|
||||
if not re.match(pattern, qualname):
|
||||
raise ValueError(
|
||||
f"Logits processor '{qualname}' is not allowed by this "
|
||||
"server. See --logits-processor-pattern engine argument "
|
||||
"for more information."
|
||||
)
|
||||
try:
|
||||
logits_processor = resolve_obj_by_qualname(qualname)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
f"Logits processor '{qualname}' could not be resolved: {e}"
|
||||
) from e
|
||||
if isinstance(processor, LogitsProcessorConstructor):
|
||||
logits_processor = logits_processor(
|
||||
*processor.args or [], **processor.kwargs or {}
|
||||
)
|
||||
logits_processors.append(logits_processor)
|
||||
return logits_processors
|
||||
elif processors:
|
||||
raise ValueError(
|
||||
"The `logits_processors` argument is not supported by this "
|
||||
"server. See --logits-processor-pattern engine argument "
|
||||
"for more information."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
class FunctionCall(OpenAIBaseModel):
|
||||
# Internal field to preserve native tool call ID from tool parser.
|
||||
# Excluded from serialization to maintain OpenAI API compatibility
|
||||
|
||||
+1
-106
@@ -1,12 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Sequence
|
||||
from collections.abc import Callable
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
LogitsProcessor: TypeAlias = (
|
||||
Callable[[list[int], torch.Tensor], torch.Tensor]
|
||||
| Callable[[list[int], list[int], torch.Tensor], torch.Tensor]
|
||||
@@ -16,106 +14,3 @@ of previously generated tokens, the logits tensor
|
||||
for the next token and, optionally, prompt tokens as a
|
||||
first argument, and returns a modified tensor of logits
|
||||
to sample from."""
|
||||
|
||||
|
||||
def get_bad_words_logits_processors(
|
||||
bad_words: list[str], tokenizer: TokenizerLike
|
||||
) -> list[LogitsProcessor]:
|
||||
bad_words_ids: list[list[int]] = list()
|
||||
|
||||
for bad_word in bad_words:
|
||||
# To prohibit words both at the beginning
|
||||
# and in the middle of text
|
||||
# (related to add_prefix_space tokenizer parameter)
|
||||
for add_prefix_space in [False, True]:
|
||||
prefix = " " if add_prefix_space else ""
|
||||
prompt = prefix + bad_word.lstrip()
|
||||
|
||||
prompt_token_ids = tokenizer.encode(text=prompt, add_special_tokens=False)
|
||||
|
||||
# If no space at the beginning
|
||||
# or if prefix space produces a new word token
|
||||
if (not add_prefix_space) or (
|
||||
add_prefix_space
|
||||
and prompt_token_ids[0] != bad_words_ids[-1][0]
|
||||
and len(prompt_token_ids) == len(bad_words_ids[-1])
|
||||
):
|
||||
bad_words_ids.append(prompt_token_ids)
|
||||
|
||||
return [NoBadWordsLogitsProcessor(bad_words_ids=bad_words_ids)]
|
||||
|
||||
|
||||
class NoBadWordsLogitsProcessor:
|
||||
_SMALLEST_LOGIT = float("-inf")
|
||||
_NEUTRAL_LOGIT = 0.0
|
||||
|
||||
def __init__(self, bad_words_ids: list[list[int]]):
|
||||
self.bad_words_ids = bad_words_ids
|
||||
self.word_bias: torch.FloatTensor = None
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
past_tokens_ids: Sequence[int],
|
||||
logits: torch.FloatTensor,
|
||||
) -> torch.Tensor:
|
||||
if self.word_bias is None:
|
||||
self._init_word_bias(logits=logits)
|
||||
|
||||
last_token_bias = torch.zeros_like(logits)
|
||||
|
||||
for bad_word_ids in self.bad_words_ids:
|
||||
if len(bad_word_ids) == 1: # 1-token words already processed
|
||||
continue
|
||||
|
||||
if len(bad_word_ids) > len(past_tokens_ids) + 1:
|
||||
continue
|
||||
|
||||
prefix_length = len(bad_word_ids) - 1
|
||||
last_token_id = bad_word_ids[-1]
|
||||
actual_prefix = past_tokens_ids[-prefix_length:]
|
||||
expected_prefix = bad_word_ids[:prefix_length]
|
||||
|
||||
assert len(actual_prefix) == len(expected_prefix)
|
||||
|
||||
is_match = tuple(actual_prefix) == tuple(expected_prefix)
|
||||
last_token_bias[last_token_id] += (
|
||||
self._SMALLEST_LOGIT if is_match else self._NEUTRAL_LOGIT
|
||||
)
|
||||
|
||||
logits = logits + self.word_bias + last_token_bias
|
||||
|
||||
return logits
|
||||
|
||||
def _init_word_bias(self, logits: torch.FloatTensor) -> None:
|
||||
# Code based on NoBadWordsLogitsProcessor and SequenceBiasLogitsProcessor # noqa: E501
|
||||
# from https://github.com/huggingface/transformers/blob/main/src/transformers/generation/logits_process.py
|
||||
|
||||
vocab_size = logits.shape[-1]
|
||||
|
||||
self._check_token_ids_bounds(vocab_size=vocab_size)
|
||||
|
||||
self.word_bias = torch.zeros(
|
||||
(vocab_size,), dtype=torch.float, device=logits.device
|
||||
)
|
||||
|
||||
for bad_word_ids in self.bad_words_ids:
|
||||
if len(bad_word_ids) == 1:
|
||||
bad_word_id = bad_word_ids[-1]
|
||||
self.word_bias[bad_word_id] = self._SMALLEST_LOGIT
|
||||
|
||||
def _check_token_ids_bounds(self, vocab_size: int) -> None:
|
||||
invalid_token_ids = []
|
||||
|
||||
for bad_word_ids in self.bad_words_ids:
|
||||
for token_id in bad_word_ids:
|
||||
if token_id < 0 or token_id >= vocab_size:
|
||||
invalid_token_ids.append(token_id)
|
||||
|
||||
if len(invalid_token_ids) > 0:
|
||||
raise ValueError(
|
||||
f"The model vocabulary size is {vocab_size},"
|
||||
f" but the following tokens"
|
||||
f" were specified as bad: {invalid_token_ids}."
|
||||
f" All token id values should be integers satisfying:"
|
||||
f" 0 <= token_id < {vocab_size}."
|
||||
)
|
||||
|
||||
@@ -453,35 +453,6 @@ def normalize_batched_scales_shape(
|
||||
return scales
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_topk_ids_weights_kernel(
|
||||
topk_ids_ptr,
|
||||
topk_weights_ptr,
|
||||
output_ptr,
|
||||
n_elements,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr, # triton metadata
|
||||
):
|
||||
pid = tl.program_id(axis=0)
|
||||
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < n_elements
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.extra.cuda.gdc_wait()
|
||||
expert_id = tl.load(topk_ids_ptr + offsets, mask=mask, other=0).to(tl.int32)
|
||||
expert_id_shifted = expert_id << 16
|
||||
|
||||
weight = tl.load(topk_weights_ptr + offsets, mask=mask, other=0.0)
|
||||
weight_bf16 = weight.to(tl.bfloat16)
|
||||
weight_int16 = weight_bf16.to(tl.int16, bitcast=True)
|
||||
|
||||
weight_int32 = weight_int16.to(tl.int32) & 0xFFFF
|
||||
|
||||
packed = expert_id_shifted | weight_int32
|
||||
tl.store(output_ptr + offsets, packed, mask=mask)
|
||||
|
||||
|
||||
def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
|
||||
"""Estimate FlashInfer's MoE autotuning maximum token count.
|
||||
|
||||
|
||||
@@ -59,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -1961,74 +1960,3 @@ direct_register_custom_op(
|
||||
op_func=qwen_gdn_attention_core_fused_norm_packed,
|
||||
mutates_args=["core_attn_out"],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fused_gdn_gating_kernel(
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
beta: tl.constexpr,
|
||||
threshold: tl.constexpr,
|
||||
BLK_HEADS: tl.constexpr,
|
||||
):
|
||||
i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
|
||||
off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
|
||||
mask = head_off < NUM_HEADS
|
||||
blk_A_log = tl.load(A_log + head_off, mask=mask)
|
||||
blk_a = tl.load(a + off, mask=mask)
|
||||
blk_b = tl.load(b + off, mask=mask)
|
||||
blk_bias = tl.load(dt_bias + head_off, mask=mask)
|
||||
# If the model is loaded in fp16, without the .float() here, A might be -inf
|
||||
x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
|
||||
softplus_x = tl.where(
|
||||
beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
|
||||
)
|
||||
blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
|
||||
tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
|
||||
# compute beta_output = sigmoid(b)
|
||||
blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
|
||||
tl.store(
|
||||
beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
|
||||
)
|
||||
|
||||
|
||||
def fused_gdn_gating(
|
||||
A_log: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
beta: float = 1.0,
|
||||
threshold: float = 20.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused computation of g and beta for Gated Delta Net.
|
||||
g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
|
||||
beta_output = b.sigmoid()
|
||||
TODO maybe use torch.compile to replace this triton kernel
|
||||
"""
|
||||
batch, num_heads = a.shape
|
||||
seq_len = 1
|
||||
grid = (batch, seq_len, triton.cdiv(num_heads, 8))
|
||||
g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
|
||||
beta_output = torch.empty(1, batch, num_heads, dtype=b.dtype, device=b.device)
|
||||
fused_gdn_gating_kernel[grid](
|
||||
g,
|
||||
beta_output,
|
||||
A_log,
|
||||
a,
|
||||
b,
|
||||
dt_bias,
|
||||
seq_len,
|
||||
num_heads,
|
||||
beta,
|
||||
threshold,
|
||||
8,
|
||||
num_warps=1,
|
||||
)
|
||||
return g, beta_output
|
||||
|
||||
@@ -12,7 +12,6 @@ import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Generator, Iterable
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
@@ -23,7 +22,7 @@ import huggingface_hub.constants
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import torch
|
||||
from safetensors.torch import load, load_file, safe_open, save_file
|
||||
from safetensors.torch import load, load_file, safe_open
|
||||
from tqdm.auto import tqdm
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
@@ -186,56 +185,6 @@ def maybe_download_from_modelscope(
|
||||
return None
|
||||
|
||||
|
||||
def _shared_pointers(tensors):
|
||||
ptrs = defaultdict(list)
|
||||
for k, v in tensors.items():
|
||||
ptrs[v.data_ptr()].append(k)
|
||||
failing = []
|
||||
for _, names in ptrs.items():
|
||||
if len(names) > 1:
|
||||
failing.append(names)
|
||||
return failing
|
||||
|
||||
|
||||
def convert_bin_to_safetensor_file(
|
||||
pt_filename: str,
|
||||
sf_filename: str,
|
||||
) -> None:
|
||||
loaded = torch.load(pt_filename, map_location="cpu", weights_only=True)
|
||||
if "state_dict" in loaded:
|
||||
loaded = loaded["state_dict"]
|
||||
shared = _shared_pointers(loaded)
|
||||
for shared_weights in shared:
|
||||
for name in shared_weights[1:]:
|
||||
loaded.pop(name)
|
||||
|
||||
# For tensors to be contiguous
|
||||
loaded = {k: v.contiguous() for k, v in loaded.items()}
|
||||
|
||||
dirname = os.path.dirname(sf_filename)
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
save_file(loaded, sf_filename, metadata={"format": "pt"})
|
||||
|
||||
# check file size
|
||||
sf_size = os.stat(sf_filename).st_size
|
||||
pt_size = os.stat(pt_filename).st_size
|
||||
if (sf_size - pt_size) / pt_size > 0.01:
|
||||
raise RuntimeError(
|
||||
f"""The file size different is more than 1%:
|
||||
- {sf_filename}: {sf_size}
|
||||
- {pt_filename}: {pt_size}
|
||||
"""
|
||||
)
|
||||
|
||||
# check if the tensors are the same
|
||||
reloaded = load_file(sf_filename)
|
||||
for k in loaded:
|
||||
pt_tensor = loaded[k]
|
||||
sf_tensor = reloaded[k]
|
||||
if not torch.equal(pt_tensor, sf_tensor):
|
||||
raise RuntimeError(f"The output tensors do not match for key {k}")
|
||||
|
||||
|
||||
# TODO(woosuk): Move this to other place.
|
||||
def get_quant_config(
|
||||
model_config: ModelConfig, load_config: LoadConfig
|
||||
@@ -1313,29 +1262,6 @@ def composed_weight_loader(
|
||||
return composed_loader
|
||||
|
||||
|
||||
def initialize_dummy_weights(
|
||||
model: torch.nn.Module,
|
||||
model_config: ModelConfig,
|
||||
low: float = -1e-3,
|
||||
high: float = 1e-3,
|
||||
seed: int = 1234,
|
||||
) -> None:
|
||||
"""Initialize model weights with random values.
|
||||
|
||||
The model weights must be randomly initialized for accurate performance
|
||||
measurements. Additionally, the model weights should not cause NaNs in the
|
||||
forward pass. We empirically found that initializing the weights with
|
||||
values between -1e-3 and 1e-3 works well for most models.
|
||||
|
||||
We use per-parameter random seed, so that dummy weights are consistent,
|
||||
even if the model is partitioned across multiple devices. When the seed
|
||||
is fixed, the random values generated by this function only depends on
|
||||
the parameter's number of elements and its data type.
|
||||
"""
|
||||
for param in model.state_dict().values():
|
||||
initialize_single_dummy_weight(param, low, high, seed)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def initialize_single_dummy_weight(
|
||||
param: torch.Tensor,
|
||||
|
||||
@@ -114,34 +114,6 @@ class InternVisionEmbeddings(nn.Module):
|
||||
return embeddings
|
||||
|
||||
|
||||
class InternVisionPatchModel(nn.Module):
|
||||
def __init__(self, config: PretrainedConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.embeddings = InternVisionEmbeddings(config)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embeddings
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.Tensor | None = None,
|
||||
pixel_embeds: torch.Tensor | None = None,
|
||||
) -> torch.FloatTensor:
|
||||
if pixel_values is None and pixel_embeds is None:
|
||||
raise ValueError("You have to specify pixel_values or pixel_embeds")
|
||||
|
||||
if pixel_embeds is not None:
|
||||
hidden_states = pixel_embeds
|
||||
elif pixel_values is not None:
|
||||
if pixel_values.ndim == 4:
|
||||
hidden_states = self.embeddings(pixel_values)
|
||||
else:
|
||||
raise ValueError(f"wrong pixel_values size: {pixel_values.shape}")
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class InternParallelAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper."""
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
# limitations under the License.
|
||||
import math
|
||||
from collections.abc import Hashable, Iterable, Mapping
|
||||
from itertools import tee
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import torch
|
||||
@@ -1041,25 +1040,6 @@ class Llama4ForConditionalGeneration(
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def separate_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
prefix: str,
|
||||
) -> tuple[Iterable[tuple[str, torch.Tensor]], Iterable[tuple[str, torch.Tensor]]]:
|
||||
weights1, weights2 = tee(weights, 2)
|
||||
|
||||
def get_prefix_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, data in weights1:
|
||||
if name.startswith(prefix):
|
||||
yield (name, data)
|
||||
|
||||
def get_other_weights() -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, data in weights2:
|
||||
if not name.startswith(prefix):
|
||||
yield (name, data)
|
||||
|
||||
return get_prefix_weights(), get_other_weights()
|
||||
|
||||
def _consolidate_qkv_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
|
||||
@@ -23,47 +23,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.models.llama import (
|
||||
LlamaDecoderLayer,
|
||||
LlamaForCausalLM,
|
||||
LlamaModel,
|
||||
)
|
||||
|
||||
|
||||
class TeleFLMModel(LlamaModel):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
layer_type: type[nn.Module] = LlamaDecoderLayer,
|
||||
):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type)
|
||||
"""
|
||||
This implementation is based on the µScaling paper presented at
|
||||
the ICLR 2025 Workshop:
|
||||
NanoLM: An Affordable LLM Study Benchmark \
|
||||
via Accurate Loss Prediction across Scales
|
||||
by Yiqun Yao et al.
|
||||
Available at: https://openreview.net/forum?id=IwaPYg1SCA
|
||||
arXiv preprint: https://arxiv.org/abs/2304.06875
|
||||
"""
|
||||
self.use_mup = self.config.use_mup
|
||||
if self.use_mup:
|
||||
self.input_mult = self.config.input_mult
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
embedding = self.embed_tokens(input_ids)
|
||||
if self.use_mup:
|
||||
embedding = embedding * self.input_mult
|
||||
return embedding
|
||||
|
||||
|
||||
class TeleFLMForCausalLM(LlamaForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
@@ -164,56 +164,6 @@ class WavFrontend(nn.Module):
|
||||
feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
|
||||
return feats_pad, feats_lens
|
||||
|
||||
def forward_fbank(
|
||||
self, input: torch.Tensor, input_lengths: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_size = input.size(0)
|
||||
feats = []
|
||||
feats_lens = []
|
||||
for i in range(batch_size):
|
||||
waveform_length = input_lengths[i]
|
||||
waveform = input[i][:waveform_length]
|
||||
waveform = waveform * (1 << 15)
|
||||
waveform = waveform.unsqueeze(0)
|
||||
mat = kaldi.fbank(
|
||||
waveform,
|
||||
num_mel_bins=self.n_mels,
|
||||
frame_length=self.frame_length,
|
||||
frame_shift=self.frame_shift,
|
||||
dither=self.dither,
|
||||
energy_floor=0.0,
|
||||
window_type=self.window,
|
||||
sample_frequency=self.fs,
|
||||
)
|
||||
|
||||
feat_length = mat.size(0)
|
||||
feats.append(mat)
|
||||
feats_lens.append(feat_length)
|
||||
|
||||
feats_lens = torch.as_tensor(feats_lens)
|
||||
feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
|
||||
return feats_pad, feats_lens
|
||||
|
||||
def forward_lfr_cmvn(
|
||||
self, input: torch.Tensor, input_lengths: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
batch_size = input.size(0)
|
||||
feats = []
|
||||
feats_lens = []
|
||||
for i in range(batch_size):
|
||||
mat = input[i, : input_lengths[i], :]
|
||||
if self.lfr_m != 1 or self.lfr_n != 1:
|
||||
mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
|
||||
if self.cmvn is not None:
|
||||
mat = apply_cmvn(mat, self.cmvn)
|
||||
feat_length = mat.size(0)
|
||||
feats.append(mat)
|
||||
feats_lens.append(feat_length)
|
||||
|
||||
feats_lens = torch.as_tensor(feats_lens)
|
||||
feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
|
||||
return feats_pad, feats_lens
|
||||
|
||||
|
||||
class FunASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
r"""Constructs a FunASR feature extractor.
|
||||
|
||||
@@ -17,12 +17,11 @@
|
||||
"""Processor class for MiniCPMO."""
|
||||
|
||||
import math
|
||||
from typing import Literal, TypeAlias
|
||||
from typing import TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import regex
|
||||
import torch
|
||||
import torchaudio
|
||||
from transformers.image_processing_utils import BatchFeature
|
||||
from transformers.image_utils import ImageInput
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
@@ -522,66 +521,3 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
padding_length.append(tensor.shape[-1] - len(item))
|
||||
|
||||
return tensor, padding_length
|
||||
|
||||
|
||||
class MelSpectrogramFeatures(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate=24000,
|
||||
n_fft=1024,
|
||||
hop_length=256,
|
||||
n_mels=100,
|
||||
padding: Literal["center", "same"] = "center",
|
||||
):
|
||||
super().__init__()
|
||||
if padding not in ["center", "same"]:
|
||||
raise ValueError("Padding must be 'center' or 'same'.")
|
||||
self.padding = padding
|
||||
self.mel_spec = torchaudio.transforms.MelSpectrogram(
|
||||
sample_rate=sample_rate,
|
||||
n_fft=n_fft,
|
||||
hop_length=hop_length,
|
||||
n_mels=n_mels,
|
||||
center=padding == "center",
|
||||
power=1,
|
||||
)
|
||||
|
||||
def __call__(self, audio: torch.Tensor) -> torch.Tensor:
|
||||
"""audio: Tensor([num_channels, num_samples])."""
|
||||
return super().__call__(audio)
|
||||
|
||||
def forward(self, audio: torch.Tensor) -> torch.Tensor:
|
||||
"""audio: Tensor([num_channels, num_samples])."""
|
||||
mel: torch.Tensor = self.mel_spec(audio)
|
||||
features = torch.log(torch.clip(mel, min=1e-5))
|
||||
return features
|
||||
|
||||
|
||||
class ChatTTSProcessor:
|
||||
def __init__(self, text_tokenizer):
|
||||
self.audio_processor = MelSpectrogramFeatures()
|
||||
self.text_tokenizer = text_tokenizer
|
||||
|
||||
def __call__(self, text_list, audio_list):
|
||||
assert len(text_list) == len(audio_list)
|
||||
input_ids_varlen = []
|
||||
for text in text_list:
|
||||
input_ids_ = self.text_tokenizer.encode(
|
||||
text, return_tensors="pt", add_special_tokens=False
|
||||
) # [1, seq_len]
|
||||
input_ids_ = input_ids_.squeeze(0) # [seq_len]
|
||||
input_ids_varlen.append(input_ids_)
|
||||
|
||||
audio_features_varlen = []
|
||||
for audio in audio_list:
|
||||
assert audio.shape.__len__() == 1 # [seq_len]
|
||||
try:
|
||||
mel = self.audio_processor(audio) # [100(num_mel_bins), seq_len_mel]
|
||||
except Exception as e:
|
||||
raise e
|
||||
audio_features_varlen.append(mel)
|
||||
|
||||
return {
|
||||
"tts_input_ids_varlen": input_ids_varlen, # return List[Tensor]
|
||||
"tts_input_features_varlen": audio_features_varlen, # return List[Tensor]
|
||||
}
|
||||
|
||||
@@ -276,48 +276,6 @@ def generate_sparse_seqlen_triton(
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def fetch_id_to_ragged_kernel(
|
||||
in_tensor_ptr, # [num_seq, topk]
|
||||
cumsum_ptr, # [num_seq + 1]
|
||||
out_tensor_ptr, # [max_num_seq * topk]
|
||||
in_tensor_ptr_stride,
|
||||
TOPK: tl.constexpr,
|
||||
TOKEN_NUM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_id = tl.program_id(0)
|
||||
block_id = tl.program_id(1)
|
||||
offset = tl.arange(0, BLOCK_SIZE)
|
||||
token_start = tl.load(cumsum_ptr + seq_id)
|
||||
token_end = tl.load(cumsum_ptr + seq_id + 1)
|
||||
token_num = token_end - token_start
|
||||
row_offset = block_id * BLOCK_SIZE
|
||||
if row_offset >= token_num:
|
||||
return
|
||||
in_tensor_offset = seq_id * in_tensor_ptr_stride + row_offset + offset
|
||||
in_tensor_mask = (row_offset + offset) < TOPK
|
||||
in_tensor_val = tl.load(in_tensor_ptr + in_tensor_offset, mask=in_tensor_mask)
|
||||
out_tensor_offset = token_start + row_offset + offset
|
||||
out_tensor_mask = (out_tensor_offset < token_end) & in_tensor_mask
|
||||
tl.store(out_tensor_ptr + out_tensor_offset, in_tensor_val, mask=out_tensor_mask)
|
||||
|
||||
|
||||
def fetch_id_to_ragged_triton(
|
||||
in_tensor: torch.Tensor, cumsum: torch.Tensor, out_tensor: torch.Tensor, topk
|
||||
):
|
||||
num_tokens = in_tensor.size(0)
|
||||
block_size = 64
|
||||
num_block_per_row = triton.cdiv(topk, block_size)
|
||||
grid = (
|
||||
num_tokens,
|
||||
num_block_per_row,
|
||||
)
|
||||
fetch_id_to_ragged_kernel[grid](
|
||||
in_tensor, cumsum, out_tensor, in_tensor.stride(0), topk, num_tokens, block_size
|
||||
)
|
||||
|
||||
|
||||
class ROCMAiterMLASparseBackend(AttentionBackend):
|
||||
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
|
||||
@@ -62,21 +62,6 @@ def apply_temperature(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def tl_rand64(seed, offset, includes_zero: tl.constexpr):
|
||||
lo, hi, _, _ = tl.randint4x(seed, offset)
|
||||
lo = lo.to(tl.uint32, bitcast=True).to(tl.uint64)
|
||||
hi = hi.to(tl.uint32, bitcast=True).to(tl.uint64)
|
||||
r = (hi << 32) | lo
|
||||
|
||||
# 1 / 2**64
|
||||
scale = 5.421010862427522170037e-20
|
||||
u = r.to(tl.float64) * scale
|
||||
if not includes_zero:
|
||||
u = tl.maximum(u, 2.2250738585072014e-308) # float64 tiny
|
||||
return u
|
||||
|
||||
|
||||
@triton.jit
|
||||
def tl_rand32(seed, offset, includes_zero: tl.constexpr):
|
||||
u = tl.rand(seed, offset)
|
||||
|
||||
Reference in New Issue
Block a user