diff --git a/vllm/entrypoints/generate/base/protocol.py b/vllm/entrypoints/generate/base/protocol.py index 76dc5e47a1..91d2e30a47 100644 --- a/vllm/entrypoints/generate/base/protocol.py +++ b/vllm/entrypoints/generate/base/protocol.py @@ -6,10 +6,8 @@ import json from typing import Annotated, Any, Literal, TypeAlias -import regex as re from pydantic import ( BaseModel, - ConfigDict, Field, model_serializer, ) @@ -21,7 +19,6 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.sampling_params import StructuredOutputsParams -from vllm.utils.import_utils import resolve_obj_by_qualname logger = init_logger(__name__) @@ -261,53 +258,6 @@ class FunctionDefinition(OpenAIBaseModel): return data -# extra="forbid" is a workaround to have kwargs as a field, -# see https://github.com/pydantic/pydantic/issues/3125 -class LogitsProcessorConstructor(BaseModel): - qualname: str - args: list[Any] | None = None - kwargs: dict[str, Any] | None = None - - model_config = ConfigDict(extra="forbid") - - -LogitsProcessors = list[str | LogitsProcessorConstructor] - - -def get_logits_processors( - processors: LogitsProcessors | None, pattern: str | None -) -> list[Any] | None: - if processors and pattern: - logits_processors = [] - for processor in processors: - qualname = processor if isinstance(processor, str) else processor.qualname - if not re.match(pattern, qualname): - raise ValueError( - f"Logits processor '{qualname}' is not allowed by this " - "server. See --logits-processor-pattern engine argument " - "for more information." - ) - try: - logits_processor = resolve_obj_by_qualname(qualname) - except Exception as e: - raise ValueError( - f"Logits processor '{qualname}' could not be resolved: {e}" - ) from e - if isinstance(processor, LogitsProcessorConstructor): - logits_processor = logits_processor( - *processor.args or [], **processor.kwargs or {} - ) - logits_processors.append(logits_processor) - return logits_processors - elif processors: - raise ValueError( - "The `logits_processors` argument is not supported by this " - "server. See --logits-processor-pattern engine argument " - "for more information." - ) - return None - - class FunctionCall(OpenAIBaseModel): # Internal field to preserve native tool call ID from tool parser. # Excluded from serialization to maintain OpenAI API compatibility diff --git a/vllm/logits_process.py b/vllm/logits_process.py index 1bf97c2535..300ec3cacc 100644 --- a/vllm/logits_process.py +++ b/vllm/logits_process.py @@ -1,12 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Callable, Sequence +from collections.abc import Callable from typing import TypeAlias import torch -from vllm.tokenizers import TokenizerLike - LogitsProcessor: TypeAlias = ( Callable[[list[int], torch.Tensor], torch.Tensor] | Callable[[list[int], list[int], torch.Tensor], torch.Tensor] @@ -16,106 +14,3 @@ of previously generated tokens, the logits tensor for the next token and, optionally, prompt tokens as a first argument, and returns a modified tensor of logits to sample from.""" - - -def get_bad_words_logits_processors( - bad_words: list[str], tokenizer: TokenizerLike -) -> list[LogitsProcessor]: - bad_words_ids: list[list[int]] = list() - - for bad_word in bad_words: - # To prohibit words both at the beginning - # and in the middle of text - # (related to add_prefix_space tokenizer parameter) - for add_prefix_space in [False, True]: - prefix = " " if add_prefix_space else "" - prompt = prefix + bad_word.lstrip() - - prompt_token_ids = tokenizer.encode(text=prompt, add_special_tokens=False) - - # If no space at the beginning - # or if prefix space produces a new word token - if (not add_prefix_space) or ( - add_prefix_space - and prompt_token_ids[0] != bad_words_ids[-1][0] - and len(prompt_token_ids) == len(bad_words_ids[-1]) - ): - bad_words_ids.append(prompt_token_ids) - - return [NoBadWordsLogitsProcessor(bad_words_ids=bad_words_ids)] - - -class NoBadWordsLogitsProcessor: - _SMALLEST_LOGIT = float("-inf") - _NEUTRAL_LOGIT = 0.0 - - def __init__(self, bad_words_ids: list[list[int]]): - self.bad_words_ids = bad_words_ids - self.word_bias: torch.FloatTensor = None - - def __call__( - self, - past_tokens_ids: Sequence[int], - logits: torch.FloatTensor, - ) -> torch.Tensor: - if self.word_bias is None: - self._init_word_bias(logits=logits) - - last_token_bias = torch.zeros_like(logits) - - for bad_word_ids in self.bad_words_ids: - if len(bad_word_ids) == 1: # 1-token words already processed - continue - - if len(bad_word_ids) > len(past_tokens_ids) + 1: - continue - - prefix_length = len(bad_word_ids) - 1 - last_token_id = bad_word_ids[-1] - actual_prefix = past_tokens_ids[-prefix_length:] - expected_prefix = bad_word_ids[:prefix_length] - - assert len(actual_prefix) == len(expected_prefix) - - is_match = tuple(actual_prefix) == tuple(expected_prefix) - last_token_bias[last_token_id] += ( - self._SMALLEST_LOGIT if is_match else self._NEUTRAL_LOGIT - ) - - logits = logits + self.word_bias + last_token_bias - - return logits - - def _init_word_bias(self, logits: torch.FloatTensor) -> None: - # Code based on NoBadWordsLogitsProcessor and SequenceBiasLogitsProcessor # noqa: E501 - # from https://github.com/huggingface/transformers/blob/main/src/transformers/generation/logits_process.py - - vocab_size = logits.shape[-1] - - self._check_token_ids_bounds(vocab_size=vocab_size) - - self.word_bias = torch.zeros( - (vocab_size,), dtype=torch.float, device=logits.device - ) - - for bad_word_ids in self.bad_words_ids: - if len(bad_word_ids) == 1: - bad_word_id = bad_word_ids[-1] - self.word_bias[bad_word_id] = self._SMALLEST_LOGIT - - def _check_token_ids_bounds(self, vocab_size: int) -> None: - invalid_token_ids = [] - - for bad_word_ids in self.bad_words_ids: - for token_id in bad_word_ids: - if token_id < 0 or token_id >= vocab_size: - invalid_token_ids.append(token_id) - - if len(invalid_token_ids) > 0: - raise ValueError( - f"The model vocabulary size is {vocab_size}," - f" but the following tokens" - f" were specified as bad: {invalid_token_ids}." - f" All token id values should be integers satisfying:" - f" 0 <= token_id < {vocab_size}." - ) diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index 34e116dfbc..a98e181cae 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -453,35 +453,6 @@ def normalize_batched_scales_shape( return scales -@triton.jit -def _pack_topk_ids_weights_kernel( - topk_ids_ptr, - topk_weights_ptr, - output_ptr, - n_elements, - BLOCK_SIZE: tl.constexpr, - USE_GDC: tl.constexpr, - launch_pdl: tl.constexpr, # triton metadata -): - pid = tl.program_id(axis=0) - offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offsets < n_elements - if USE_GDC: - tl.extra.cuda.gdc_launch_dependents() - tl.extra.cuda.gdc_wait() - expert_id = tl.load(topk_ids_ptr + offsets, mask=mask, other=0).to(tl.int32) - expert_id_shifted = expert_id << 16 - - weight = tl.load(topk_weights_ptr + offsets, mask=mask, other=0.0) - weight_bf16 = weight.to(tl.bfloat16) - weight_int16 = weight_bf16.to(tl.int16, bitcast=True) - - weight_int32 = weight_int16.to(tl.int32) & 0xFFFF - - packed = expert_id_shifted | weight_int32 - tl.store(output_ptr + offsets, packed, mask=mask) - - def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int: """Estimate FlashInfer's MoE autotuning maximum token count. diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index 681ff98ae6..094dd74e78 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -59,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import ( from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig -from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, @@ -1961,74 +1960,3 @@ direct_register_custom_op( op_func=qwen_gdn_attention_core_fused_norm_packed, mutates_args=["core_attn_out"], ) - - -@triton.jit -def fused_gdn_gating_kernel( - g, - beta_output, - A_log, - a, - b, - dt_bias, - seq_len, - NUM_HEADS: tl.constexpr, - beta: tl.constexpr, - threshold: tl.constexpr, - BLK_HEADS: tl.constexpr, -): - i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2) - head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS) - off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off - mask = head_off < NUM_HEADS - blk_A_log = tl.load(A_log + head_off, mask=mask) - blk_a = tl.load(a + off, mask=mask) - blk_b = tl.load(b + off, mask=mask) - blk_bias = tl.load(dt_bias + head_off, mask=mask) - # If the model is loaded in fp16, without the .float() here, A might be -inf - x = blk_a.to(tl.float32) + blk_bias.to(tl.float32) - softplus_x = tl.where( - beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x - ) - blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x - tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask) - # compute beta_output = sigmoid(b) - blk_beta_output = tl.sigmoid(blk_b.to(tl.float32)) - tl.store( - beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask - ) - - -def fused_gdn_gating( - A_log: torch.Tensor, - a: torch.Tensor, - b: torch.Tensor, - dt_bias: torch.Tensor, - beta: float = 1.0, - threshold: float = 20.0, -) -> tuple[torch.Tensor, torch.Tensor]: - """Fused computation of g and beta for Gated Delta Net. - g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias) - beta_output = b.sigmoid() - TODO maybe use torch.compile to replace this triton kernel - """ - batch, num_heads = a.shape - seq_len = 1 - grid = (batch, seq_len, triton.cdiv(num_heads, 8)) - g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device) - beta_output = torch.empty(1, batch, num_heads, dtype=b.dtype, device=b.device) - fused_gdn_gating_kernel[grid]( - g, - beta_output, - A_log, - a, - b, - dt_bias, - seq_len, - num_heads, - beta, - threshold, - 8, - num_warps=1, - ) - return g, beta_output diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index c781a8bf87..452c03a8ce 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -12,7 +12,6 @@ import os import tempfile import threading import time -from collections import defaultdict from collections.abc import Callable, Generator, Iterable from contextlib import contextmanager from pathlib import Path @@ -23,7 +22,7 @@ import huggingface_hub.constants import numpy as np import regex as re import torch -from safetensors.torch import load, load_file, safe_open, save_file +from safetensors.torch import load, load_file, safe_open from tqdm.auto import tqdm from transformers.utils import SAFE_WEIGHTS_INDEX_NAME @@ -186,56 +185,6 @@ def maybe_download_from_modelscope( return None -def _shared_pointers(tensors): - ptrs = defaultdict(list) - for k, v in tensors.items(): - ptrs[v.data_ptr()].append(k) - failing = [] - for _, names in ptrs.items(): - if len(names) > 1: - failing.append(names) - return failing - - -def convert_bin_to_safetensor_file( - pt_filename: str, - sf_filename: str, -) -> None: - loaded = torch.load(pt_filename, map_location="cpu", weights_only=True) - if "state_dict" in loaded: - loaded = loaded["state_dict"] - shared = _shared_pointers(loaded) - for shared_weights in shared: - for name in shared_weights[1:]: - loaded.pop(name) - - # For tensors to be contiguous - loaded = {k: v.contiguous() for k, v in loaded.items()} - - dirname = os.path.dirname(sf_filename) - os.makedirs(dirname, exist_ok=True) - save_file(loaded, sf_filename, metadata={"format": "pt"}) - - # check file size - sf_size = os.stat(sf_filename).st_size - pt_size = os.stat(pt_filename).st_size - if (sf_size - pt_size) / pt_size > 0.01: - raise RuntimeError( - f"""The file size different is more than 1%: - - {sf_filename}: {sf_size} - - {pt_filename}: {pt_size} - """ - ) - - # check if the tensors are the same - reloaded = load_file(sf_filename) - for k in loaded: - pt_tensor = loaded[k] - sf_tensor = reloaded[k] - if not torch.equal(pt_tensor, sf_tensor): - raise RuntimeError(f"The output tensors do not match for key {k}") - - # TODO(woosuk): Move this to other place. def get_quant_config( model_config: ModelConfig, load_config: LoadConfig @@ -1313,29 +1262,6 @@ def composed_weight_loader( return composed_loader -def initialize_dummy_weights( - model: torch.nn.Module, - model_config: ModelConfig, - low: float = -1e-3, - high: float = 1e-3, - seed: int = 1234, -) -> None: - """Initialize model weights with random values. - - The model weights must be randomly initialized for accurate performance - measurements. Additionally, the model weights should not cause NaNs in the - forward pass. We empirically found that initializing the weights with - values between -1e-3 and 1e-3 works well for most models. - - We use per-parameter random seed, so that dummy weights are consistent, - even if the model is partitioned across multiple devices. When the seed - is fixed, the random values generated by this function only depends on - the parameter's number of elements and its data type. - """ - for param in model.state_dict().values(): - initialize_single_dummy_weight(param, low, high, seed) - - @torch.no_grad() def initialize_single_dummy_weight( param: torch.Tensor, diff --git a/vllm/model_executor/models/intern_vit.py b/vllm/model_executor/models/intern_vit.py index 959f241de9..2b1c247d8f 100644 --- a/vllm/model_executor/models/intern_vit.py +++ b/vllm/model_executor/models/intern_vit.py @@ -114,34 +114,6 @@ class InternVisionEmbeddings(nn.Module): return embeddings -class InternVisionPatchModel(nn.Module): - def __init__(self, config: PretrainedConfig): - super().__init__() - self.config = config - self.embeddings = InternVisionEmbeddings(config) - - def get_input_embeddings(self): - return self.embeddings - - def forward( - self, - pixel_values: torch.Tensor | None = None, - pixel_embeds: torch.Tensor | None = None, - ) -> torch.FloatTensor: - if pixel_values is None and pixel_embeds is None: - raise ValueError("You have to specify pixel_values or pixel_embeds") - - if pixel_embeds is not None: - hidden_states = pixel_embeds - elif pixel_values is not None: - if pixel_values.ndim == 4: - hidden_states = self.embeddings(pixel_values) - else: - raise ValueError(f"wrong pixel_values size: {pixel_values.shape}") - - return hidden_states - - class InternParallelAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper.""" diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 89d4edbe86..4984a4d17d 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -18,7 +18,6 @@ # limitations under the License. import math from collections.abc import Hashable, Iterable, Mapping -from itertools import tee from typing import Annotated, Any, Literal import torch @@ -1041,25 +1040,6 @@ class Llama4ForConditionalGeneration( ) -> torch.Tensor | None: return self.language_model.compute_logits(hidden_states) - def separate_weights( - self, - weights: Iterable[tuple[str, torch.Tensor]], - prefix: str, - ) -> tuple[Iterable[tuple[str, torch.Tensor]], Iterable[tuple[str, torch.Tensor]]]: - weights1, weights2 = tee(weights, 2) - - def get_prefix_weights() -> Iterable[tuple[str, torch.Tensor]]: - for name, data in weights1: - if name.startswith(prefix): - yield (name, data) - - def get_other_weights() -> Iterable[tuple[str, torch.Tensor]]: - for name, data in weights2: - if not name.startswith(prefix): - yield (name, data) - - return get_prefix_weights(), get_other_weights() - def _consolidate_qkv_weights( self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[tuple[str, torch.Tensor]]: diff --git a/vllm/model_executor/models/teleflm.py b/vllm/model_executor/models/teleflm.py index bebd7bcaa9..b8235daacb 100644 --- a/vllm/model_executor/models/teleflm.py +++ b/vllm/model_executor/models/teleflm.py @@ -23,47 +23,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import torch -import torch.nn as nn from vllm.config import VllmConfig from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.models.llama import ( - LlamaDecoderLayer, LlamaForCausalLM, - LlamaModel, ) -class TeleFLMModel(LlamaModel): - def __init__( - self, - *, - vllm_config: VllmConfig, - prefix: str = "", - layer_type: type[nn.Module] = LlamaDecoderLayer, - ): - super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type) - """ - This implementation is based on the µScaling paper presented at - the ICLR 2025 Workshop: - NanoLM: An Affordable LLM Study Benchmark \ - via Accurate Loss Prediction across Scales - by Yiqun Yao et al. - Available at: https://openreview.net/forum?id=IwaPYg1SCA - arXiv preprint: https://arxiv.org/abs/2304.06875 - """ - self.use_mup = self.config.use_mup - if self.use_mup: - self.input_mult = self.config.input_mult - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - embedding = self.embed_tokens(input_ids) - if self.use_mup: - embedding = embedding * self.input_mult - return embedding - - class TeleFLMForCausalLM(LlamaForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) diff --git a/vllm/transformers_utils/processors/funasr.py b/vllm/transformers_utils/processors/funasr.py index aaa0bec48c..0a2f9ca6e1 100644 --- a/vllm/transformers_utils/processors/funasr.py +++ b/vllm/transformers_utils/processors/funasr.py @@ -164,56 +164,6 @@ class WavFrontend(nn.Module): feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) return feats_pad, feats_lens - def forward_fbank( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - waveform_length = input_lengths[i] - waveform = input[i][:waveform_length] - waveform = waveform * (1 << 15) - waveform = waveform.unsqueeze(0) - mat = kaldi.fbank( - waveform, - num_mel_bins=self.n_mels, - frame_length=self.frame_length, - frame_shift=self.frame_shift, - dither=self.dither, - energy_floor=0.0, - window_type=self.window, - sample_frequency=self.fs, - ) - - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - - def forward_lfr_cmvn( - self, input: torch.Tensor, input_lengths: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - batch_size = input.size(0) - feats = [] - feats_lens = [] - for i in range(batch_size): - mat = input[i, : input_lengths[i], :] - if self.lfr_m != 1 or self.lfr_n != 1: - mat = apply_lfr(mat, self.lfr_m, self.lfr_n) - if self.cmvn is not None: - mat = apply_cmvn(mat, self.cmvn) - feat_length = mat.size(0) - feats.append(mat) - feats_lens.append(feat_length) - - feats_lens = torch.as_tensor(feats_lens) - feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0) - return feats_pad, feats_lens - class FunASRFeatureExtractor(SequenceFeatureExtractor): r"""Constructs a FunASR feature extractor. diff --git a/vllm/transformers_utils/processors/minicpmo.py b/vllm/transformers_utils/processors/minicpmo.py index e59a50ccbe..17a915b2d6 100644 --- a/vllm/transformers_utils/processors/minicpmo.py +++ b/vllm/transformers_utils/processors/minicpmo.py @@ -17,12 +17,11 @@ """Processor class for MiniCPMO.""" import math -from typing import Literal, TypeAlias +from typing import TypeAlias import numpy as np import regex import torch -import torchaudio from transformers.image_processing_utils import BatchFeature from transformers.image_utils import ImageInput from transformers.processing_utils import ProcessorMixin @@ -522,66 +521,3 @@ class MiniCPMOProcessor(ProcessorMixin): padding_length.append(tensor.shape[-1] - len(item)) return tensor, padding_length - - -class MelSpectrogramFeatures(torch.nn.Module): - def __init__( - self, - sample_rate=24000, - n_fft=1024, - hop_length=256, - n_mels=100, - padding: Literal["center", "same"] = "center", - ): - super().__init__() - if padding not in ["center", "same"]: - raise ValueError("Padding must be 'center' or 'same'.") - self.padding = padding - self.mel_spec = torchaudio.transforms.MelSpectrogram( - sample_rate=sample_rate, - n_fft=n_fft, - hop_length=hop_length, - n_mels=n_mels, - center=padding == "center", - power=1, - ) - - def __call__(self, audio: torch.Tensor) -> torch.Tensor: - """audio: Tensor([num_channels, num_samples]).""" - return super().__call__(audio) - - def forward(self, audio: torch.Tensor) -> torch.Tensor: - """audio: Tensor([num_channels, num_samples]).""" - mel: torch.Tensor = self.mel_spec(audio) - features = torch.log(torch.clip(mel, min=1e-5)) - return features - - -class ChatTTSProcessor: - def __init__(self, text_tokenizer): - self.audio_processor = MelSpectrogramFeatures() - self.text_tokenizer = text_tokenizer - - def __call__(self, text_list, audio_list): - assert len(text_list) == len(audio_list) - input_ids_varlen = [] - for text in text_list: - input_ids_ = self.text_tokenizer.encode( - text, return_tensors="pt", add_special_tokens=False - ) # [1, seq_len] - input_ids_ = input_ids_.squeeze(0) # [seq_len] - input_ids_varlen.append(input_ids_) - - audio_features_varlen = [] - for audio in audio_list: - assert audio.shape.__len__() == 1 # [seq_len] - try: - mel = self.audio_processor(audio) # [100(num_mel_bins), seq_len_mel] - except Exception as e: - raise e - audio_features_varlen.append(mel) - - return { - "tts_input_ids_varlen": input_ids_varlen, # return List[Tensor] - "tts_input_features_varlen": audio_features_varlen, # return List[Tensor] - } diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 69b6e342cd..f1c3b558be 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -276,48 +276,6 @@ def generate_sparse_seqlen_triton( return out -@triton.jit -def fetch_id_to_ragged_kernel( - in_tensor_ptr, # [num_seq, topk] - cumsum_ptr, # [num_seq + 1] - out_tensor_ptr, # [max_num_seq * topk] - in_tensor_ptr_stride, - TOPK: tl.constexpr, - TOKEN_NUM: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - seq_id = tl.program_id(0) - block_id = tl.program_id(1) - offset = tl.arange(0, BLOCK_SIZE) - token_start = tl.load(cumsum_ptr + seq_id) - token_end = tl.load(cumsum_ptr + seq_id + 1) - token_num = token_end - token_start - row_offset = block_id * BLOCK_SIZE - if row_offset >= token_num: - return - in_tensor_offset = seq_id * in_tensor_ptr_stride + row_offset + offset - in_tensor_mask = (row_offset + offset) < TOPK - in_tensor_val = tl.load(in_tensor_ptr + in_tensor_offset, mask=in_tensor_mask) - out_tensor_offset = token_start + row_offset + offset - out_tensor_mask = (out_tensor_offset < token_end) & in_tensor_mask - tl.store(out_tensor_ptr + out_tensor_offset, in_tensor_val, mask=out_tensor_mask) - - -def fetch_id_to_ragged_triton( - in_tensor: torch.Tensor, cumsum: torch.Tensor, out_tensor: torch.Tensor, topk -): - num_tokens = in_tensor.size(0) - block_size = 64 - num_block_per_row = triton.cdiv(topk, block_size) - grid = ( - num_tokens, - num_block_per_row, - ) - fetch_id_to_ragged_kernel[grid]( - in_tensor, cumsum, out_tensor, in_tensor.stride(0), topk, num_tokens, block_size - ) - - class ROCMAiterMLASparseBackend(AttentionBackend): supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16] supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index 44cbadb9e0..e54a458709 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -62,21 +62,6 @@ def apply_temperature( ) -@triton.jit -def tl_rand64(seed, offset, includes_zero: tl.constexpr): - lo, hi, _, _ = tl.randint4x(seed, offset) - lo = lo.to(tl.uint32, bitcast=True).to(tl.uint64) - hi = hi.to(tl.uint32, bitcast=True).to(tl.uint64) - r = (hi << 32) | lo - - # 1 / 2**64 - scale = 5.421010862427522170037e-20 - u = r.to(tl.float64) * scale - if not includes_zero: - u = tl.maximum(u, 2.2250738585072014e-308) # float64 tiny - return u - - @triton.jit def tl_rand32(seed, offset, includes_zero: tl.constexpr): u = tl.rand(seed, offset)