vLLM/vllm/transformers_utils/processors/kimi_audio.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

# ruff: noqa
# mypy: ignore-errors
# coding=utf-8
# Copyright 2026 The Moonshot AI team and the HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Processor for Kimi-Audio ASR model."""

from collections.abc import Mapping
from typing import Any

import numpy as np
import torch
from transformers import AutoFeatureExtractor, BatchFeature, ProcessorMixin
from transformers.audio_utils import AudioInput
from transformers.tokenization_utils_base import TextInput

from vllm.tokenizers.kimi_audio import KimiAudioTokenizer


def _get_feat_extract_output_lengths(input_lengths: torch.Tensor) -> torch.Tensor:
    """Compute output lengths after Whisper feature extraction."""
    input_lengths_leave = input_lengths % 100
    feat_lengths = (input_lengths_leave - 1) // 2 + 1
    output_lengths = (
        ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
    )
    return output_lengths


class KimiAudioProcessor(ProcessorMixin):
    r"""
    Constructs a Kimi-Audio processor.

    [`KimiAudioProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`], and a tokenizer.
    See the [`~KimiAudioProcessor.__call__`] and [`~KimiAudioProcessor.decode`] for more information.

    Args:
        feature_extractor ([`WhisperFeatureExtractor`], *optional*):
            The audio feature extractor.
        tokenizer ([`PreTrainedTokenizer`], *optional*):
            The text tokenizer.
    """

    # Required for ProcessorMixin
    attributes = ["feature_extractor", "tokenizer"]
    feature_extractor_class = "AutoFeatureExtractor"
    tokenizer_class = "AutoTokenizer"

    # Special token IDs
    KIMIA_MEDIA_BEGIN: int = 151661
    KIMIA_MEDIA_END: int = 151663
    KIMIA_TEXT_BLANK: int = 151666

    # Audio processing constants
    AUDIO_SEQ_LEN: int = 376

    def __init__(self, feature_extractor=None, tokenizer=None, **kwargs):
        # Pass feature_extractor and tokenizer to parent ProcessorMixin
        super().__init__(
            feature_extractor=feature_extractor,
            tokenizer=tokenizer,
            **kwargs,
        )

    def check_argument_for_proper_class(self, attribute_name: str, argument: Any):
        """Override to skip class validation for custom tokenizer."""
        # Skip validation for tokenizer since KimiAudioTokenizer doesn't inherit
        # from PreTrainedTokenizerBase but is compatible
        if attribute_name == "tokenizer" and argument is not None:
            return
        # For other attributes, use default validation
        super().check_argument_for_proper_class(attribute_name, argument)

    def __call__(
        self,
        text: TextInput = None,
        audio: AudioInput = None,
        return_tensors: str = "pt",
        **kwargs,
    ) -> BatchFeature:
        """
        Main method to prepare for the model one or several sequences(s) and audio(s).

        Args:
            text (`str`, `List[str]`):
                The sequence or batch of sequences to be encoded.
            audio (`np.ndarray`, `List[np.ndarray]`):
                The audio or batch of audio to be prepared. Each audio can be a NumPy array.
            return_tensors (`str`):
                The type of tensors to return ("pt", "np", etc.)
        """
        if text is None:
            raise ValueError("You need to specify either a `text` input to process.")

        # Process audio if provided
        if audio is not None:
            # Ensure audio is a list
            if isinstance(audio, np.ndarray):
                audio = [audio]

            # Pad audio to hop length (required by WhisperFeatureExtractor)
            hop_length = self.feature_extractor.hop_length
            padded_audio = []
            for aud in audio:
                length = aud.shape[-1]
                if length % hop_length != 0:
                    pad_length = hop_length - (length % hop_length)
                    aud = np.pad(
                        aud, (0, pad_length), mode="constant", constant_values=0
                    )
                padded_audio.append(aud)

            # Use feature_extractor directly like Qwen3ASR does
            audio_inputs = self.feature_extractor(
                padded_audio,
                sampling_rate=16000,
                padding=True,
                return_attention_mask=True,
                return_tensors=return_tensors,
            )
            # Rename to match Kimi-Audio expectations
            if "input_features" in audio_inputs:
                audio_inputs["whisper_input_features"] = audio_inputs.pop(
                    "input_features"
                )
            if "attention_mask" in audio_inputs:
                audio_inputs["feature_attention_mask"] = audio_inputs.pop(
                    "attention_mask"
                )
        else:
            audio_inputs = {}

        # Handle text input - can be string or token IDs from vLLM processor
        if isinstance(text, list) and len(text) > 0 and isinstance(text[0], int):
            # Text is already token IDs (from vLLM processor) - just wrap
            text_inputs = {"input_ids": torch.tensor([text], dtype=torch.long)}
        else:
            # Text is string - tokenize
            if not isinstance(text, list):
                text = [text]

            text_inputs = self.tokenizer(
                text, return_tensors=return_tensors, padding=True
            )

        return BatchFeature(
            data={**text_inputs, **audio_inputs},
            tensor_type=return_tensors,
        )