Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+137 -113
View File
@@ -3,6 +3,7 @@
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
"""PyTorch Ultravox model."""
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, Optional, Union
@@ -20,21 +21,37 @@ from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
MultiModalKwargsItems, NestedTensors)
from vllm.multimodal.inputs import (
MultiModalDataDict,
MultiModalFieldConfig,
MultiModalKwargsItems,
NestedTensors,
)
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (BaseMultiModalProcessor,
BaseProcessingInfo, PromptReplacement,
PromptUpdate)
from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
SupportsMultiModal, SupportsPP)
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
init_vllm_registered_model, maybe_prefix)
from .interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
flatten_bn,
init_vllm_registered_model,
maybe_prefix,
)
_AUDIO_PLACEHOLDER_OVERRIDE = "<|audio|>"
_MAX_ENCODER_BATCH_SIZE = 16
@@ -48,15 +65,21 @@ class UltravoxAudioFeatureInputs(TensorSchema):
- t: Time frames (M)
- nmb: Number of mel bins
"""
type: Literal["audio_features"]
data: Annotated[Union[torch.Tensor, list[torch.Tensor],
list[list[torch.Tensor]]],
TensorShape("b", "n", "nmb", "t", dynamic_dims={"n"})]
lens: Annotated[Union[torch.Tensor, list[torch.Tensor]],
TensorShape("b", "n", dynamic_dims={"n"})]
data: Annotated[
Union[torch.Tensor, list[torch.Tensor], list[list[torch.Tensor]]],
TensorShape("b", "n", "nmb", "t", dynamic_dims={"n"}),
]
lens: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
TensorShape("b", "n", dynamic_dims={"n"}),
]
"""Length of the audio frames. Used for attention mask in WhisperEncoder."""
token_len: Annotated[Union[torch.Tensor, list[torch.Tensor]],
TensorShape("b", "n", dynamic_dims={"n"})]
token_len: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
TensorShape("b", "n", dynamic_dims={"n"}),
]
"""Length of the audio tokens. Used for flattening the audio features."""
@@ -68,17 +91,17 @@ class UltravoxAudioEmbeddingInputs(TensorSchema):
- afs: audio feature size
- hs: hidden size
"""
type: Literal["audio_embeds"]
data: Annotated[Union[torch.Tensor, list[torch.Tensor]],
TensorShape("b", "na", "afs", "hs")]
data: Annotated[
Union[torch.Tensor, list[torch.Tensor]], TensorShape("b", "na", "afs", "hs")
]
UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
UltravoxAudioEmbeddingInputs]
UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs, UltravoxAudioEmbeddingInputs]
class UltravoxProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
config = self.ctx.model_config.hf_config
hf_processor = self.ctx.get_hf_processor(**kwargs)
@@ -91,8 +114,7 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
return hf_processor
def get_feature_extractor(self,
**kwargs: object) -> WhisperFeatureExtractor:
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
hf_processor = self.get_hf_processor(**kwargs)
audio_processor = hf_processor.audio_processor # type: ignore
feature_extractor = audio_processor.feature_extractor # type: ignore
@@ -103,9 +125,7 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
return {"audio": None}
class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
):
class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
@@ -120,23 +140,21 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = (feature_extractor.chunk_length * sampling_rate *
_MAX_ENCODER_BATCH_SIZE)
audio_len = (
feature_extractor.chunk_length * sampling_rate * _MAX_ENCODER_BATCH_SIZE
)
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") if mm_options else None
return {
"audio":
self._get_dummy_audios(length=audio_len,
num_audios=num_audios,
overrides=audio_overrides)
"audio": self._get_dummy_audios(
length=audio_len, num_audios=num_audios, overrides=audio_overrides
)
}
class UltravoxMultiModalProcessor(
BaseMultiModalProcessor[UltravoxProcessingInfo]):
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
def _get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.info.get_feature_extractor()
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
@@ -151,7 +169,8 @@ class UltravoxMultiModalProcessor(
# Text-only input not supported in composite processor
if not mm_data.get("audios", []):
prompt_ids = self.info.get_tokenizer().encode(
prompt, add_special_tokens=False)
prompt, add_special_tokens=False
)
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
@@ -178,7 +197,7 @@ class UltravoxMultiModalProcessor(
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
output['audio_features'] = output.pop('audio_values')
output["audio_features"] = output.pop("audio_values")
return output
@@ -187,17 +206,14 @@ class UltravoxMultiModalProcessor(
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
num_chunks = hf_inputs.get('audio_num_chunks', torch.zeros(0))
num_chunks = hf_inputs.get("audio_num_chunks", torch.zeros(0))
return dict(
# to handle longer than 30s audio, each audio might be split
# into multiple chunks as such, their batch dimension can be
# higher than the number of audio samples
audio_features=MultiModalFieldConfig.flat_from_sizes(
"audio", num_chunks),
audio_token_len=MultiModalFieldConfig.flat_from_sizes(
"audio", num_chunks),
audio_lens=MultiModalFieldConfig.flat_from_sizes(
"audio", num_chunks),
audio_features=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
audio_token_len=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
audio_lens=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
# num_chunks can convert audio_chunked to audio batch dimension
audio_num_chunks=MultiModalFieldConfig.batched("audio"),
audio_embeds=MultiModalFieldConfig.batched("audio"),
@@ -218,11 +234,12 @@ class UltravoxMultiModalProcessor(
# belonging to the i-th audio.
out_mm_data = out_mm_kwargs.get_data()
num_chunks = out_mm_data.get("audio_num_chunks", torch.zeros(0))
chunks_start_idx: torch.Tensor = torch.cumsum(num_chunks,
dim=0,
dtype=torch.int32)
chunks_start_idx: torch.Tensor = torch.cumsum(
num_chunks, dim=0, dtype=torch.int32
)
chunks_start_idx = torch.cat(
[torch.tensor([0], dtype=torch.int32), chunks_start_idx])
[torch.tensor([0], dtype=torch.int32), chunks_start_idx]
)
def get_replacement_ultravox(item_idx: int):
start = chunks_start_idx[item_idx]
@@ -251,17 +268,16 @@ class StackAudioFrames(nn.Module):
def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
B, T, C = audio_embeds.shape
T_pad = (T + self.stack_factor -
1) // self.stack_factor * self.stack_factor
T_pad = (T + self.stack_factor - 1) // self.stack_factor * self.stack_factor
audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T))
B, T, C = audio_embeds.shape
audio_embeds = audio_embeds.view(B, T // self.stack_factor,
C * self.stack_factor)
audio_embeds = audio_embeds.view(
B, T // self.stack_factor, C * self.stack_factor
)
return audio_embeds
class UltravoxProjector(nn.Module):
def __init__(self, config: UltravoxConfig):
super().__init__()
self.hidden_dim = config.hidden_size
@@ -325,12 +341,15 @@ class ModifiedWhisperEncoder(WhisperEncoder):
@property
def max_context_length(self):
return (self.config.max_source_positions * self.conv1.stride[0] *
self.conv2.stride[0])
return (
self.config.max_source_positions
* self.conv1.stride[0]
* self.conv2.stride[0]
)
def get_attention_mask_by_audio_len(self,
audio_lens: Optional[torch.Tensor],
hidden_states: torch.Tensor):
def get_attention_mask_by_audio_len(
self, audio_lens: Optional[torch.Tensor], hidden_states: torch.Tensor
):
"""
Create attention mask based on audio lengths to mask out padding tokens
For each sample in batch:
@@ -346,9 +365,9 @@ class ModifiedWhisperEncoder(WhisperEncoder):
audio_feature_len = self._get_feat_extract_output_lengths(audio_lens)
max_seq_len = hidden_states.shape[1]
attention_mask = torch.arange(max_seq_len,
device=hidden_states.device)[None, :].lt(
audio_feature_len.view(-1, 1))
attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
None, :
].lt(audio_feature_len.view(-1, 1))
attention_mask = self.get_extended_attention_mask(
attention_mask,
None,
@@ -367,21 +386,21 @@ class ModifiedWhisperEncoder(WhisperEncoder):
f"Whisper expects the mel input features to be of length "
f"{expected_seq_length} or less, but found "
f"{input_features.shape[-1]}. Make sure to pad the input mel "
f"features to {expected_seq_length}.")
f"features to {expected_seq_length}."
)
inputs_embeds = nn.functional.gelu(self.conv1(input_features))
inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
inputs_embeds = inputs_embeds.permute(0, 2, 1)
embed_pos = self.embed_positions.weight[:inputs_embeds.size(-2)]
embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
hidden_states = inputs_embeds + embed_pos
hidden_states = nn.functional.dropout(hidden_states,
p=self.dropout,
training=self.training)
hidden_states = nn.functional.dropout(
hidden_states, p=self.dropout, training=self.training
)
attention_mask = self.get_attention_mask_by_audio_len(
audio_lens, hidden_states)
attention_mask = self.get_attention_mask_by_audio_len(audio_lens, hidden_states)
for encoder_layer in self.layers:
layer_outputs = encoder_layer(
@@ -399,16 +418,17 @@ class ModifiedWhisperEncoder(WhisperEncoder):
@MULTIMODAL_REGISTRY.register_processor(
UltravoxMultiModalProcessor,
info=UltravoxProcessingInfo,
dummy_inputs=UltravoxDummyInputsBuilder)
dummy_inputs=UltravoxDummyInputsBuilder,
)
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"]
"gate_up_proj": ["gate_proj", "up_proj"],
}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."}
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
@@ -435,7 +455,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
model_or_path=config.audio_model_id,
revision=None,
prefix="audio_tower.",
))
)
)
self.multi_modal_projector = UltravoxProjector(config)
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
@@ -446,12 +467,16 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
# this prefix is not for initialization, but for loading weights
# note the trailing dot
self.secondary_weights.append(
DefaultModelLoader.Source(model_or_path=config.text_model_id,
revision=None,
prefix="language_model."))
DefaultModelLoader.Source(
model_or_path=config.text_model_id,
revision=None,
prefix="language_model.",
)
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors)
self.language_model.make_empty_intermediate_tensors
)
def get_mm_mapping(self) -> MultiModelKeys:
"""
@@ -464,8 +489,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
)
def _audio_features_to_embeddings(
self, input_features: torch.Tensor,
audio_lens: torch.Tensor) -> torch.Tensor:
self, input_features: torch.Tensor, audio_lens: torch.Tensor
) -> torch.Tensor:
audio_features = input_features.to(self.audio_tower.dtype)
batch_size = audio_features.size(0)
audio_embeddings = []
@@ -474,8 +499,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
for start in range(0, batch_size, _MAX_ENCODER_BATCH_SIZE):
end = min(start + _MAX_ENCODER_BATCH_SIZE, batch_size)
# Process through audio tower
batch_features = self.audio_tower(audio_features[start:end],
audio_lens[start:end])
batch_features = self.audio_tower(
audio_features[start:end], audio_lens[start:end]
)
batch_features = batch_features.to(self.audio_tower.dtype)
# Process through projector
@@ -487,7 +513,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
return audio_embeddings
def _parse_and_validate_audio_input(
self, **kwargs: object) -> Optional[UltravoxAudioInputs]:
self, **kwargs: object
) -> Optional[UltravoxAudioInputs]:
audio_features = kwargs.pop("audio_features", None)
audio_embeds = kwargs.pop("audio_embeds", None)
audio_lens = kwargs.pop("audio_lens", None)
@@ -497,14 +524,15 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
return None
if audio_features is not None:
return UltravoxAudioFeatureInputs(type="audio_features",
data=audio_features,
lens=audio_lens,
token_len=audio_token_len)
return UltravoxAudioFeatureInputs(
type="audio_features",
data=audio_features,
lens=audio_lens,
token_len=audio_token_len,
)
if audio_embeds is not None:
return UltravoxAudioEmbeddingInputs(type="audio_embeds",
data=audio_embeds)
return UltravoxAudioEmbeddingInputs(type="audio_embeds", data=audio_embeds)
raise AssertionError("This line should be unreachable.")
@@ -520,11 +548,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
audio_features = pad_and_concat_to_dim3(audio_input["data"])
# [B1, B2] -> [B1+B2]
audio_lens = flatten_bn(audio_input['lens'], concat=True)
audio_token_len = flatten_bn(audio_input['token_len'], concat=True)
audio_lens = flatten_bn(audio_input["lens"], concat=True)
audio_token_len = flatten_bn(audio_input["token_len"], concat=True)
embeddings = self._audio_features_to_embeddings(
audio_features, audio_lens)
embeddings = self._audio_features_to_embeddings(audio_features, audio_lens)
# We should flatten and concatenate embeddings based on token lengths
# For example, with token_len = [4, 2, 3], flattened_embeddings will be
@@ -533,23 +560,22 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
# Create a mask of valid indices based on token lengths
max_len = embeddings.shape[1]
indices = torch.arange(max_len, device=embeddings.device).expand(
embeddings.shape[0], -1)
embeddings.shape[0], -1
)
mask = indices < audio_token_len[:, None]
# Apply mask and flatten
flattened_embeddings = embeddings[mask]
# Return one tensor per input audio
embed_lens = [
token_len_item.sum().item()
for token_len_item in audio_input['token_len']
token_len_item.sum().item() for token_len_item in audio_input["token_len"]
]
return flattened_embeddings.split(embed_lens)
def get_language_model(self) -> torch.nn.Module:
return self.language_model
def get_multimodal_embeddings(self,
**kwargs: object) -> MultiModalEmbeddings:
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
audio_input = self._parse_and_validate_audio_input(**kwargs)
if audio_input is None:
return []
@@ -576,12 +602,14 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
handle_oov_mm_token=handle_oov_mm_token,
)
def forward(self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[torch.Tensor] = None,
inputs_embeds: Optional[torch.Tensor] = None,
**kwargs) -> Union[torch.Tensor, IntermediateTensors]:
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[torch.Tensor] = None,
inputs_embeds: Optional[torch.Tensor] = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
"""Run forward pass for Ultravox
One key thing to understand is the `input_ids` already accounts for the
@@ -607,25 +635,21 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
if hasattr(language_model, "language_model"):
language_model = language_model.language_model
hidden_states = language_model.model(input_ids,
positions,
intermediate_tensors,
inputs_embeds=inputs_embeds)
hidden_states = language_model.model(
input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
)
return hidden_states
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str,
torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self,
ignore_unexpected_prefixes=["audio_tower."])
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["audio_tower."])
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def pad_and_concat_to_dim3(
features: Union[torch.Tensor, list[torch.Tensor], list[list[torch.Tensor]]]
features: Union[torch.Tensor, list[torch.Tensor], list[list[torch.Tensor]]],
) -> torch.Tensor:
"""
Pad and concatenate a list of tensors.