Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
|
||||
"""PyTorch Ultravox model."""
|
||||
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal, Optional, Union
|
||||
|
||||
@@ -20,21 +21,37 @@ from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalKwargsItems, NestedTensors)
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalDataDict,
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
NestedTensors,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (BaseMultiModalProcessor,
|
||||
BaseProcessingInfo, PromptReplacement,
|
||||
PromptUpdate)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.multimodal.profiling import BaseDummyInputsBuilder
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (MultiModalEmbeddings, SupportsLoRA,
|
||||
SupportsMultiModal, SupportsPP)
|
||||
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
|
||||
init_vllm_registered_model, maybe_prefix)
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
flatten_bn,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
_AUDIO_PLACEHOLDER_OVERRIDE = "<|audio|>"
|
||||
_MAX_ENCODER_BATCH_SIZE = 16
|
||||
@@ -48,15 +65,21 @@ class UltravoxAudioFeatureInputs(TensorSchema):
|
||||
- t: Time frames (M)
|
||||
- nmb: Number of mel bins
|
||||
"""
|
||||
|
||||
type: Literal["audio_features"]
|
||||
data: Annotated[Union[torch.Tensor, list[torch.Tensor],
|
||||
list[list[torch.Tensor]]],
|
||||
TensorShape("b", "n", "nmb", "t", dynamic_dims={"n"})]
|
||||
lens: Annotated[Union[torch.Tensor, list[torch.Tensor]],
|
||||
TensorShape("b", "n", dynamic_dims={"n"})]
|
||||
data: Annotated[
|
||||
Union[torch.Tensor, list[torch.Tensor], list[list[torch.Tensor]]],
|
||||
TensorShape("b", "n", "nmb", "t", dynamic_dims={"n"}),
|
||||
]
|
||||
lens: Annotated[
|
||||
Union[torch.Tensor, list[torch.Tensor]],
|
||||
TensorShape("b", "n", dynamic_dims={"n"}),
|
||||
]
|
||||
"""Length of the audio frames. Used for attention mask in WhisperEncoder."""
|
||||
token_len: Annotated[Union[torch.Tensor, list[torch.Tensor]],
|
||||
TensorShape("b", "n", dynamic_dims={"n"})]
|
||||
token_len: Annotated[
|
||||
Union[torch.Tensor, list[torch.Tensor]],
|
||||
TensorShape("b", "n", dynamic_dims={"n"}),
|
||||
]
|
||||
"""Length of the audio tokens. Used for flattening the audio features."""
|
||||
|
||||
|
||||
@@ -68,17 +91,17 @@ class UltravoxAudioEmbeddingInputs(TensorSchema):
|
||||
- afs: audio feature size
|
||||
- hs: hidden size
|
||||
"""
|
||||
|
||||
type: Literal["audio_embeds"]
|
||||
data: Annotated[Union[torch.Tensor, list[torch.Tensor]],
|
||||
TensorShape("b", "na", "afs", "hs")]
|
||||
data: Annotated[
|
||||
Union[torch.Tensor, list[torch.Tensor]], TensorShape("b", "na", "afs", "hs")
|
||||
]
|
||||
|
||||
|
||||
UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
|
||||
UltravoxAudioEmbeddingInputs]
|
||||
UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs, UltravoxAudioEmbeddingInputs]
|
||||
|
||||
|
||||
class UltravoxProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
|
||||
config = self.ctx.model_config.hf_config
|
||||
hf_processor = self.ctx.get_hf_processor(**kwargs)
|
||||
@@ -91,8 +114,7 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
return hf_processor
|
||||
|
||||
def get_feature_extractor(self,
|
||||
**kwargs: object) -> WhisperFeatureExtractor:
|
||||
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
|
||||
hf_processor = self.get_hf_processor(**kwargs)
|
||||
audio_processor = hf_processor.audio_processor # type: ignore
|
||||
feature_extractor = audio_processor.feature_extractor # type: ignore
|
||||
@@ -103,9 +125,7 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
|
||||
return {"audio": None}
|
||||
|
||||
|
||||
class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
|
||||
):
|
||||
|
||||
class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
|
||||
@@ -120,23 +140,21 @@ class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
audio_len = (feature_extractor.chunk_length * sampling_rate *
|
||||
_MAX_ENCODER_BATCH_SIZE)
|
||||
audio_len = (
|
||||
feature_extractor.chunk_length * sampling_rate * _MAX_ENCODER_BATCH_SIZE
|
||||
)
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
|
||||
audio_overrides = mm_options.get("audio") if mm_options else None
|
||||
|
||||
return {
|
||||
"audio":
|
||||
self._get_dummy_audios(length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=audio_overrides)
|
||||
"audio": self._get_dummy_audios(
|
||||
length=audio_len, num_audios=num_audios, overrides=audio_overrides
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class UltravoxMultiModalProcessor(
|
||||
BaseMultiModalProcessor[UltravoxProcessingInfo]):
|
||||
|
||||
class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo]):
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
|
||||
@@ -151,7 +169,8 @@ class UltravoxMultiModalProcessor(
|
||||
# Text-only input not supported in composite processor
|
||||
if not mm_data.get("audios", []):
|
||||
prompt_ids = self.info.get_tokenizer().encode(
|
||||
prompt, add_special_tokens=False)
|
||||
prompt, add_special_tokens=False
|
||||
)
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
|
||||
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
|
||||
|
||||
@@ -178,7 +197,7 @@ class UltravoxMultiModalProcessor(
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
output['audio_features'] = output.pop('audio_values')
|
||||
output["audio_features"] = output.pop("audio_values")
|
||||
|
||||
return output
|
||||
|
||||
@@ -187,17 +206,14 @@ class UltravoxMultiModalProcessor(
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
num_chunks = hf_inputs.get('audio_num_chunks', torch.zeros(0))
|
||||
num_chunks = hf_inputs.get("audio_num_chunks", torch.zeros(0))
|
||||
return dict(
|
||||
# to handle longer than 30s audio, each audio might be split
|
||||
# into multiple chunks as such, their batch dimension can be
|
||||
# higher than the number of audio samples
|
||||
audio_features=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", num_chunks),
|
||||
audio_token_len=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", num_chunks),
|
||||
audio_lens=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", num_chunks),
|
||||
audio_features=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
|
||||
audio_token_len=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
|
||||
audio_lens=MultiModalFieldConfig.flat_from_sizes("audio", num_chunks),
|
||||
# num_chunks can convert audio_chunked to audio batch dimension
|
||||
audio_num_chunks=MultiModalFieldConfig.batched("audio"),
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
@@ -218,11 +234,12 @@ class UltravoxMultiModalProcessor(
|
||||
# belonging to the i-th audio.
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
num_chunks = out_mm_data.get("audio_num_chunks", torch.zeros(0))
|
||||
chunks_start_idx: torch.Tensor = torch.cumsum(num_chunks,
|
||||
dim=0,
|
||||
dtype=torch.int32)
|
||||
chunks_start_idx: torch.Tensor = torch.cumsum(
|
||||
num_chunks, dim=0, dtype=torch.int32
|
||||
)
|
||||
chunks_start_idx = torch.cat(
|
||||
[torch.tensor([0], dtype=torch.int32), chunks_start_idx])
|
||||
[torch.tensor([0], dtype=torch.int32), chunks_start_idx]
|
||||
)
|
||||
|
||||
def get_replacement_ultravox(item_idx: int):
|
||||
start = chunks_start_idx[item_idx]
|
||||
@@ -251,17 +268,16 @@ class StackAudioFrames(nn.Module):
|
||||
|
||||
def forward(self, audio_embeds: torch.Tensor) -> torch.Tensor:
|
||||
B, T, C = audio_embeds.shape
|
||||
T_pad = (T + self.stack_factor -
|
||||
1) // self.stack_factor * self.stack_factor
|
||||
T_pad = (T + self.stack_factor - 1) // self.stack_factor * self.stack_factor
|
||||
audio_embeds = F.pad(audio_embeds, (0, 0, 0, T_pad - T))
|
||||
B, T, C = audio_embeds.shape
|
||||
audio_embeds = audio_embeds.view(B, T // self.stack_factor,
|
||||
C * self.stack_factor)
|
||||
audio_embeds = audio_embeds.view(
|
||||
B, T // self.stack_factor, C * self.stack_factor
|
||||
)
|
||||
return audio_embeds
|
||||
|
||||
|
||||
class UltravoxProjector(nn.Module):
|
||||
|
||||
def __init__(self, config: UltravoxConfig):
|
||||
super().__init__()
|
||||
self.hidden_dim = config.hidden_size
|
||||
@@ -325,12 +341,15 @@ class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
|
||||
@property
|
||||
def max_context_length(self):
|
||||
return (self.config.max_source_positions * self.conv1.stride[0] *
|
||||
self.conv2.stride[0])
|
||||
return (
|
||||
self.config.max_source_positions
|
||||
* self.conv1.stride[0]
|
||||
* self.conv2.stride[0]
|
||||
)
|
||||
|
||||
def get_attention_mask_by_audio_len(self,
|
||||
audio_lens: Optional[torch.Tensor],
|
||||
hidden_states: torch.Tensor):
|
||||
def get_attention_mask_by_audio_len(
|
||||
self, audio_lens: Optional[torch.Tensor], hidden_states: torch.Tensor
|
||||
):
|
||||
"""
|
||||
Create attention mask based on audio lengths to mask out padding tokens
|
||||
For each sample in batch:
|
||||
@@ -346,9 +365,9 @@ class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
|
||||
audio_feature_len = self._get_feat_extract_output_lengths(audio_lens)
|
||||
max_seq_len = hidden_states.shape[1]
|
||||
attention_mask = torch.arange(max_seq_len,
|
||||
device=hidden_states.device)[None, :].lt(
|
||||
audio_feature_len.view(-1, 1))
|
||||
attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
|
||||
None, :
|
||||
].lt(audio_feature_len.view(-1, 1))
|
||||
attention_mask = self.get_extended_attention_mask(
|
||||
attention_mask,
|
||||
None,
|
||||
@@ -367,21 +386,21 @@ class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
f"Whisper expects the mel input features to be of length "
|
||||
f"{expected_seq_length} or less, but found "
|
||||
f"{input_features.shape[-1]}. Make sure to pad the input mel "
|
||||
f"features to {expected_seq_length}.")
|
||||
f"features to {expected_seq_length}."
|
||||
)
|
||||
|
||||
inputs_embeds = nn.functional.gelu(self.conv1(input_features))
|
||||
inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
|
||||
|
||||
inputs_embeds = inputs_embeds.permute(0, 2, 1)
|
||||
embed_pos = self.embed_positions.weight[:inputs_embeds.size(-2)]
|
||||
embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
|
||||
|
||||
hidden_states = inputs_embeds + embed_pos
|
||||
hidden_states = nn.functional.dropout(hidden_states,
|
||||
p=self.dropout,
|
||||
training=self.training)
|
||||
hidden_states = nn.functional.dropout(
|
||||
hidden_states, p=self.dropout, training=self.training
|
||||
)
|
||||
|
||||
attention_mask = self.get_attention_mask_by_audio_len(
|
||||
audio_lens, hidden_states)
|
||||
attention_mask = self.get_attention_mask_by_audio_len(audio_lens, hidden_states)
|
||||
|
||||
for encoder_layer in self.layers:
|
||||
layer_outputs = encoder_layer(
|
||||
@@ -399,16 +418,17 @@ class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
UltravoxMultiModalProcessor,
|
||||
info=UltravoxProcessingInfo,
|
||||
dummy_inputs=UltravoxDummyInputsBuilder)
|
||||
dummy_inputs=UltravoxDummyInputsBuilder,
|
||||
)
|
||||
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"]
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
|
||||
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
|
||||
@@ -435,7 +455,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
model_or_path=config.audio_model_id,
|
||||
revision=None,
|
||||
prefix="audio_tower.",
|
||||
))
|
||||
)
|
||||
)
|
||||
self.multi_modal_projector = UltravoxProjector(config)
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
@@ -446,12 +467,16 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
# this prefix is not for initialization, but for loading weights
|
||||
# note the trailing dot
|
||||
self.secondary_weights.append(
|
||||
DefaultModelLoader.Source(model_or_path=config.text_model_id,
|
||||
revision=None,
|
||||
prefix="language_model."))
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=config.text_model_id,
|
||||
revision=None,
|
||||
prefix="language_model.",
|
||||
)
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors)
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
@@ -464,8 +489,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
)
|
||||
|
||||
def _audio_features_to_embeddings(
|
||||
self, input_features: torch.Tensor,
|
||||
audio_lens: torch.Tensor) -> torch.Tensor:
|
||||
self, input_features: torch.Tensor, audio_lens: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
audio_features = input_features.to(self.audio_tower.dtype)
|
||||
batch_size = audio_features.size(0)
|
||||
audio_embeddings = []
|
||||
@@ -474,8 +499,9 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
for start in range(0, batch_size, _MAX_ENCODER_BATCH_SIZE):
|
||||
end = min(start + _MAX_ENCODER_BATCH_SIZE, batch_size)
|
||||
# Process through audio tower
|
||||
batch_features = self.audio_tower(audio_features[start:end],
|
||||
audio_lens[start:end])
|
||||
batch_features = self.audio_tower(
|
||||
audio_features[start:end], audio_lens[start:end]
|
||||
)
|
||||
batch_features = batch_features.to(self.audio_tower.dtype)
|
||||
|
||||
# Process through projector
|
||||
@@ -487,7 +513,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
return audio_embeddings
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
self, **kwargs: object) -> Optional[UltravoxAudioInputs]:
|
||||
self, **kwargs: object
|
||||
) -> Optional[UltravoxAudioInputs]:
|
||||
audio_features = kwargs.pop("audio_features", None)
|
||||
audio_embeds = kwargs.pop("audio_embeds", None)
|
||||
audio_lens = kwargs.pop("audio_lens", None)
|
||||
@@ -497,14 +524,15 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
return None
|
||||
|
||||
if audio_features is not None:
|
||||
return UltravoxAudioFeatureInputs(type="audio_features",
|
||||
data=audio_features,
|
||||
lens=audio_lens,
|
||||
token_len=audio_token_len)
|
||||
return UltravoxAudioFeatureInputs(
|
||||
type="audio_features",
|
||||
data=audio_features,
|
||||
lens=audio_lens,
|
||||
token_len=audio_token_len,
|
||||
)
|
||||
|
||||
if audio_embeds is not None:
|
||||
return UltravoxAudioEmbeddingInputs(type="audio_embeds",
|
||||
data=audio_embeds)
|
||||
return UltravoxAudioEmbeddingInputs(type="audio_embeds", data=audio_embeds)
|
||||
|
||||
raise AssertionError("This line should be unreachable.")
|
||||
|
||||
@@ -520,11 +548,10 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
audio_features = pad_and_concat_to_dim3(audio_input["data"])
|
||||
|
||||
# [B1, B2] -> [B1+B2]
|
||||
audio_lens = flatten_bn(audio_input['lens'], concat=True)
|
||||
audio_token_len = flatten_bn(audio_input['token_len'], concat=True)
|
||||
audio_lens = flatten_bn(audio_input["lens"], concat=True)
|
||||
audio_token_len = flatten_bn(audio_input["token_len"], concat=True)
|
||||
|
||||
embeddings = self._audio_features_to_embeddings(
|
||||
audio_features, audio_lens)
|
||||
embeddings = self._audio_features_to_embeddings(audio_features, audio_lens)
|
||||
|
||||
# We should flatten and concatenate embeddings based on token lengths
|
||||
# For example, with token_len = [4, 2, 3], flattened_embeddings will be
|
||||
@@ -533,23 +560,22 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
# Create a mask of valid indices based on token lengths
|
||||
max_len = embeddings.shape[1]
|
||||
indices = torch.arange(max_len, device=embeddings.device).expand(
|
||||
embeddings.shape[0], -1)
|
||||
embeddings.shape[0], -1
|
||||
)
|
||||
mask = indices < audio_token_len[:, None]
|
||||
# Apply mask and flatten
|
||||
flattened_embeddings = embeddings[mask]
|
||||
|
||||
# Return one tensor per input audio
|
||||
embed_lens = [
|
||||
token_len_item.sum().item()
|
||||
for token_len_item in audio_input['token_len']
|
||||
token_len_item.sum().item() for token_len_item in audio_input["token_len"]
|
||||
]
|
||||
return flattened_embeddings.split(embed_lens)
|
||||
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.language_model
|
||||
|
||||
def get_multimodal_embeddings(self,
|
||||
**kwargs: object) -> MultiModalEmbeddings:
|
||||
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
if audio_input is None:
|
||||
return []
|
||||
@@ -576,12 +602,14 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
handle_oov_mm_token=handle_oov_mm_token,
|
||||
)
|
||||
|
||||
def forward(self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[torch.Tensor] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: Optional[torch.Tensor] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
) -> Union[torch.Tensor, IntermediateTensors]:
|
||||
"""Run forward pass for Ultravox
|
||||
|
||||
One key thing to understand is the `input_ids` already accounts for the
|
||||
@@ -607,25 +635,21 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
if hasattr(language_model, "language_model"):
|
||||
language_model = language_model.language_model
|
||||
|
||||
hidden_states = language_model.model(input_ids,
|
||||
positions,
|
||||
intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds)
|
||||
hidden_states = language_model.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
|
||||
loader = AutoWeightsLoader(self,
|
||||
ignore_unexpected_prefixes=["audio_tower."])
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["audio_tower."])
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
def pad_and_concat_to_dim3(
|
||||
features: Union[torch.Tensor, list[torch.Tensor], list[list[torch.Tensor]]]
|
||||
features: Union[torch.Tensor, list[torch.Tensor], list[list[torch.Tensor]]],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pad and concatenate a list of tensors.
|
||||
|
||||
Reference in New Issue
Block a user