Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -526,6 +526,9 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -134,6 +134,8 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction):
|
||||
|
||||
# Mark this as a pooling model so vLLM routes to pooler path
|
||||
is_pooling_model = True
|
||||
# Mark as false to avoid collision with the checkpoint's custom code path
|
||||
supports_mm_device_do_normalize = False
|
||||
|
||||
# Override hf_to_vllm_mapper to handle ColQwen3 weight naming.
|
||||
# NOTE: WeightsMapper applies ALL matching prefix rules sequentially
|
||||
|
||||
@@ -1493,7 +1493,7 @@ class DiffusionSampler:
|
||||
# [tile * W, vocab] copies, so a tile is also bounded by free memory.
|
||||
widths_np = states.canvas_width_np[decode_slots_np]
|
||||
order = np.argsort(widths_np, kind="stable")
|
||||
free = current_platform.mem_get_info()[0] if num_decode > 0 else 0
|
||||
free = torch.accelerator.get_memory_info()[0] if num_decode > 0 else 0
|
||||
run_start = 0
|
||||
while run_start < num_decode:
|
||||
W = int(widths_np[order[run_start]])
|
||||
|
||||
@@ -529,7 +529,7 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
|
||||
return TokenizersBackend.from_pretrained(
|
||||
self.model_id,
|
||||
subfolder="speech_tokenizer",
|
||||
revision=self.ctx.model_config.tokenizer_revision,
|
||||
revision=self.ctx.model_config.revision,
|
||||
)
|
||||
|
||||
def get_feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
|
||||
@@ -846,13 +846,22 @@ class Gemma4SelfDecoderLayers(nn.Module):
|
||||
# Shared references to modules owned by Gemma4Model — must be
|
||||
# inside this nn.Module so torch.compile captures them.
|
||||
self.embed_tokens = embed_tokens
|
||||
self.normalizer = normalizer
|
||||
self.embed_tokens_per_layer = embed_tokens_per_layer
|
||||
self.embed_scale_per_layer = embed_scale_per_layer
|
||||
self.per_layer_model_projection = per_layer_model_projection
|
||||
self.per_layer_projection_norm = per_layer_projection_norm
|
||||
self.per_layer_input_scale = per_layer_input_scale
|
||||
self.per_layer_projection_scale = per_layer_projection_scale
|
||||
|
||||
# Register the shared scalars as buffers so they follow the model on
|
||||
# .to(device); plain attributes stay on CPU.
|
||||
self.register_buffer("normalizer", normalizer, persistent=False)
|
||||
self.register_buffer(
|
||||
"embed_scale_per_layer", embed_scale_per_layer, persistent=False
|
||||
)
|
||||
self.register_buffer(
|
||||
"per_layer_input_scale", per_layer_input_scale, persistent=False
|
||||
)
|
||||
self.register_buffer(
|
||||
"per_layer_projection_scale", per_layer_projection_scale, persistent=False
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids) * self.normalizer
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -575,6 +576,7 @@ class InternS1ProForConditionalGeneration(
|
||||
self.visual = Qwen3_VisionTransformer(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
@@ -496,6 +497,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
from typing import Annotated, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
|
||||
) -> PixtralHFEncoderInfo:
|
||||
processor = self.get_hf_processor()
|
||||
size = processor.image_processor.size
|
||||
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
|
||||
if override_size := merged_kwargs.get("size"):
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(
|
||||
mm_processor_kwargs or {}
|
||||
)
|
||||
image_mm_kwargs = cast(
|
||||
Mapping[str, Any],
|
||||
merged_mm_kwargs.get("images_kwargs", {}),
|
||||
)
|
||||
if override_size := image_mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
|
||||
image_size = size["longest_edge"]
|
||||
|
||||
@@ -1976,7 +1976,8 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
for _ in range(max_batch_size)
|
||||
]
|
||||
|
||||
# Create dummy pixel_values
|
||||
# Create dummy pixel_values; uint8 when normalization is fused
|
||||
# on-device. Contents are overwritten before every replay.
|
||||
patch_embed = self.visual.patch_embed
|
||||
in_channels = patch_embed.proj.in_channels
|
||||
patch_size = patch_embed.patch_size
|
||||
@@ -1985,8 +1986,11 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
flattened_patch_size = (
|
||||
in_channels * temporal_patch_size * patch_size * patch_size
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
total_patches, flattened_patch_size, device=device, dtype=dtype
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
flattened_patch_size,
|
||||
device=device,
|
||||
dtype=self.visual.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# Override max_seqlen with a safe upper bound for capture.
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
import math
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
from functools import partial
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
from typing import Annotated, Any, Literal, TypeAlias, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None, "video": None}
|
||||
|
||||
@staticmethod
|
||||
def _complete_mm_processor_size_aliases(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Complete missing Qwen-VL size aliases.
|
||||
|
||||
Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and
|
||||
``size.longest_edge`` / ``max_pixels`` as equivalent controls. When
|
||||
only one representation is provided with a non-``None`` value, add the
|
||||
missing equivalent representation. If both are already present, leave
|
||||
them unchanged even when their values conflict.
|
||||
|
||||
Apply this independently to the flat kwargs and to each existing
|
||||
``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated
|
||||
between these mappings.
|
||||
"""
|
||||
|
||||
def add_missing_size_aliases(
|
||||
kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""Add missing size aliases within a kwargs mapping."""
|
||||
out_kwargs = dict(kwargs)
|
||||
# Copy `size` separately because `out_kwargs` is a shallow copy and
|
||||
# size edges may be added below.
|
||||
raw_size = cast(
|
||||
Mapping[str, object] | None,
|
||||
out_kwargs.get("size"),
|
||||
)
|
||||
size_kwargs = {} if raw_size is None else dict(raw_size)
|
||||
|
||||
for edge_key, pixel_key in (
|
||||
("shortest_edge", "min_pixels"),
|
||||
("longest_edge", "max_pixels"),
|
||||
):
|
||||
# Add the pixel alias from `size` when only the size edge is
|
||||
# present.
|
||||
if pixel_key not in out_kwargs:
|
||||
if edge_key in size_kwargs and size_kwargs[edge_key] is not None:
|
||||
out_kwargs[pixel_key] = size_kwargs[edge_key]
|
||||
continue
|
||||
|
||||
pixel_value = out_kwargs[pixel_key]
|
||||
# HF treats `min_pixels` / `max_pixels` set to None as no size
|
||||
# override, so do not propagate them to the equivalent size edge.
|
||||
if pixel_value is None:
|
||||
continue
|
||||
|
||||
# Add the size edge when only the pixel alias is present.
|
||||
if edge_key not in size_kwargs:
|
||||
size_kwargs[edge_key] = pixel_value
|
||||
out_kwargs["size"] = size_kwargs
|
||||
|
||||
return out_kwargs
|
||||
|
||||
prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs)
|
||||
# Complete aliases independently inside existing image/video kwargs;
|
||||
# flat values are not copied into the nested mappings.
|
||||
for nested_kwargs_key in ("images_kwargs", "videos_kwargs"):
|
||||
nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key)
|
||||
if isinstance(nested_kwargs, Mapping):
|
||||
prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases(
|
||||
nested_kwargs
|
||||
)
|
||||
|
||||
return prepared_mm_kwargs
|
||||
|
||||
def _merge_and_resolve_mm_processor_kwargs(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> dict[str, Any]:
|
||||
"""Merge configured and request Qwen-VL ``mm_processor_kwargs``.
|
||||
|
||||
Request-side ``size.shortest_edge`` / ``min_pixels`` and
|
||||
``size.longest_edge`` / ``max_pixels`` aliases are completed before the
|
||||
merge so request precedence is preserved across equivalent
|
||||
representations.
|
||||
"""
|
||||
# Complete request-side Qwen-VL size aliases before merging with
|
||||
# configured kwargs.
|
||||
prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs)
|
||||
|
||||
return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _get_vision_size(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
default_size: Mapping[str, int] | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Resolve Qwen-VL vision size kwargs with optional processor defaults.
|
||||
|
||||
``size`` overrides any processor defaults, then ``min_pixels`` and
|
||||
``max_pixels`` override the corresponding size edges.
|
||||
"""
|
||||
size = dict(default_size) if default_size is not None else {}
|
||||
|
||||
override_size = cast(
|
||||
Mapping[str, int] | None,
|
||||
mm_kwargs.get("size"),
|
||||
)
|
||||
if override_size is not None:
|
||||
size.update(override_size)
|
||||
|
||||
min_pixels = cast(int | None, mm_kwargs.get("min_pixels"))
|
||||
if min_pixels is not None:
|
||||
size["shortest_edge"] = min_pixels
|
||||
|
||||
max_pixels = cast(int | None, mm_kwargs.get("max_pixels"))
|
||||
if max_pixels is not None:
|
||||
size["longest_edge"] = max_pixels
|
||||
|
||||
return size
|
||||
|
||||
def get_mm_max_tokens_per_item(
|
||||
self,
|
||||
seq_len: int,
|
||||
@@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
do_resize: bool = True,
|
||||
image_processor: Qwen2VLImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str | None = None,
|
||||
modality: Literal["image", "video"] | None = None,
|
||||
) -> tuple[ImageSize, int]:
|
||||
hf_config = self.get_hf_config()
|
||||
vision_config = hf_config.vision_config
|
||||
@@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
|
||||
size = size | {"shortest_edge": override_min_pixels}
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
size = size | {"longest_edge": override_max_pixels}
|
||||
if modality is None:
|
||||
modality = "image"
|
||||
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
scope = "images_kwargs" if modality == "image" else "videos_kwargs"
|
||||
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
|
||||
size = self._get_vision_size(
|
||||
scoped_mm_kwargs,
|
||||
default_size=image_processor.size,
|
||||
)
|
||||
|
||||
if do_resize:
|
||||
resized_height, resized_width = smart_resize(
|
||||
@@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
if max_pixels is None:
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
|
||||
size = size | {"shortest_edge": override_min_pixels}
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
size = size | {"longest_edge": override_max_pixels}
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
|
||||
image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {})
|
||||
size = self._get_vision_size(
|
||||
image_mm_kwargs,
|
||||
default_size=image_processor.size,
|
||||
)
|
||||
|
||||
max_pixels = size["longest_edge"]
|
||||
|
||||
@@ -1616,7 +1728,8 @@ class Qwen2VLForConditionalGeneration(
|
||||
for _ in range(max_batch_size)
|
||||
]
|
||||
|
||||
# Create dummy pixel_values.
|
||||
# Create dummy pixel_values; uint8 when normalization is fused
|
||||
# on-device. Contents are overwritten before every replay.
|
||||
patch_embed = self.visual.patch_embed
|
||||
in_channels = patch_embed.proj.in_channels
|
||||
patch_size = patch_embed.patch_size
|
||||
@@ -1625,8 +1738,11 @@ class Qwen2VLForConditionalGeneration(
|
||||
flattened_patch_size = (
|
||||
in_channels * temporal_patch_size * patch_size * patch_size
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
total_patches, flattened_patch_size, device=device, dtype=dtype
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
flattened_patch_size,
|
||||
device=device,
|
||||
dtype=self.visual.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# max_seqlen.item() gets baked into the CUDA graph at capture time.
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
@@ -519,6 +520,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
@@ -741,6 +743,7 @@ class Qwen3_5MoeForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
@@ -622,15 +621,6 @@ class DFlashQwen3Model(nn.Module):
|
||||
# --- Grouped RMSNorm K across all layers ([L, num_ctx, nkv, hd]) ---
|
||||
# The weight is selected per layer by the outermost (layer) index.
|
||||
all_k_normed = torch.empty_like(all_k)
|
||||
if current_platform.is_xpu():
|
||||
for layer_idx in range(all_k.shape[0]):
|
||||
ops.rms_norm(
|
||||
all_k_normed[layer_idx],
|
||||
all_k[layer_idx],
|
||||
self._k_norm_weights[layer_idx],
|
||||
self._rms_norm_eps,
|
||||
)
|
||||
return all_k_normed
|
||||
ops.rms_norm(
|
||||
all_k_normed,
|
||||
all_k,
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
||||
from functools import lru_cache, partial
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -67,6 +67,10 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -566,6 +570,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
vision_config: Qwen3VLVisionConfig,
|
||||
norm_eps: float = 1e-6,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -605,6 +610,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
in_channels=vision_config.in_channels,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
|
||||
|
||||
@@ -858,7 +864,9 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = x.to(device=self.device, dtype=self.dtype, non_blocking=True)
|
||||
hidden_states = self.input_norm(
|
||||
x.to(device=self.device, non_blocking=True), self.dtype
|
||||
)
|
||||
hidden_states = self.patch_embed(hidden_states)
|
||||
|
||||
if encoder_metadata is None:
|
||||
@@ -936,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
|
||||
supported = super().get_supported_mm_processor_kwargs()
|
||||
|
||||
# vLLM has historically accepted `min_pixels` / `max_pixels` for
|
||||
# Qwen3-VL video. Include them in the routing schema even though
|
||||
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
|
||||
# folded into `size` before calling HF.
|
||||
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
|
||||
return supported
|
||||
|
||||
def _get_vision_info(
|
||||
self,
|
||||
*,
|
||||
@@ -945,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
do_resize: bool = True,
|
||||
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str | None = None,
|
||||
modality: Literal["image", "video"] | None = None,
|
||||
) -> tuple[ImageSize, int]:
|
||||
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
|
||||
|
||||
@@ -957,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
|
||||
if modality is None:
|
||||
modality = "video" if is_video else "image"
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
|
||||
size = size | {"shortest_edge": override_min_pixels}
|
||||
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
|
||||
size = size | {"longest_edge": override_max_pixels}
|
||||
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
|
||||
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
|
||||
size = self._get_vision_size(
|
||||
scoped_mm_kwargs,
|
||||
default_size=image_processor.size,
|
||||
)
|
||||
|
||||
if do_resize:
|
||||
if is_video:
|
||||
@@ -1022,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
) -> int:
|
||||
video_processor = self.get_video_processor()
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
|
||||
video_size = mm_kwargs.get("size", video_processor.size)
|
||||
temporal_patch_size = mm_kwargs.get(
|
||||
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
|
||||
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
|
||||
video_size = self._get_vision_size(
|
||||
video_mm_kwargs,
|
||||
default_size=video_processor.size,
|
||||
)
|
||||
temporal_patch_size = video_mm_kwargs.get(
|
||||
"temporal_patch_size", video_processor.temporal_patch_size
|
||||
)
|
||||
|
||||
@@ -1154,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
|
||||
video_processor = self.info.get_video_processor()
|
||||
|
||||
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
|
||||
video_size = mm_kwargs.get("size", video_processor.size)
|
||||
temporal_patch_size = mm_kwargs.get(
|
||||
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
|
||||
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
|
||||
video_size = self.info._get_vision_size(
|
||||
video_mm_kwargs,
|
||||
default_size=video_processor.size,
|
||||
)
|
||||
temporal_patch_size = video_mm_kwargs.get(
|
||||
"temporal_patch_size", video_processor.temporal_patch_size
|
||||
)
|
||||
|
||||
@@ -1172,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
# over enough frames that the cap is not binding for the dummy.
|
||||
# This takes precedence over a shrinking num_frames override, which
|
||||
# would otherwise reintroduce the under-profiling.
|
||||
if mm_kwargs.get("cap_pixels_per_frame"):
|
||||
max_video_tokens = mm_kwargs.get(
|
||||
if video_mm_kwargs.get("cap_pixels_per_frame"):
|
||||
max_video_tokens = video_mm_kwargs.get(
|
||||
"max_video_tokens",
|
||||
getattr(video_processor, "max_video_tokens", 768),
|
||||
)
|
||||
patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
|
||||
merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
|
||||
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
|
||||
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
|
||||
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
|
||||
target_num_frames = max(
|
||||
target_num_frames,
|
||||
@@ -1340,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
# Merge and resolve the multimodal processor kwargs once so every HF
|
||||
# call below starts from the same engine/request state and scoped form.
|
||||
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
|
||||
|
||||
if "images_kwargs" in merged_mm_kwargs:
|
||||
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
|
||||
|
||||
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
|
||||
# override is passed, fill missing edges from the processor defaults
|
||||
# before any HF call.
|
||||
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
default_image_size = self.info.get_image_processor().size
|
||||
image_scoped_kwargs["size"] = self.info._get_vision_size(
|
||||
image_scoped_kwargs,
|
||||
default_size=default_image_size,
|
||||
)
|
||||
|
||||
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
|
||||
|
||||
if "videos_kwargs" in merged_mm_kwargs:
|
||||
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
|
||||
|
||||
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
|
||||
# override is passed, fill missing edges from the processor defaults
|
||||
# before any HF call.
|
||||
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
default_video_size = self.info.get_video_processor().size
|
||||
video_scoped_kwargs["size"] = self.info._get_vision_size(
|
||||
video_scoped_kwargs,
|
||||
default_size=default_video_size,
|
||||
)
|
||||
|
||||
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
|
||||
# compatibility kwargs. They have already been folded into `size`,
|
||||
# so remove them before any HF call, including calls that contain
|
||||
# only text/images.
|
||||
video_scoped_kwargs.pop("min_pixels", None)
|
||||
video_scoped_kwargs.pop("max_pixels", None)
|
||||
|
||||
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
|
||||
|
||||
# Separate video processing from image processing. Because the videos
|
||||
# are processed into several image patches
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
@@ -1365,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# used to calculate the timestamps. Make sure that
|
||||
# do_sample_frames in mm_kwargs is false for presampled videos.
|
||||
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
|
||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
video_size = dict(self.info.get_video_processor().size)
|
||||
size_override = merged.get("size")
|
||||
if size_override is not None:
|
||||
video_size = video_size | size_override
|
||||
min_pixels = merged.get("min_pixels")
|
||||
if min_pixels is not None:
|
||||
video_size["shortest_edge"] = min_pixels
|
||||
max_pixels = merged.get("max_pixels")
|
||||
if max_pixels is not None:
|
||||
video_size["longest_edge"] = max_pixels
|
||||
video_mm_kwargs["size"] = video_size
|
||||
sampled_fps = video_mm_kwargs.get("fps")
|
||||
# The root copy is shallow, so give each item its own video
|
||||
# scope before modifying per-video values below.
|
||||
video_mm_kwargs = dict(merged_mm_kwargs)
|
||||
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
|
||||
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
|
||||
|
||||
sampled_fps = video_scoped_kwargs.get("fps")
|
||||
if is_list_of(sampled_fps, float):
|
||||
video_mm_kwargs["fps"] = sampled_fps[item_idx]
|
||||
sampled_num_frames = video_mm_kwargs.get("num_frames")
|
||||
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
|
||||
sampled_num_frames = video_scoped_kwargs.get("num_frames")
|
||||
if is_list_of(sampled_num_frames, int):
|
||||
video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
|
||||
if "do_sample_frames" not in video_mm_kwargs:
|
||||
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
|
||||
if "do_sample_frames" not in video_scoped_kwargs:
|
||||
# qwen_vl_utils already has "do_sample_frames" in
|
||||
# mm_kwargs, don't overwrite it.
|
||||
video_mm_kwargs["do_sample_frames"] = metadata.get(
|
||||
video_scoped_kwargs["do_sample_frames"] = metadata.get(
|
||||
"do_sample_frames", False
|
||||
)
|
||||
|
||||
@@ -1401,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# Compute timestamps here where we have access to metadata
|
||||
timestamps = self.info._get_video_second_idx(
|
||||
metadata=metadata,
|
||||
do_sample_frames=video_mm_kwargs["do_sample_frames"],
|
||||
sampled_fps=video_mm_kwargs.get("fps"),
|
||||
sampled_num_frames=video_mm_kwargs.get("num_frames"),
|
||||
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
|
||||
sampled_fps=video_scoped_kwargs.get("fps"),
|
||||
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
|
||||
)
|
||||
timestamps_per_video.append(timestamps)
|
||||
|
||||
@@ -1415,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# to prevent HF's BaseVideoProcessor.preprocess() from
|
||||
# filling in the class default (fps=2) via setdefault(),
|
||||
# which would conflict with num_frames (mutually exclusive).
|
||||
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
|
||||
video_mm_kwargs["fps"] = None
|
||||
if (
|
||||
"num_frames" in video_scoped_kwargs
|
||||
and "fps" not in video_scoped_kwargs
|
||||
):
|
||||
video_scoped_kwargs["fps"] = None
|
||||
|
||||
video_outputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**video_mm_kwargs),
|
||||
@@ -1477,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
else:
|
||||
video_outputs = dict()
|
||||
|
||||
# fps/num_frames are video-only kwargs already consumed by the loop;
|
||||
# exclude them so the text/image processor call below never gets a list.
|
||||
non_video_mm_kwargs = {
|
||||
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
|
||||
}
|
||||
# The root copy is shallow, so copy the video scope before removing
|
||||
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
|
||||
non_video_mm_kwargs = dict(merged_mm_kwargs)
|
||||
if "videos_kwargs" in non_video_mm_kwargs:
|
||||
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
|
||||
non_video_scoped_kwargs.pop("fps", None)
|
||||
non_video_scoped_kwargs.pop("num_frames", None)
|
||||
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||
hf_data,
|
||||
@@ -1813,6 +1876,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_tower_connector_lora = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
supported_video_pruning_methods = ("evs", "vidcom2")
|
||||
|
||||
@@ -1863,6 +1927,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
@@ -2172,8 +2237,11 @@ class Qwen3VLForConditionalGeneration(
|
||||
flattened_patch_size = (
|
||||
in_channels * temporal_patch_size * patch_size * patch_size
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
total_patches, flattened_patch_size, device=device, dtype=dtype
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
flattened_patch_size,
|
||||
device=device,
|
||||
dtype=self.visual.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# Override max_seqlen with a safe upper bound for capture.
|
||||
@@ -2307,7 +2375,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
if image_input["type"] == "image_embeds":
|
||||
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
|
||||
else:
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
pixel_values = image_input["pixel_values"]
|
||||
if self.use_data_parallel:
|
||||
return run_dp_sharded_mrope_vision_model(
|
||||
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
|
||||
@@ -2329,9 +2397,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
if video_input["type"] == "video_embeds":
|
||||
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
|
||||
else:
|
||||
pixel_values_videos = video_input["pixel_values_videos"].type(
|
||||
self.visual.dtype
|
||||
)
|
||||
pixel_values_videos = video_input["pixel_values_videos"]
|
||||
if self.use_data_parallel:
|
||||
grid_thw_list = grid_thw.tolist()
|
||||
return run_dp_sharded_mrope_vision_model(
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -254,6 +255,7 @@ class Qwen3VLMoeForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import itertools
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
from typing import Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
@@ -24,10 +24,12 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_C = TypeVar("_C", bound=PreTrainedConfig)
|
||||
_C_co = TypeVar("_C_co", bound=PreTrainedConfig, covariant=True)
|
||||
|
||||
|
||||
class _RootConfig(Protocol[_C]):
|
||||
vision_config: _C
|
||||
class _RootConfig(Protocol[_C_co]):
|
||||
@property
|
||||
def vision_config(self) -> _C_co: ...
|
||||
|
||||
|
||||
class VisionEncoderInfo(ABC, Generic[_C]):
|
||||
@@ -60,7 +62,8 @@ class VisionEncoderInfo(ABC, Generic[_C]):
|
||||
|
||||
|
||||
class VisionLanguageConfig(Protocol):
|
||||
vision_config: Final[PreTrainedConfig]
|
||||
@property
|
||||
def vision_config(self) -> PreTrainedConfig: ...
|
||||
|
||||
|
||||
def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
|
||||
@@ -317,7 +317,7 @@ class VoxtralForConditionalGeneration(
|
||||
|
||||
# update quant config to so that ignored module and target module names
|
||||
# match the vLLM model names
|
||||
if hasattr(vllm_config, "quant_config"):
|
||||
if vllm_config.quant_config is not None:
|
||||
vllm_config.quant_config = self.maybe_update_quant_config(
|
||||
vllm_config.quant_config
|
||||
)
|
||||
@@ -724,6 +724,7 @@ class VoxtralEncoderModel(nn.Module):
|
||||
self.config = cast(WhisperConfig, vllm_config.model_config.hf_config)
|
||||
self.dtype: torch.dtype = vllm_config.model_config.dtype
|
||||
self.is_causal = getattr(self.config, "is_causal", False)
|
||||
WhisperEncoderCls: Callable[..., WhisperEncoder | WhisperCausalEncoder]
|
||||
if self.is_causal:
|
||||
WhisperEncoderCls = WhisperCausalEncoder
|
||||
else:
|
||||
@@ -830,7 +831,7 @@ class VoxtralEncoderModel(nn.Module):
|
||||
return results
|
||||
|
||||
def load_weight(self, weight: tuple[str, torch.Tensor]) -> str:
|
||||
stacked_params_mapping = [
|
||||
stacked_params_mapping: list[tuple[str, str, str | int]] = [
|
||||
# (param_name, shard_name, shard_id)
|
||||
("qkv_proj", "q_proj", "q"),
|
||||
("qkv_proj", "k_proj", "k"),
|
||||
|
||||
@@ -16,7 +16,6 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.inputs import PromptType, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime
|
||||
@@ -26,6 +25,7 @@ from vllm.model_executor.models.voxtral import (
|
||||
VoxtralMultiModalProcessor,
|
||||
VoxtralProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import ProcessorInputs, TimingContext
|
||||
@@ -56,7 +56,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
mm_items: MultiModalDataItems,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_kwargs = mm_res.kwargs
|
||||
mm_kwargs = mm_res.kwargs.require_data()
|
||||
|
||||
# there are no placeholder audio tokens for streaming
|
||||
# so we need to build the place placeholder positions manually
|
||||
@@ -69,7 +69,9 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
audio_config = tokenizer.instruct.audio_encoder.audio_config
|
||||
|
||||
num_audio_samples = audios[0]["audio_arrays"].data.shape[0]
|
||||
audio_array = audios[0]["audio_arrays"].data
|
||||
assert isinstance(audio_array, (torch.Tensor, np.ndarray))
|
||||
num_audio_samples = audio_array.shape[0]
|
||||
length = audio_config.num_audio_tokens(num_audio_samples)
|
||||
|
||||
features_info = PlaceholderFeaturesInfo(
|
||||
@@ -162,7 +164,7 @@ class VoxtralRealtimeBuffer:
|
||||
for token in tokens:
|
||||
await self._token_queue.put(token)
|
||||
|
||||
async def get_input_stream(self) -> AsyncGenerator[StreamingInput]:
|
||||
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
|
||||
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
|
||||
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
|
||||
|
||||
@@ -188,11 +190,9 @@ class VoxtralRealtimeBuffer:
|
||||
|
||||
self._leftover = audio_array[stride:]
|
||||
|
||||
yield StreamingInput(
|
||||
TokensPrompt(
|
||||
prompt_token_ids=next_tokens,
|
||||
multi_modal_data={"audio": (frame, None)},
|
||||
)
|
||||
yield TokensPrompt(
|
||||
prompt_token_ids=next_tokens,
|
||||
multi_modal_data={"audio": (frame, None)},
|
||||
)
|
||||
|
||||
|
||||
@@ -274,8 +274,8 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
token_task = asyncio.create_task(feed_tokens())
|
||||
|
||||
try:
|
||||
async for streaming_input in buffer.get_input_stream():
|
||||
yield streaming_input.prompt
|
||||
async for prompt in buffer.get_input_stream():
|
||||
yield prompt
|
||||
finally:
|
||||
audio_task.cancel()
|
||||
token_task.cancel()
|
||||
@@ -420,6 +420,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
|
||||
|
||||
seq_lens = [mel.shape[1] for mel in mel_features]
|
||||
# [total_num_20ms_frames, hidden_size]
|
||||
assert isinstance(self.whisper_encoder.whisper_encoder, WhisperCausalEncoder)
|
||||
audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv(
|
||||
mel_features
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user