Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
+3 -5
View File
@@ -116,7 +116,6 @@ class AXK1MoE(nn.Module):
self.ep_size = self.ep_group.size()
assert config.n_routed_experts is not None
assert config.num_experts_per_tok is not None
assert config.scoring_func is not None
assert config.hidden_act is not None
self.n_routed_experts: int = config.n_routed_experts
self.n_shared_experts: int | None = config.n_shared_experts
@@ -134,7 +133,7 @@ class AXK1MoE(nn.Module):
config.n_routed_experts,
prefix=f"{prefix}.gate",
)
if config.topk_method == "noaux_tc":
if getattr(config, "topk_method", "noaux_tc") == "noaux_tc":
self.gate.e_score_correction_bias = nn.Parameter(
torch.empty(config.n_routed_experts, dtype=torch.float32)
)
@@ -186,7 +185,7 @@ class AXK1MoE(nn.Module):
num_expert_group=config.n_group,
topk_group=config.topk_group,
prefix=f"{prefix}.experts",
scoring_func=config.scoring_func,
scoring_func=getattr(config, "scoring_func", "sigmoid"),
# we do scaling outside, set factor to 1.0 to avoid double mul
# aiter applies routed_scaling_factor internally
routed_scaling_factor=self.routed_scaling_factor,
@@ -645,11 +644,10 @@ class AXK1DecoderLayer(nn.Module):
self.routed_scaling_factor = config.routed_scaling_factor
def _is_layer_sparse(self) -> bool:
assert self.config.moe_layer_freq is not None
return (
self.config.n_routed_experts is not None
and self.layer_idx >= self.config.first_k_dense_replace
and self.layer_idx % self.config.moe_layer_freq == 0
and self.layer_idx % getattr(self.config, "moe_layer_freq", 1) == 0
)
def forward(
+4
View File
@@ -3,6 +3,7 @@
from .interfaces import (
HasInnerState,
SupportsHCSP,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -10,6 +11,7 @@ from .interfaces import (
SupportsPP,
SupportsTranscription,
has_inner_state,
supports_hc_sp,
supports_lora,
supports_mrope,
supports_multimodal,
@@ -33,6 +35,8 @@ __all__ = [
"is_text_generation_model",
"HasInnerState",
"has_inner_state",
"SupportsHCSP",
"supports_hc_sp",
"SupportsLoRA",
"supports_lora",
"SupportsMultiModal",
+733
View File
@@ -0,0 +1,733 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only AliceAI model."""
from collections.abc import Iterable
from copy import copy
from itertools import islice
from typing import cast
import torch
from torch import nn
from transformers import PreTrainedConfig, Qwen3NextConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import (
get_ep_group,
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, GateLinear
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
resolve_layer_fused_shared_expert,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
KimiGatedDeltaNetAttention,
)
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncCalculator,
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.quantization.utils.config_utils import (
get_quark_ocp_mx_group_size,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.interfaces import (
HasInnerState,
IsHybrid,
MixtureOfExperts,
SupportsLoRA,
)
from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP
from vllm.model_executor.models.qwen3_next import (
Qwen3NextAttention,
_should_replicate_misaligned_shared_expert,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
extract_layer_index,
make_layers,
maybe_fuse_shared_experts,
maybe_prefix,
)
from vllm.model_executor.offloader import (
PrefetchOffloader,
UVAOffloader,
get_offloader,
)
from vllm.models.kimi_k3.nvidia.ops import attn_res
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
_SPLIT_KDA_CONV_SHARDS = {
".linear_attn.q_conv1d": (".linear_attn.conv1d", 0),
".linear_attn.k_conv1d": (".linear_attn.conv1d", 1),
".linear_attn.v_conv1d": (".linear_attn.conv1d", 2),
}
def _load_split_kda_conv_weights(
weights: Iterable[tuple[str, torch.Tensor]],
params: dict[str, nn.Parameter],
loaded: set[str],
) -> Iterable[tuple[str, torch.Tensor]]:
"""Load split AliceAI conv weights through the upstream fused-conv ABI."""
for name, weight in weights:
for source_name, (target_name, shard_id) in _SPLIT_KDA_CONV_SHARDS.items():
if source_name not in name:
continue
mapped_name = name.replace(source_name, target_name, 1)
param = params.get(mapped_name)
if param is None:
raise ValueError(f"No target parameter for {name!r}: {mapped_name!r}")
weight_loader = getattr(param, "weight_loader", None)
if weight_loader is None:
raise ValueError(
f"Target parameter {mapped_name!r} has no weight loader"
)
weight_loader(param, weight, shard_id)
loaded.add(mapped_name)
break
else:
yield name, weight
def get_attn_res_block_size(hf_config: object) -> int:
"""Return the architecture-rebase AttnRes grouping, not legacy metadata."""
block_size = getattr(hf_config, "block_attn_res_block_size", None)
if isinstance(block_size, bool) or not isinstance(block_size, int):
raise ValueError(
"AliceAI requires a positive integer block_attn_res_block_size"
)
if block_size <= 0:
raise ValueError("block_attn_res_block_size must be positive")
return block_size
def _make_kimi_kda_config(config: PreTrainedConfig) -> KimiLinearConfig:
"""Adapt the flat AliceAI KDA geometry to the existing upstream Kimi layer."""
num_heads = config.linear_num_key_heads
num_value_heads = config.linear_num_value_heads
head_dim = config.linear_key_head_dim
value_head_dim = config.linear_value_head_dim
if num_heads != num_value_heads or head_dim != value_head_dim:
raise ValueError(
"AliceAI KDA requires matching key/value head counts and dimensions"
)
adapted = copy(config)
adapted.linear_attn_config = {
"head_dim": head_dim,
"num_heads": num_heads,
"short_conv_kernel_size": config.linear_conv_kernel_dim,
"use_full_rank_gate": False,
"gate_lower_bound": None,
}
return cast(KimiLinearConfig, adapted)
def _make_alice_ai_full_attention_config(
config: PreTrainedConfig,
) -> Qwen3NextConfig:
adapted = copy(config)
adapted.attn_output_gate = True
adapted.qkv_bias = False
adapted.dual_chunk_attention_config = None
adapted.is_causal = True
return cast(Qwen3NextConfig, adapted)
class _AttentionResidualFinal(nn.Module):
def __init__(self, hidden_size: int, rms_norm_eps: float, prefix: str) -> None:
super().__init__()
self.rms_norm_eps = rms_norm_eps
self.res_proj = ReplicatedLinear(
input_size=hidden_size,
output_size=1,
bias=False,
prefix=f"{prefix}.res_proj",
)
self.res_norm_weight = nn.Parameter(torch.ones(hidden_size))
def forward(
self,
residual_bank: torch.Tensor,
num_completed_sources: int,
partial: torch.Tensor | None,
) -> torch.Tensor:
if partial is None:
if num_completed_sources < 1:
raise RuntimeError("AttnRes requires at least one residual source")
prefix = residual_bank[:, num_completed_sources - 1]
num_blocks = num_completed_sources - 1
else:
prefix = partial
num_blocks = num_completed_sources
return attn_res(
prefix=prefix,
delta=None,
blocks=residual_bank,
norm_weight=self.res_norm_weight,
qk_weight=self.res_proj.weight.squeeze(0),
output_norm_weight=None,
num_blocks=num_blocks,
block_write_idx=-1,
eps=self.rms_norm_eps,
output_norm_eps=0.0,
)
class AliceAISparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_text_config
parallel_config = vllm_config.parallel_config
quant_config = vllm_config.quant_config
self.tp_size = get_tensor_model_parallel_world_size()
self.ep_size = get_ep_group().device_group.size()
self.n_routed_experts = config.num_experts
if self.tp_size > self.n_routed_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
f"the number of experts {self.n_routed_experts}."
)
self.is_fused_shared_expert_enabled = False
if config.shared_expert_intermediate_size > 0:
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
quant_config=quant_config,
prefix=prefix,
shared_expert_name="shared_expert",
)
self.replicate_shared_expert = False
if not self.is_fused_shared_expert_enabled:
shared_expert_group_size = get_quark_ocp_mx_group_size(
quant_config=quant_config,
layer_name=f"{prefix}.shared_expert.down_proj",
)
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
intermediate_size=config.shared_expert_intermediate_size,
tp_size=self.tp_size,
group_size=shared_expert_group_size,
enable_expert_parallel=parallel_config.enable_expert_parallel,
is_sequence_parallel=False,
)
self.enable_eplb = parallel_config.enable_eplb
self.n_logical_experts = self.n_routed_experts
self.n_redundant_experts = parallel_config.eplb_config.num_redundant_experts
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.gate = GateLinear(
input_size=config.hidden_size,
output_size=config.num_experts,
prefix=f"{prefix}.gate",
)
self.gate.e_score_correction_bias = nn.Parameter(
torch.empty(config.num_experts, dtype=torch.float32),
requires_grad=False,
)
self.shared_expert_gate = ReplicatedLinear(
input_size=config.hidden_size,
output_size=1,
bias=False,
prefix=f"{prefix}.shared_expert_gate",
)
if (
self.is_fused_shared_expert_enabled
or config.shared_expert_intermediate_size <= 0
):
self.shared_expert = None
else:
self.shared_expert = Qwen2MoeMLP(
hidden_size=config.hidden_size,
intermediate_size=config.shared_expert_intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
reduce_results=False,
expert_gate=self.shared_expert_gate,
disable_tp=self.replicate_shared_expert,
prefix=f"{prefix}.shared_expert",
)
self.experts = FusedMoEFactory(
shared_experts=(
None if self.replicate_shared_expert else self.shared_expert
),
gate=self.gate,
num_experts=self.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
renormalize=getattr(config, "norm_topk_prob", True),
scoring_func="sigmoid",
e_score_correction_bias=self.gate.e_score_correction_bias,
quant_config=quant_config,
prefix=f"{prefix}.experts",
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
n_shared_experts=1 if self.shared_expert is None else None,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_expert",
shared_expert_gate=self.shared_expert_gate
if self.shared_expert is None
else None,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
orig_shape = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
shared_output = (
self.shared_expert(hidden_states)
if self.replicate_shared_expert and self.shared_expert is not None
else None
)
output = self.experts(hidden_states=hidden_states, router_logits=hidden_states)
if shared_output is not None:
output += shared_output
return output.view(orig_shape)
class AliceAIDecoderBlock(nn.Module):
"""AliceAI topology with upstream Qwen, Kimi KDA, MoE and AttnRes operations."""
def __init__(
self,
vllm_config: VllmConfig,
layer_type: str,
prefix: str = "",
) -> None:
super().__init__()
config = vllm_config.model_config.hf_text_config
model_config = vllm_config.model_config
cache_config = vllm_config.cache_config
parallel_config = vllm_config.parallel_config
quant_config = vllm_config.quant_config
if config.num_experts <= 0:
raise ValueError("AliceAI requires sparse experts")
if parallel_config.use_sequence_parallel_moe:
raise NotImplementedError("AliceAI does not support sequence-parallel MoE")
self.layer_type = layer_type
self.layer_idx = extract_layer_index(prefix)
self.attn_res_block_size = get_attn_res_block_size(config)
self.attn_res_rms_norm_eps = float(config.rms_norm_eps)
if layer_type == "linear_attention":
self.linear_attn = KimiGatedDeltaNetAttention(
config=_make_kimi_kda_config(config),
vllm_config=vllm_config,
prefix=f"{prefix}.linear_attn",
)
self.linear_attn.o_norm.eps = float(config.rms_norm_eps)
elif layer_type == "full_attention":
self.self_attn = Qwen3NextAttention(
config=_make_alice_ai_full_attention_config(config),
model_config=model_config,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
else:
raise ValueError(f"Invalid layer_type {layer_type}")
self.mlp = AliceAISparseMoeBlock(
vllm_config=vllm_config,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = GemmaRMSNorm(
hidden_size=config.hidden_size, eps=config.rms_norm_eps
)
self.post_attention_layernorm = GemmaRMSNorm(
hidden_size=config.hidden_size, eps=config.rms_norm_eps
)
if self.layer_idx > 0:
self.attn_res_proj = ReplicatedLinear(
input_size=config.hidden_size,
output_size=1,
bias=False,
prefix=f"{prefix}.attn_res_proj",
)
self.attn_res_norm_weight = nn.Parameter(torch.ones(config.hidden_size))
self.mlp_res_proj = ReplicatedLinear(
input_size=config.hidden_size,
output_size=1,
bias=False,
prefix=f"{prefix}.mlp_res_proj",
)
self.mlp_res_norm_weight = nn.Parameter(torch.ones(config.hidden_size))
def _apply_attention_residual(
self,
residual_bank: torch.Tensor,
num_completed_sources: int,
partial: torch.Tensor | None,
*,
projection: ReplicatedLinear,
norm_weight_name: str,
) -> torch.Tensor:
if partial is None:
if num_completed_sources < 1:
raise RuntimeError("AttnRes requires at least one residual source")
prefix = residual_bank[:, num_completed_sources - 1]
num_blocks = num_completed_sources - 1
else:
prefix = partial
num_blocks = num_completed_sources
return attn_res(
prefix=prefix,
delta=None,
blocks=residual_bank,
norm_weight=getattr(self, norm_weight_name),
qk_weight=projection.weight.squeeze(0),
output_norm_weight=None,
num_blocks=num_blocks,
block_write_idx=-1,
eps=self.attn_res_rms_norm_eps,
output_norm_eps=0.0,
)
def _apply_attention(
self, hidden_states: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor:
if self.layer_type == "linear_attention":
return self.linear_attn(hidden_states=hidden_states, positions=positions)
return self.self_attn(hidden_states=hidden_states, positions=positions)
def forward_attn_res(
self,
positions: torch.Tensor,
residual_bank: torch.Tensor,
num_completed_sources: int,
partial: torch.Tensor | None,
) -> tuple[int, torch.Tensor]:
if self.layer_idx > 0 and self.layer_idx % self.attn_res_block_size == 0:
if partial is None:
raise RuntimeError("cannot finalize an empty AttnRes block")
residual_bank[:, num_completed_sources].copy_(partial)
num_completed_sources += 1
partial = None
if self.layer_idx == 0:
hidden_states = residual_bank[:, 0]
else:
hidden_states = self._apply_attention_residual(
residual_bank=residual_bank,
num_completed_sources=num_completed_sources,
partial=partial,
projection=self.attn_res_proj,
norm_weight_name="attn_res_norm_weight",
)
hidden_states = self.input_layernorm(hidden_states)
attention_output = self._apply_attention(
hidden_states=hidden_states, positions=positions
)
partial = attention_output if partial is None else partial + attention_output
hidden_states = self._apply_attention_residual(
residual_bank=residual_bank,
num_completed_sources=num_completed_sources,
partial=partial,
projection=self.mlp_res_proj,
norm_weight_name="mlp_res_norm_weight",
)
hidden_states = self.post_attention_layernorm(hidden_states)
mlp_output = self.mlp(hidden_states)
return num_completed_sources, partial + mlp_output
@support_torch_compile
class AliceAIModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
".linear_attn.a_log_bias": ".linear_attn.A_log",
},
orig_to_new_stacked={
".linear_attn.q_proj": (".linear_attn.in_proj_qkvgfab", 0),
".linear_attn.k_proj": (".linear_attn.in_proj_qkvgfab", 1),
".linear_attn.v_proj": (".linear_attn.in_proj_qkvgfab", 2),
".linear_attn.b_proj": (".linear_attn.in_proj_qkvgfab", 3),
".linear_attn.f_a_proj": (".linear_attn.in_proj_qkvgfab", 4),
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_text_config
self.config = config
self.vocab_size = config.vocab_size
self.num_redundant_experts = (
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
self.attn_res_block_size = get_attn_res_block_size(config)
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
)
def get_layer(prefix: str) -> AliceAIDecoderBlock:
return AliceAIDecoderBlock(
vllm_config=vllm_config,
layer_type=config.layer_types[extract_layer_index(prefix)],
prefix=prefix,
)
self.start_layer, self.end_layer, self.layers = make_layers(
num_hidden_layers=config.num_hidden_layers,
layer_fn=get_layer,
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
layers=self.layers, moe_cls=AliceAISparseMoeBlock, moe_name="mlp"
)
self.norm = GemmaRMSNorm(
hidden_size=config.hidden_size, eps=config.rms_norm_eps
)
self.attnres_final = _AttentionResidualFinal(
hidden_size=config.hidden_size,
rms_norm_eps=float(config.rms_norm_eps),
prefix=f"{prefix}.attnres_final",
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def _init_residual_bank(self, hidden_states: torch.Tensor) -> torch.Tensor:
residual_bank = hidden_states.new_empty(
(
hidden_states.shape[0],
(self.config.num_hidden_layers + self.attn_res_block_size - 1)
// self.attn_res_block_size,
hidden_states.shape[-1],
)
)
residual_bank[:, 0].copy_(hidden_states)
return residual_bank
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
raise NotImplementedError("AliceAI does not support pipeline parallelism")
if not get_pp_group().is_first_rank or not get_pp_group().is_last_rank:
raise NotImplementedError("AliceAI requires pipeline_parallel_size=1")
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
assert input_ids is not None
hidden_states = self.embed_input_ids(input_ids)
residual_bank = self._init_residual_bank(hidden_states)
num_completed_sources = 1
partial = None
for layer in islice(self.layers, self.start_layer, self.end_layer):
num_completed_sources, partial = layer.forward_attn_res(
positions=positions,
residual_bank=residual_bank,
num_completed_sources=num_completed_sources,
partial=partial,
)
if partial is None:
raise RuntimeError("AttnRes produced an empty final block")
hidden_states = self.attnres_final(
residual_bank=residual_bank,
num_completed_sources=num_completed_sources,
partial=partial,
)
return self.norm(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = maybe_fuse_shared_experts(
weights=weights,
enabled=self.is_fused_shared_expert_enabled,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights=weights, mapper=self.hf_to_vllm_mapper)
class AliceAIForCausalLM(
nn.Module, HasInnerState, IsHybrid, SupportsLoRA, MixtureOfExperts
):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"mtp.": None})
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"in_proj_qkvgfab": [
"q_proj",
"k_proj",
"v_proj",
"b_proj",
"f_a_proj",
],
"conv1d": ["q_conv1d", "k_conv1d", "v_conv1d"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
# AttnRes calls bypass the per-layer forward hooks used by offloading.
offloader = get_offloader()
if isinstance(offloader, PrefetchOffloader):
raise NotImplementedError("AliceAI does not support prefetch offloading")
if (
isinstance(offloader, UVAOffloader)
and offloader.cpu_offload_max_bytes > 0
and not offloader.uva_offloading
):
raise NotImplementedError("AliceAI CPU offloading requires UVA")
super().__init__()
config = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
self.scheduler_config = vllm_config.scheduler_config
self.quant_config = vllm_config.quant_config
self.config = config
self.model = AliceAIModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
self.moe_layers = [layer.mlp.experts for layer in self.model.layers]
moe = self.model.layers[self.model.start_layer].mlp
self.num_moe_layers = len(self.moe_layers)
self.num_expert_groups = 1
self.num_shared_experts = 0
self.num_logical_experts = moe.n_logical_experts
self.num_physical_experts = moe.n_physical_experts
self.num_local_physical_experts = moe.n_local_physical_experts
self.num_routed_experts = moe.n_routed_experts
self.num_redundant_experts = moe.n_redundant_experts
def update_physical_experts_metadata(
self,
num_physical_experts: int,
num_local_physical_experts: int,
) -> None:
assert self.num_local_physical_experts == num_local_physical_experts
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for layer in self.model.layers:
moe = layer.mlp
moe.n_local_physical_experts = num_local_physical_experts
moe.n_physical_experts = num_physical_experts
moe.n_redundant_experts = self.num_redundant_experts
moe.experts.update_expert_map()
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
return self.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
return self.logits_processor(lm_head=self.lm_head, hidden_states=hidden_states)
@classmethod
def get_mamba_state_dtype_from_config(
cls, vllm_config: VllmConfig
) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.kda_state_dtype(
model_dtype=vllm_config.model_config.dtype,
mamba_cache_dtype=vllm_config.cache_config.mamba_cache_dtype,
)
@classmethod
def get_mamba_state_shape_from_config(
cls, vllm_config: VllmConfig
) -> tuple[tuple[int, int], tuple[int, int, int]]:
config = vllm_config.model_config.hf_text_config
num_spec = (
vllm_config.speculative_config.num_speculative_tokens
if vllm_config.speculative_config
else 0
)
return MambaStateShapeCalculator.kda_state_shape(
tp_world_size=vllm_config.parallel_config.tensor_parallel_size,
num_heads=config.linear_num_key_heads,
head_dim=config.linear_key_head_dim,
conv_kernel_size=config.linear_conv_kernel_dim,
num_spec=num_spec,
)
@classmethod
def get_mamba_state_copy_func(
cls,
) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
manually_loaded: set[str] = set()
weights = _load_split_kda_conv_weights(
weights=weights,
params=dict(self.named_parameters()),
loaded=manually_loaded,
)
loader = AutoWeightsLoader(self)
loaded = loader.load_weights(weights=weights)
loaded.update(manually_loaded)
expected_target_weights = {
name
for name, _ in self.named_parameters()
if ".attnres_final." in name
or ".attn_res_" in name
or ".mlp_res_" in name
or name.endswith(".e_score_correction_bias")
}
if missing := expected_target_weights - loaded:
raise ValueError(
"AliceAI checkpoint is missing target-specific weights: "
+ ", ".join(sorted(missing))
)
return loaded
+40 -4
View File
@@ -20,7 +20,13 @@ from einops import rearrange
from transformers.configuration_utils import PreTrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.config import (
CacheConfig,
ModelConfig,
VllmConfig,
get_current_vllm_config,
get_current_vllm_config_or_none,
)
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
@@ -230,6 +236,12 @@ def _configure_ling_fp8_quant_config(
quant_config.ignored_layers_match_mode = "suffix"
hf_quant_config = getattr(config, "quantization_config", None)
if hf_quant_config is None:
vllm_config = get_current_vllm_config_or_none()
model_config = getattr(vllm_config, "model_config", None)
hf_quant_config = getattr(
getattr(model_config, "hf_config", None), "quantization_config", None
)
if not isinstance(hf_quant_config, dict):
return
@@ -526,6 +538,26 @@ class BailingMoeV3MLAAttention(nn.Module):
g_out = self.num_heads * self.v_head_dim
else:
g_out = 0
block_size = (
quant_config.weight_block_size
if _is_block_fp8_config(quant_config)
else None
)
self.replicate_g_proj = (
self.gated_attention_proj_granularity_type == "head_wise"
and tp_size > 1
and block_size is not None
and g_out <= block_size[0]
and not _is_fp8_module_excluded(quant_config, f"{prefix}.g_proj")
and self.num_local_heads % block_size[0] != 0
)
# A gate spanning at most one FP8 row block cannot split its scales
# across TP ranks. Replicate it and select each rank's heads instead.
self.g_proj_start = (
get_tensor_model_parallel_rank() * self.num_local_heads
if self.replicate_g_proj
else 0
)
self.g_proj = (
ColumnParallelLinear(
self.hidden_size,
@@ -533,6 +565,7 @@ class BailingMoeV3MLAAttention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.g_proj",
disable_tp=self.replicate_g_proj,
)
if g_out > 0
else None
@@ -581,9 +614,12 @@ class BailingMoeV3MLAAttention(nn.Module):
def forward(self, hidden_states: torch.Tensor, positions: torch.Tensor):
attn_out = self.mla_attn(positions, hidden_states)
if self.g_proj is not None:
gate = torch.sigmoid(self.g_proj(hidden_states)[0].float()).to(
hidden_states.dtype
)
gate = self.g_proj(hidden_states)[0]
if self.replicate_g_proj:
gate = gate[
..., self.g_proj_start : self.g_proj_start + self.num_local_heads
]
gate = torch.sigmoid(gate.float()).to(hidden_states.dtype)
if self.gated_attention_proj_granularity_type == "head_wise":
attn_out = attn_out.view(-1, self.num_local_heads, self.v_head_dim)
attn_out = attn_out * gate.unsqueeze(-1)
+15 -5
View File
@@ -41,7 +41,7 @@ from vllm.sequence import IntermediateTensors
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .interfaces import SupportsCrossEncoding, SupportsQuant
from .interfaces import SupportsCrossEncoding, SupportsLoRA, SupportsQuant
from .interfaces_base import attn_type, default_pooling_type
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
@@ -448,7 +448,7 @@ class BertPoolingModel(BertModel):
@default_pooling_type(seq_pooling_type="CLS")
class BertEmbeddingModel(nn.Module, SupportsQuant):
class BertEmbeddingModel(nn.Module, SupportsQuant, SupportsLoRA):
"""A model that uses Bert to provide embedding functionalities.
This class encapsulates the BertModel and provides an interface for
@@ -463,9 +463,22 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
is_pooling_model = True
embedding_class: type[nn.Module] = BertEmbedding
packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
# PEFT adapters name modules relative to the HF backbone (optionally under
# `bert.`/`roberta.`), while vLLM nests the backbone under `model.`.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"bert.": "model.",
"roberta.": "model.",
"encoder.": "model.encoder.",
"embeddings.": "model.embeddings.",
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
@@ -564,9 +577,6 @@ class BertMLMHead(nn.Module):
self.layer_norm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
self.decoder = nn.Linear(hidden_size, vocab_size, bias=True)
def tie_weights_with_embeddings(self, embeddings_weight: torch.Tensor):
self.decoder.weight = embeddings_weight
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
x = self.dense(hidden_states)
x = self.activation(x)
-7
View File
@@ -129,13 +129,6 @@ class BlipAttention(nn.Module):
prefix=f"{prefix}.attn",
)
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
return (
tensor.view(bsz, seq_len, self.num_heads, self.head_dim)
.transpose(1, 2)
.contiguous()
)
def forward(
self,
hidden_states: torch.Tensor,
+44
View File
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Clef-Flash: the Qwen3.5 backbone with a joint schema decision head."""
import torch
from vllm.config import VllmConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.multimodal import MULTIMODAL_REGISTRY
from .clef_joint_schema import JOINT_HEAD_WEIGHTS, JointSchemaHead, JointSchemaPooler
from .interfaces_base import default_pooling_type
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
@default_pooling_type(tok_pooling_type="ALL")
@MULTIMODAL_REGISTRY.register_processor(
Qwen3VLMultiModalProcessor,
info=Qwen3_5ProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class ClefFlashForJointSchema(Qwen3_5ForConditionalGeneration):
is_pooling_model = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
model_config = vllm_config.model_config
self.pooler = JointSchemaPooler(
JointSchemaHead(**model_config.hf_config.joint_head_config),
self._embed_lexical,
)
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=model_config.model,
revision=model_config.revision,
prefix="pooler.head.",
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
)
]
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
@@ -0,0 +1,343 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared joint schema head for Clef decision models.
The head reads the final hidden states of the whole prompt, so it runs as a
``token_classify`` pooler.
Each request carries its schema in ``PoolingParams.extra_kwargs`` under
``"joint_schema"``:
- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
- ``question_spans``: the ``[start, end)`` of each question's instructions.
- ``option_spans``: the ``[start, end)`` of each option, question by question.
- ``option_counts``: the number of options of each question.
Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
so they hold after multimodal placeholders are expanded. The output has one
value per option, in ``option_spans`` order: its probability within its
question, or its logit with ``use_activation=False``.
"""
import math
from collections.abc import Callable, Set
from dataclasses import dataclass
from typing import Any
import torch
import torch.nn.functional as F
from torch import nn
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.metadata import PoolingMetadata
JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
NUM_QUESTION_TYPES = 3
MAX_LOGIT_SCALE = math.log(100.0)
def segment_mean(
values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
) -> torch.Tensor:
sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
sums.index_add_(0, segment_ids, values.float())
return (sums / lengths[:, None]).to(values.dtype)
@dataclass
class SchemaIndices:
"""Where one prompt's questions and options are, on the model's device."""
positions: torch.Tensor
"""Every token of every span, questions first."""
span_ids: torch.Tensor
"""The span of each of ``positions``."""
span_lengths: torch.Tensor
"""``[Q + O]``."""
question_types: torch.Tensor
"""``[Q]``."""
option_question: torch.Tensor
"""The question of each option, ``[O]``."""
option_slots: torch.Tensor
"""Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
option_mask: torch.Tensor
"""``[Q, max_options]``, true at the slots of options."""
class EvidenceRoutingLayer(nn.Module):
def __init__(self, width: int, heads: int, feedforward: int) -> None:
super().__init__()
self.query_norm = nn.LayerNorm(width)
self.memory_norm = nn.LayerNorm(width)
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
self.feedforward_norm = nn.LayerNorm(width)
# The Identity takes the place of the checkpoint's dropout.
self.feedforward = nn.Sequential(
nn.Linear(width, feedforward),
nn.GELU(),
nn.Identity(),
nn.Linear(feedforward, width),
)
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
# Separate key and value tensors, like the reference, keep attention's
# unpacked input projections.
routed, _ = self.attention(
self.query_norm(queries),
self.memory_norm(memory),
self.memory_norm(memory),
need_weights=False,
)
queries = queries + routed
return queries + self.feedforward(self.feedforward_norm(queries))
class JointSchemaHead(nn.Module):
def __init__(
self,
hidden_size: int,
width: int,
routing_layers: int,
layers: int,
heads: int,
feedforward: int,
) -> None:
super().__init__()
self.hidden_norm = nn.LayerNorm(hidden_size)
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
self.question_projection = nn.Linear(hidden_size, width, bias=False)
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
self.global_projection = nn.Linear(hidden_size, width, bias=False)
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
self.evidence_layers = nn.ModuleList(
EvidenceRoutingLayer(width, heads, feedforward)
for _ in range(routing_layers)
)
self.option_summary_norm = nn.LayerNorm(width)
self.layers = nn.ModuleList(
nn.TransformerDecoderLayer(
d_model=width,
nhead=heads,
dim_feedforward=feedforward,
dropout=0.0,
activation="gelu",
batch_first=True,
norm_first=True,
)
for _ in range(layers)
)
self.field_norm = nn.LayerNorm(width)
self.option_norm = nn.LayerNorm(width)
self.residual_scorer = nn.Sequential(
nn.Linear(width * 4, width),
nn.GELU(),
nn.Identity(),
nn.Linear(width, 1),
)
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
self.residual_gate = nn.Parameter(torch.zeros(()))
def forward(
self,
hidden_states: torch.Tensor,
lexical: torch.Tensor,
schema: SchemaIndices,
) -> torch.Tensor:
"""The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
final hidden states and each option's ``[O, hidden]`` mean output
embedding."""
num_questions = schema.question_types.shape[0]
option_question = schema.option_question
hidden_states = self.hidden_norm(hidden_states)
memory = self.memory_projection(hidden_states).unsqueeze(0)
global_vector = hidden_states[-1]
means = segment_mean(
hidden_states[schema.positions], schema.span_ids, schema.span_lengths
)
question_vectors, contexts = means[:num_questions], means[num_questions:]
options = (
self.option_context_projection(contexts)
+ self.option_lexical_projection(lexical)
+ self.option_question_projection(question_vectors)[option_question]
).unsqueeze(0)
for layer in self.evidence_layers:
options = layer(options, memory)
options = options[0]
# Each field summarizes its options, weighted by their match to it.
base_fields = self.question_projection(question_vectors)
width = options.shape[-1]
grid = options.new_zeros(schema.option_mask.numel(), width)
grid[schema.option_slots] = options
grid = grid.view(*schema.option_mask.shape, width)
routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
routing = routing.masked_fill(~schema.option_mask, float("-inf"))
summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
fields = (
base_fields
+ self.option_summary_norm(summaries)
+ self.global_projection(global_vector).unsqueeze(0)
+ self.type_embedding(schema.question_types)
).unsqueeze(0)
for layer in self.layers:
fields = layer(fields, memory)
fields = self.field_norm(fields[0])[option_question]
anchors = F.normalize(question_vectors + global_vector, dim=-1)
prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
prior = prior_scale * (
F.normalize(lexical, dim=-1) * anchors[option_question]
).sum(dim=-1)
options = self.option_norm(options)
cosine = F.cosine_similarity(fields, options, dim=-1)
features = torch.cat(
[fields, options, fields * options, torch.abs(fields - options)], dim=-1
)
residual = self.residual_scorer(features).squeeze(-1)
joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
joint = joint_scale * cosine + residual
return prior + torch.sigmoid(self.residual_gate) * joint
def parse_joint_schema(
extra_kwargs: dict[str, Any] | None, prompt_len: int
) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
"""The question types, the absolute spans of the questions then of the
options, and the option counts. None if the schema is missing or invalid."""
schema = (extra_kwargs or {}).get("joint_schema")
if not isinstance(schema, dict):
return None
try:
types = [int(t) for t in schema["question_types"]]
counts = [int(c) for c in schema["option_counts"]]
question_spans = [
(int(start) + prompt_len, int(end) + prompt_len)
for start, end in schema["question_spans"]
]
option_spans = [
(int(start) + prompt_len, int(end) + prompt_len)
for start, end in schema["option_spans"]
]
except (KeyError, TypeError, ValueError):
return None
spans = question_spans + option_spans
if (
not types
or len(counts) != len(types)
or len(question_spans) != len(types)
or len(option_spans) != sum(counts)
or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
or any(c < 1 for c in counts)
or any(not 0 <= start < end <= prompt_len for start, end in spans)
):
return None
return types, spans, counts
class JointSchemaPooler(Pooler):
"""Runs the joint schema head on each finished prompt.
It never raises, because it runs in the engine core: a prompt without a
valid schema, such as a profiling dummy, gets an empty output.
"""
def __init__(
self,
head: JointSchemaHead,
embed_lexical: Callable[[torch.Tensor], torch.Tensor],
) -> None:
super().__init__()
self.pooling = AllPool()
self.head = head
self.embed_lexical = embed_lexical
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
pooled = self.pooling(hidden_states, pooling_metadata)
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
outputs: list[torch.Tensor | None] = []
for hidden, token_ids, params in zip(
pooled, prompt_token_ids, pooling_metadata.pooling_params
):
if hidden is None:
outputs.append(None)
continue
parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
if parsed is None:
outputs.append(hidden.new_empty(0, dtype=torch.float32))
continue
schema, lexical_ids, lexical_span_ids = self._indices(
*parsed, hidden.device
)
lexical = segment_mean(
self.embed_lexical(
async_tensor_h2d(token_ids[lexical_ids], hidden.device)
),
lexical_span_ids,
schema.span_lengths[len(parsed[0]) :],
)
logits = self.head(hidden, lexical, schema).float()
if params.use_activation is not False:
mask = schema.option_mask
grid = logits.new_full((mask.numel(),), float("-inf"))
grid[schema.option_slots] = logits
probs = grid.view(mask.shape).softmax(dim=-1)
logits = probs.view(-1)[schema.option_slots]
outputs.append(logits)
return outputs
@staticmethod
def _indices(
types: list[int],
spans: list[tuple[int, int]],
counts: list[int],
device: torch.device,
) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
"""The schema's indices, plus the option tokens' positions (on the CPU)
and their option spans, for the lexical embeddings."""
num_questions = len(types)
lengths = torch.tensor([end - start for start, end in spans])
span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
positions = torch.cat([torch.arange(start, end) for start, end in spans])
num_question_tokens = int(lengths[:num_questions].sum())
counts_cpu = torch.tensor(counts)
max_options = max(counts)
option_question = torch.repeat_interleave(
torch.arange(num_questions), counts_cpu
)
option_slots = torch.cat(
[torch.arange(count) + q * max_options for q, count in enumerate(counts)]
)
schema = SchemaIndices(
positions=async_tensor_h2d(positions, device),
span_ids=async_tensor_h2d(span_ids, device),
span_lengths=async_tensor_h2d(lengths, device),
question_types=async_tensor_h2d(types, device, dtype=torch.long),
option_question=async_tensor_h2d(option_question, device),
option_slots=async_tensor_h2d(option_slots, device),
option_mask=async_tensor_h2d(
torch.arange(max_options) < counts_cpu[:, None], device
),
)
lexical_span_ids = async_tensor_h2d(
span_ids[num_question_tokens:] - num_questions, device
)
return schema, positions[num_question_tokens:], lexical_span_ids
+104
View File
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The Qwen3-Omni thinker with a Clef joint schema head."""
import math
from collections.abc import Mapping
from typing import Any
import numpy as np
import torch
import torch.nn.functional as F
from transformers.feature_extraction_utils import BatchFeature
from vllm.config import VllmConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.multimodal import MULTIMODAL_REGISTRY
from .clef_joint_schema import JOINT_HEAD_WEIGHTS, JointSchemaHead, JointSchemaPooler
from .interfaces_base import default_pooling_type
from .qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerDummyInputsBuilder,
Qwen3OmniMoeThinkerForConditionalGeneration,
Qwen3OmniMoeThinkerMultiModalProcessor,
Qwen3OmniMoeThinkerProcessingInfo,
)
MAX_FRAME_PIXELS = 256 * 32 * 32
def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
"""The reference reads each video frame at at most 256 * 32 * 32 pixels,
in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on
their device (a CUDA tensor from torchcodec must not round-trip)."""
height, width = frames.shape[1:3]
if height * width <= MAX_FRAME_PIXELS:
return frames
scale = math.sqrt(MAX_FRAME_PIXELS / (width * height))
new_width = max(2, round(width * scale / 2) * 2)
new_height = max(2, round(height * scale / 2) * 2)
is_tensor = isinstance(frames, torch.Tensor)
tensor = frames if is_tensor else torch.from_numpy(frames)
resized = F.interpolate(
tensor.permute(0, 3, 1, 2).float(),
size=(new_height, new_width),
mode="bilinear",
)
if tensor.dtype == torch.uint8:
resized = resized.round()
resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous()
return resized if is_tensor else resized.numpy()
def _cap_video(item: Any) -> Any:
if isinstance(item, tuple):
frames, metadata = item
return cap_frame_pixels(frames), metadata
return cap_frame_pixels(item)
class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor):
"""Caps video frames at the reference's per-frame pixel count."""
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
if videos := hf_data.get("videos"):
assert isinstance(videos, list)
hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]}
return super()._call_hf_processor(hf_data, hf_kwargs)
@default_pooling_type(tok_pooling_type="ALL")
@MULTIMODAL_REGISTRY.register_processor(
ClefOmniMultiModalProcessor,
info=Qwen3OmniMoeThinkerProcessingInfo,
dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
)
class ClefOmniForJointSchema(Qwen3OmniMoeThinkerForConditionalGeneration):
is_pooling_model = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
model_config = vllm_config.model_config
head_config = model_config.hf_config.joint_head_config
self.pooler = JointSchemaPooler(
JointSchemaHead(**head_config), self._embed_lexical
)
# The head's weights ship as their own safetensors file.
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=model_config.model,
revision=model_config.revision,
prefix="pooler.head.",
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
)
]
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
# The head embeds options with the output embeddings. ParallelLMHead
# forbids lookups, so use the vocab-parallel embedding's.
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
+134 -1
View File
@@ -25,6 +25,51 @@ class VerifyAndUpdateConfig:
return
class AliceAIForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
hf_config = model_config.hf_config
if getattr(hf_config, "kda_allow_negative_eigenvalues", None) is not False:
raise NotImplementedError(
"AliceAI requires kda_allow_negative_eigenvalues=false"
)
if getattr(hf_config, "router_score_function", None) != "sigmoid":
raise ValueError("AliceAI requires router_score_function='sigmoid'")
if getattr(hf_config, "router_bias_correction", None) is not True:
raise ValueError("AliceAI requires router_bias_correction=true")
mtp_layers = getattr(hf_config, "mtp_num_hidden_layers", 0)
if mtp_layers:
logger.warning(
"AliceAI checkpoint declares %s MTP layer(s), but this "
"implementation ignores the MTP configuration and weights; "
"MTP speculative decoding is unavailable.",
mtp_layers,
)
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
if vllm_config.parallel_config.pipeline_parallel_size != 1:
raise NotImplementedError(
"AliceAI does not support pipeline parallelism; "
"pipeline_parallel_size=1 is required"
)
speculative_config = vllm_config.speculative_config
speculative_method = (
speculative_config.method if speculative_config is not None else None
)
if speculative_method in {
"dflash",
"dspark",
"eagle3",
"extract_hidden_states",
}:
raise NotImplementedError(
"AliceAI does not support speculative method "
f"{speculative_method!r} because it requires auxiliary hidden states"
)
class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
@@ -70,6 +115,32 @@ class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig):
)
class Glm5NextModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
from vllm.platforms import current_platform
cache_config = vllm_config.cache_config
if (
cache_config.cache_dtype != "auto"
or not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
or getattr(vllm_config.model_config.hf_text_config, "index_topk", None)
is None
):
return
cache_config.cache_dtype = "fp8"
logger.info_once("Using fp8 KV cache for GLM-5.3-Flash")
spec_config = vllm_config.speculative_config
if (
spec_config is not None
and spec_config.method != "mtp"
and spec_config.kv_cache_dtype is None
):
spec_config.kv_cache_dtype = "auto"
class Ernie4_5_VLMoeForConditionalGenerationConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
@@ -692,7 +763,7 @@ class MambaModelConfig(VerifyAndUpdateConfig):
model_config.architecture,
)
assert vllm_config.scheduler_config.enable_chunked_prefill, (
"Chunked prefill is required for mamba cache mode 'align'."
"Chunked prefill is required for Mamba prefix caching."
)
# By default, mamba block size will be set to max_model_len (see
# below). When enabling prefix caching, we align mamba block size
@@ -912,11 +983,21 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
from vllm.platforms import current_platform
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
text_config = vllm_config.model_config.hf_text_config
if text_config.hc_count <= 1:
raise ValueError("Qwen4Exp requires hc_count > 1")
parallel_config = vllm_config.parallel_config
# MoE SP also enables HC SP without the explicit HC flag.
if (
current_platform.is_cuda()
and parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size != 1
):
raise ValueError("Qwen4Exp MoE SP requires PP=1")
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
getattr(text_config, "indexer_n_heads", None) is not None
)
@@ -1006,6 +1087,54 @@ class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
config.is_causal = is_causal
class D3ForSequenceClassificationConfig(Qwen3_5ForConditionalGenerationConfig):
"""Apply the contract declared by a Decision 3.0 `decision_config.json`."""
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
hf_config = model_config.hf_config
decision = getattr(hf_config, "decision_config", None)
if not isinstance(decision, dict):
raise ValueError("Decision 3.0 checkpoints need a decision_config.json")
if decision.get("format_version") != 1:
raise ValueError(
"unsupported decision_config.json format_version "
f"{decision.get('format_version')!r}; vLLM reads version 1"
)
if decision.get("pooling", "last") != "last":
raise ValueError(
f"unsupported decision_config.json pooling {decision['pooling']!r}"
)
attention_mode = decision.get("attention_mode", "causal")
if attention_mode not in ("causal", "noncausal_full_attention"):
raise ValueError(
f"unsupported decision_config.json attention_mode {attention_mode!r}"
)
codes = decision.get("codes")
token_ids = decision.get("token_ids")
if not codes or not isinstance(token_ids, list) or len(codes) != len(token_ids):
raise ValueError(
"decision_config.json must list one token id per answer code"
)
# The full attention layers read the whole prompt in both directions;
# the Gated DeltaNet layers stay causal.
is_causal = attention_mode == "causal"
for config in {
id(config): config for config in (hf_config, model_config.hf_text_config)
}.values():
config.is_causal = is_causal
# One readout row per answer code.
hf_config.id2label = dict(enumerate(codes))
# Probabilities only exist over a question's own options, so the head
# returns raw code logits unless activation is requested.
pooler_config = model_config.pooler_config
assert pooler_config is not None
if pooler_config.use_activation is None:
pooler_config.use_activation = False
class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -1051,8 +1180,10 @@ class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"AliceAIForCausalLM": AliceAIForCausalLMConfig,
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
"ColQwen3_5": ColQwen3_5Config,
"D3ForSequenceClassification": D3ForSequenceClassificationConfig,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
@@ -1065,6 +1196,8 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Gemma4ForCausalLM": Gemma4Config,
"Gemma4ForConditionalGeneration": Gemma4Config,
"Gemma4UnifiedForConditionalGeneration": Gemma4Config,
"Glm5NextForCausalLM": Glm5NextModelConfig,
"Glm5NextForConditionalGeneration": Glm5NextModelConfig,
"GlmMoeDsaForCausalLM": GlmMoeDsaForCausalLM,
"GptOssForCausalLM": GptOssForCausalLMConfig,
"LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig,
+83
View File
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Decision 3.0 (vllm-sr/d3): a Qwen3.5 backbone with an answer-code readout.
The checkpoint is a bare `Qwen3_5Model` plus `readout.safetensors`, one row per
answer code, and `decision_config.json`. The readout scores the codes at the
last prompt token and is served as the `score` head of a classification model.
`D3ForSequenceClassificationConfig` applies the checkpoint's attention mode and
label space.
"""
from typing import TYPE_CHECKING
from transformers.models.qwen3_vl import Qwen3VLProcessor
from vllm.config import VllmConfig
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.multimodal import MULTIMODAL_REGISTRY
from .adapters import as_seq_cls_model
from .interfaces_base import default_pooling_type
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
from .utils import WeightsMapper
READOUT_WEIGHTS = "readout.safetensors"
# The media budgets of the checkpoint's runtime (`d3_runtime.py`), which differ
# from its processor config: images at 65,536 to 1,638,400 pixels, video frames
# at up to 200,704 pixels and at most 32 frames per video.
IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_638_400}
VIDEO_FRAME_MAX_PIXELS = 200_704
VIDEO_FRAME_MIN_PIXELS = 4_096
VIDEO_MAX_FRAMES = 32
class D3ProcessingInfo(Qwen3_5ProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
processor = super().get_hf_processor(**kwargs)
processor.image_processor.size = dict(IMAGE_SIZE)
video = processor.video_processor
factor = video.patch_size * video.merge_size
video.size = {
"shortest_edge": VIDEO_FRAME_MIN_PIXELS,
"longest_edge": VIDEO_FRAME_MAX_PIXELS * VIDEO_MAX_FRAMES,
}
video.cap_pixels_per_frame = True
video.max_video_tokens = VIDEO_FRAME_MAX_PIXELS // factor**2
video.do_sample_frames = False
return processor
if TYPE_CHECKING:
_D3ForSequenceClassificationBase = Qwen3_5ForConditionalGeneration
else:
_D3ForSequenceClassificationBase = as_seq_cls_model(Qwen3_5ForConditionalGeneration)
@default_pooling_type(seq_pooling_type="LAST")
@MULTIMODAL_REGISTRY.register_processor(
Qwen3VLMultiModalProcessor,
info=D3ProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class D3ForSequenceClassification(_D3ForSequenceClassificationBase):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"language_model.": "language_model.model.",
"mtp.": None,
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
model_config = vllm_config.model_config
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=model_config.model,
revision=model_config.revision,
prefix="score.",
allow_patterns_overrides=[READOUT_WEIGHTS],
)
]
@@ -269,17 +269,6 @@ class DiffusionGemmaForConditionalGeneration(
eps=getattr(text_config, "rms_norm_eps", 1e-6),
)
def compute_self_conditioning(
self,
inputs_embeds: torch.Tensor,
probs: torch.Tensor,
) -> torch.Tensor:
embed_weight = self.model.embed_tokens.weight
soft_embeds = torch.matmul(
probs.to(embed_weight.dtype), embed_weight
) * self.model.normalizer.to(inputs_embeds.dtype)
return self.self_conditioning(inputs_embeds, soft_embeds)
# ------------------------------------------------------------------ #
# Multimodal: reuse Gemma4's image parsing, processing & embedding
# ------------------------------------------------------------------ #
@@ -18,6 +18,7 @@ from transformers.models.embedding_gemma2.processing_embedding_gemma2 import (
)
from transformers.video_utils import VideoMetadata
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
@@ -334,6 +335,7 @@ class EmbeddingGemma2TextPLE(nn.Module):
return self.per_layer_projection_norm(proj)
@support_torch_compile
class EmbeddingGemma2TextModel(nn.Module):
"""Text embedding backbone for EmbeddingGemma2 with bidirectional attention."""
+6
View File
@@ -36,6 +36,7 @@ from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -272,6 +273,7 @@ class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
vision_config: Exaone4_5_VisionConfig,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -279,6 +281,7 @@ class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
vision_config=vision_config,
norm_eps=norm_eps,
quant_config=quant_config,
input_norm=input_norm,
prefix=prefix,
)
depth = vision_config.depth
@@ -335,7 +338,9 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.config = config
self.model_config = vllm_config.model_config
self.multimodal_config = multimodal_config
self.video_pruning_rate = multimodal_config.video_pruning_rate
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
@@ -345,6 +350,7 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=self.quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
use_data_parallel=self.use_data_parallel,
)
@@ -75,6 +75,7 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
self.layers = nn.ModuleList(
Exaone4DecoderLayer(
text_config,
cache_config=vllm_config.cache_config,
quant_config=quant_config,
prefix=f"{prefix}.layers.{idx}",
)
+22 -15
View File
@@ -227,24 +227,31 @@ class ExaoneMoeAttention(nn.Module):
layer_idx = extract_layer_index(prefix)
if config.sliding_windows is not None:
self.sliding_window_size = config.sliding_windows[layer_idx]
if config.layer_types[layer_idx] == "full_attention":
self.sliding_window_size = None
# Configs published before K-EXAONE 2.0 only have the scalar
# `sliding_window` and no MTP fields; their MTP layer follows the main
# layer with the same index.
layer_types = config.layer_types
sliding_windows = getattr(config, "sliding_windows", None)
if is_mtp:
self.sliding_window = (
config.mtp_layer_types[layer_idx] == "sliding_attention"
layer_types = getattr(config, "mtp_layer_types", None) or layer_types
sliding_windows = (
getattr(config, "mtp_sliding_windows", None) or sliding_windows
)
if config.mtp_sliding_windows is not None:
self.sliding_window_size = config.mtp_sliding_windows[layer_idx]
if config.mtp_layer_types[layer_idx] == "full_attention":
self.sliding_window_size = None
if layer_types[layer_idx] == "full_attention":
self.sliding_window_size = None
elif sliding_windows is not None:
self.sliding_window_size = sliding_windows[layer_idx]
else:
self.sliding_window_size = config.sliding_window
# apply rotary embeddings to every layer in full attention models
self.apply_rope_all_layers = "sliding_attention" not in config.layer_types
# Full-attention layers are NoPE when sliding layers are present, except
# the MTP layer, which is trained with RoPE.
self.use_rope = (
bool(self.sliding_window_size)
or is_mtp
or "sliding_attention" not in config.layer_types
)
set_default_rope_theta(config, default_theta=1000000)
self.rotary_emb = get_rope(
@@ -279,7 +286,7 @@ class ExaoneMoeAttention(nn.Module):
k = self.k_norm(k)
k = k.flatten(-2, -1)
if self.sliding_window_size or self.apply_rope_all_layers:
if self.use_rope:
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from functools import wraps
from typing import Any, TypeVar
import torch
from torch import nn
from vllm.model_executor.layers.fused_moe.expert_substitution import (
APPROX_VALUE_SUFFIX,
ConstantExpertSubstitution,
decoder_layer_index,
get_expert_substitution_spec,
)
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
_T = TypeVar("_T", bound=nn.Module)
def _collect_expert_substitutions(
model: nn.Module,
) -> dict[int, tuple[str, ConstantExpertSubstitution]]:
substitutions: dict[int, tuple[str, ConstantExpertSubstitution]] = {}
for name, module in model.named_modules():
if not isinstance(module, ConstantExpertSubstitution):
continue
if module.layer_idx in substitutions:
raise ValueError(
f"layer {module.layer_idx} has more than one MoE with substituted "
"experts"
)
substitutions[module.layer_idx] = (f"{name}.values", module)
return substitutions
def _parse_approx_value_name(name: str) -> tuple[int, int] | None:
"""Return (layer index, expert ID) of an ``approx_value`` tensor name."""
if not name.endswith(APPROX_VALUE_SUFFIX):
return None
experts_path, _, expert_id = name.removesuffix(APPROX_VALUE_SUFFIX).rpartition(".")
layer_idx = decoder_layer_index(experts_path)
if layer_idx is None or not expert_id.isdigit():
return None
return layer_idx, int(expert_id)
def as_expert_substitution_model(model_cls: type[_T], model_config: Any) -> type[_T]:
"""Load constant expert values at the ``load_weights`` boundary.
Initial loading and direct weight updates both go through ``load_weights``.
Explicitly named values are always consumed and loaded if their layer is
built by this model. ``approx_value`` tensors of layers built elsewhere
(other pipeline stages, MTP layers) are passed through, so the model's own
loader skips them like any other weight of a layer it does not own.
"""
spec = get_expert_substitution_spec(model_config)
if spec is None:
return model_cls
value_names = spec.value_names
class ExpertSubstitutionModel(model_cls): # type: ignore[valid-type, misc]
@wraps(model_cls.__init__) # type: ignore[misc]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._expert_substitutions = _collect_expert_substitutions(self)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
if any(s.values.is_meta for _, s in self._expert_substitutions.values()):
# Put the previous weights back so the model stays usable.
finalize_layerwise_reload(self, model_config)
raise NotImplementedError(
"expert substitution does not support layerwise weight reload"
)
loaded_substitutions: set[str] = set()
def remaining_weights():
for name, loaded_weight in weights:
targets = value_names.get(name)
if targets is None:
key = _parse_approx_value_name(name)
if key is None or key[0] not in self._expert_substitutions:
yield name, loaded_weight
continue
targets = (key,)
for layer_idx, expert_id in targets:
if layer_idx not in self._expert_substitutions:
continue
param_name, substitution = self._expert_substitutions[layer_idx]
param = substitution.values
param.weight_loader(param, loaded_weight, expert_id)
loaded_substitutions.add(param_name)
loaded = super().load_weights(remaining_weights())
if loaded is not None:
loaded.update(loaded_substitutions)
return loaded
def process_weights_after_loading(self) -> None:
for param_name, substitution in self._expert_substitutions.values():
substitution.validate_loaded_values(param_name)
finalize = getattr(super(), "process_weights_after_loading", None)
if finalize is not None:
finalize()
ExpertSubstitutionModel.__name__ = model_cls.__name__
ExpertSubstitutionModel.__qualname__ = model_cls.__qualname__
return ExpertSubstitutionModel
@@ -126,10 +126,6 @@ class CacheOnlyAttentionBackend(AttentionBackend):
def get_builder_cls() -> type["CacheOnlyAttentionMetadataBuilder"]:
return CacheOnlyAttentionMetadataBuilder
@staticmethod
def use_cascade_attention(*args, **kwargs) -> bool:
return False
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
return []
@@ -158,11 +154,6 @@ class CacheOnlyAttentionMetadataBuilder(
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> CacheOnlyAttentionMetadata:
use_cascade = common_prefix_len > 0
if use_cascade:
raise NotImplementedError(
"Cascade attention not supported by CacheOnlyAttention"
)
causal = common_attn_metadata.causal
if not causal:
raise NotImplementedError(
-2
View File
@@ -42,7 +42,6 @@ from .interfaces import (
HasInnerState,
IsHybrid,
SupportsLoRA,
SupportsMambaPrefixCaching,
SupportsPP,
)
from .utils import (
@@ -499,7 +498,6 @@ class FalconH1ForCausalLM(
SupportsLoRA,
SupportsPP,
IsHybrid,
SupportsMambaPrefixCaching,
):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
+4
View File
@@ -557,6 +557,10 @@ class Gemma3ForConditionalGeneration(
)
self.multi_modal_projector = Gemma3MultiModalProjector(config)
# https://huggingface.co/blog/gemma3, (image tokens attend to each other)
# explicit mention here as text side in gemma3 standalone does not need
# prefix_lm but multimodal does
config.text_config.is_mm_prefix_lm = True
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
+32 -23
View File
@@ -98,6 +98,28 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_tower_quant_config(
tower_config: Any,
quant_config: "QuantizationConfig | None",
) -> "QuantizationConfig | None":
# BNB/torchao/compressed-tensors handle arbitrary sizes; other methods
# (Marlin, FP8, …) require dimensions divisible by 64.
# TODO(mgoin): remove this by fixing kernel padding.
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
"compressed-tensors",
]:
return quant_config
# The audio config has no intermediate_size; treat it as always divisible.
intermediate_size = getattr(tower_config, "intermediate_size", None)
quantizable = tower_config.hidden_size % 64 == 0 and (
intermediate_size is None or intermediate_size % 64 == 0
)
return quant_config if quantizable else None
# Video constants — match transformers Gemma4VideoProcessor defaults.
_SUPPORTED_SOFT_TOKENS = (70, 140, 280, 560, 1120)
_VIDEO_MAX_SOFT_TOKENS = 70 # soft tokens per video frame (vs 280 for images)
@@ -1071,25 +1093,9 @@ class Gemma4ForConditionalGeneration(
lora_config is not None and lora_config.enable_tower_connector_lora
)
# Only quantize towers when the quant method supports their
# dimensions. BNB/torchao handle arbitrary sizes; other methods
# (Marlin, FP8, …) require dimensions divisible by 64, which
# the vision tower (intermediate_size=4304) does not satisfy.
# TODO(mgoin): remove this by fixing kernel padding.
tower_quant: QuantizationConfig | None
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
"compressed-tensors",
]:
tower_quant = quant_config
else:
vision_cfg = config.vision_config
quantizable = (
vision_cfg.hidden_size % 64 == 0
and vision_cfg.intermediate_size % 64 == 0
)
tower_quant = quant_config if quantizable else None
# Apply the dimension check independently because the vision and audio
# towers do not necessarily have the same hidden and intermediate sizes.
vision_tower_quant = _get_tower_quant_config(config.vision_config, quant_config)
# ---- Vision tower (shared by image and video) ----
with self._mark_tower_model(vllm_config, {"image", "video"}):
@@ -1097,18 +1103,21 @@ class Gemma4ForConditionalGeneration(
self.embed_vision = Gemma4MultimodalEmbedder(
config.vision_config,
config.text_config,
quant_config=tower_quant,
quant_config=vision_tower_quant,
prefix=maybe_prefix(prefix, "embed_vision"),
)
recursive_replace_linear(
self.vision_tower,
tower_quant,
vision_tower_quant,
prefix=maybe_prefix(prefix, "vision_tower"),
)
# ---- Audio tower (variants with audio_config) ----
self.embed_audio: Gemma4MultimodalEmbedder | None
if config.audio_config is not None:
audio_tower_quant = _get_tower_quant_config(
config.audio_config, quant_config
)
with self._mark_tower_model(vllm_config, "audio"):
self.audio_tower = AutoModel.from_config(config=config.audio_config)
# AutoModel.from_config does NOT call post_init(),
@@ -1119,12 +1128,12 @@ class Gemma4ForConditionalGeneration(
self.embed_audio = Gemma4MultimodalEmbedder(
config.audio_config,
config.text_config,
quant_config=tower_quant,
quant_config=audio_tower_quant,
prefix=maybe_prefix(prefix, "embed_audio"),
)
recursive_replace_linear(
self.audio_tower,
tower_quant,
audio_tower_quant,
prefix=maybe_prefix(prefix, "audio_tower"),
)
else:
+5 -44
View File
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
make_attention_sink,
maybe_remap_kv_scale_name,
remap_moe_expert_weights,
)
@@ -127,9 +128,7 @@ class OAIAttention(nn.Module):
tp_size = get_tensor_model_parallel_world_size()
self.sinks = torch.nn.Parameter(
torch.empty(config.num_attention_heads // tp_size, requires_grad=False)
)
self.sinks = make_attention_sink(config.num_attention_heads // tp_size)
self.q_size = self.num_attention_heads * self.head_dim // tp_size
self.kv_size = self.num_key_value_heads * self.head_dim // tp_size
@@ -556,8 +555,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
self,
ep_rank_end: int,
ep_rank_start: int,
heads_per_rank: int,
head_start: int,
weights: Iterable[tuple[str, torch.Tensor]],
stacked_params_mapping: list[tuple[str, str, str]],
) -> set[str]:
@@ -720,13 +717,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
)
loaded_params.add(name)
continue
elif "sinks" in name:
# Handle attention sinks (distributed across ranks)
param = params_dict[name]
narrow_weight = weight.narrow(0, head_start, heads_per_rank)
param.data.copy_(narrow_weight)
loaded_params.add(name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
@@ -752,8 +742,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
self,
ep_rank_end: int,
ep_rank_start: int,
heads_per_rank: int,
head_start: int,
weights: Iterable[tuple[str, torch.Tensor]],
stacked_params_mapping: list[tuple[str, str, str]],
) -> set[str]:
@@ -1082,14 +1070,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
loaded_params.add(fused_name)
continue
elif "sinks" in name:
# Handle attention sinks (distributed across ranks)
param = params_dict[name]
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
param.data.copy_(narrow_weight)
loaded_params.add(name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
@@ -1168,8 +1148,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
self,
ep_rank_end: int,
ep_rank_start: int,
heads_per_rank: int,
head_start: int,
weights: Iterable[tuple[str, torch.Tensor]],
stacked_params_mapping: list[tuple[str, str, str]],
) -> set[str]:
@@ -1256,13 +1234,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
param.copy_(weight)
loaded_params.add(name)
continue
elif "sinks" in name:
# Handle attention sinks (distributed across ranks)
param = params_dict[name]
narrow_weight = weight.narrow(0, head_start, heads_per_rank)
param.data.copy_(narrow_weight)
loaded_params.add(name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
@@ -1345,13 +1316,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
(".qkv_proj", ".v_proj", "v"),
]
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
# Attention heads per rank
heads_per_rank = self.config.num_attention_heads // tp_size
head_start = tp_rank * heads_per_rank
ep_size = get_ep_group().world_size
ep_rank = get_ep_group().rank_in_group
num_experts = self.config.num_local_experts
@@ -1383,8 +1347,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
return self._load_weights_mxfp4(
ep_rank_end,
ep_rank_start,
heads_per_rank,
head_start,
weights,
stacked_params_mapping,
)
@@ -1392,8 +1354,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
return self._load_weights_quark(
ep_rank_end,
ep_rank_start,
heads_per_rank,
head_start,
weights,
stacked_params_mapping,
)
@@ -1401,8 +1361,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
return self._load_weights_other(
ep_rank_end,
ep_rank_start,
heads_per_rank,
head_start,
weights,
stacked_params_mapping,
)
@@ -1482,6 +1440,9 @@ class GptOssForCausalLM(
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+22 -5
View File
@@ -60,7 +60,13 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
from .interfaces import (
EagleModelMixin,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
SupportsQuant,
)
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
@@ -307,7 +313,7 @@ class GraniteDecoderLayer(nn.Module):
@support_torch_compile
class GraniteModel(nn.Module):
class GraniteModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
@@ -363,7 +369,7 @@ class GraniteModel(nn.Module):
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -375,8 +381,14 @@ class GraniteModel(nn.Module):
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states = layer(positions, hidden_states)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, None
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
@@ -386,6 +398,9 @@ class GraniteModel(nn.Module):
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -393,7 +408,9 @@ class GraniteModel(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
class GraniteForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsQuant, SupportsEagle3
):
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
# LoRA specific attributes
packed_modules_mapping: dict[str, list[str]] = {
+14 -5
View File
@@ -61,7 +61,7 @@ from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .granite import granite_layer_attn_params
from .interfaces import SupportsLoRA, SupportsPP
from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -301,7 +301,7 @@ class GraniteMoeDecoderLayer(nn.Module):
@support_torch_compile
class GraniteMoeModel(nn.Module):
class GraniteMoeModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper: WeightsMapper = WeightsMapper(
orig_to_new_suffix={
# Legacy names to new names
@@ -353,7 +353,7 @@ class GraniteMoeModel(nn.Module):
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -363,8 +363,14 @@ class GraniteMoeModel(nn.Module):
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states = layer(positions, hidden_states)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, None
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{
@@ -372,6 +378,9 @@ class GraniteMoeModel(nn.Module):
}
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -379,7 +388,7 @@ class GraniteMoeModel(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
fall_back_to_pt_during_load = False
packed_modules_mapping = {
@@ -42,7 +42,6 @@ from .interfaces import (
HasInnerState,
IsHybrid,
SupportsLoRA,
SupportsMambaPrefixCaching,
SupportsPP,
SupportsQuant,
)
@@ -587,7 +586,6 @@ class GraniteMoeHybridForCausalLM(
SupportsPP,
IsHybrid,
SupportsQuant,
SupportsMambaPrefixCaching,
):
packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": [
+14 -5
View File
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.sequence import IntermediateTensors
from .granitemoe import GraniteMoeAttention, GraniteMoeModel, GraniteMoeMoE
from .interfaces import SupportsLoRA, SupportsPP
from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP
from .utils import AutoWeightsLoader, make_layers, maybe_prefix
@@ -151,7 +151,7 @@ class GraniteMoeSharedDecoderLayer(nn.Module):
@support_torch_compile
class GraniteMoeSharedModel(nn.Module):
class GraniteMoeSharedModel(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -190,7 +190,7 @@ class GraniteMoeSharedModel(nn.Module):
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -200,8 +200,14 @@ class GraniteMoeSharedModel(nn.Module):
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states = layer(positions, hidden_states)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, None
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{
@@ -209,13 +215,16 @@ class GraniteMoeSharedModel(nn.Module):
}
)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
hf_to_vllm_mapper = GraniteMoeModel.hf_to_vllm_mapper
load_weights = GraniteMoeModel.load_weights
class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
fall_back_to_pt_during_load = False
packed_modules_mapping = {
+32 -25
View File
@@ -142,6 +142,13 @@ _language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
)
def _is_language_model(model: object) -> TypeIs[VllmModel]:
# Text towers can use config/position_ids signatures and omit loader hooks.
return callable(getattr(model, "embed_input_ids", None)) and callable(
getattr(model, "forward", None)
)
@runtime_checkable
class SupportsMultiModalEmbeddings(Protocol):
"""The interface for models that can merge external multimodal embeddings."""
@@ -285,14 +292,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
mod = getattr(mod, attr)
candidate: object = mod
if candidate is not self_module and isinstance(candidate, VllmModel):
if candidate is not self_module and _is_language_model(candidate):
_language_model_by_module[self_module] = candidate
return candidate
# Fallback
for mod in self_module.children():
candidate = mod
if isinstance(candidate, VllmModel):
if _is_language_model(candidate):
_language_model_by_module[self_module] = candidate
return candidate
@@ -859,6 +866,27 @@ def _supports_pp_inspect(model: type[object] | object) -> bool:
return supports_kw(model_forward, "intermediate_tensors")
@runtime_checkable
class SupportsHCSP(Protocol):
"""Support sequence-parallel HC without changing MoE parallelism."""
supports_hc_sp: ClassVar[Literal[True]] = True
@overload
def supports_hc_sp(model: type[object]) -> TypeIs[type[SupportsHCSP]]: ...
@overload
def supports_hc_sp(model: object) -> TypeIs[SupportsHCSP]: ...
def supports_hc_sp(
model: type[object] | object,
) -> TypeIs[type[SupportsHCSP]] | TypeIs[SupportsHCSP]:
return getattr(model, "supports_hc_sp", False)
@runtime_checkable
class HasInnerState(Protocol):
"""The interface required for all models that has inner state."""
@@ -1114,13 +1142,10 @@ def has_noops(
@runtime_checkable
class SupportsMambaPrefixCaching(Protocol):
"""The interface for models whose mamba layers support prefix caching.
This is currently experimental.
"""The interface for attention-free Mamba models that support prefix
caching. Hybrid models get the same methods from `IsHybrid`.
"""
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
@classmethod
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
"""Return copy functions for the model's Mamba states."""
@@ -1136,24 +1161,6 @@ class SupportsMambaPrefixCaching(Protocol):
return {mamba_type: copy_funcs for mamba_type in mamba_types}
@overload
def supports_mamba_prefix_caching(
model: type[object],
) -> TypeIs[type[SupportsMambaPrefixCaching]]: ...
@overload
def supports_mamba_prefix_caching(
model: object,
) -> TypeIs[SupportsMambaPrefixCaching]: ...
def supports_mamba_prefix_caching(
model: type[object] | object,
) -> TypeIs[type[SupportsMambaPrefixCaching]] | TypeIs[SupportsMambaPrefixCaching]:
return getattr(model, "supports_mamba_prefix_caching", False)
@runtime_checkable
class SupportsReplaySSM(Protocol):
"""The interface for models whose recurrent layers support ReplaySSM
+19 -16
View File
@@ -32,19 +32,13 @@ else:
logger = init_logger(__name__)
# The type of hidden states
# Currently, T = torch.Tensor for all models except for Medusa
# which has T = list[torch.Tensor]
T = TypeVar("T", default=torch.Tensor)
T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
# NOTE: Unlike those in `interfaces.py`, we don't define `ClassVar` tags
# for the base interfaces to avoid breaking OOT registration for existing models
# that don't inherit from the base interface classes
@runtime_checkable
class VllmModel(Protocol[T_co]):
class VllmModel(Protocol):
"""The interface required for all models in vLLM."""
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None: ...
@@ -53,7 +47,18 @@ class VllmModel(Protocol[T_co]):
"""Apply token embeddings to `input_ids`."""
...
def forward(self, input_ids: torch.Tensor, positions: torch.Tensor) -> T_co: ...
def forward(
self, input_ids: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor: ...
def is_unused_checkpoint_weight(self, name: str) -> bool:
"""Return True for checkpoint weights this model does not load.
Defining this method is optional for satisfying this interface.
Override with the same rule used in ``load_weights``. Models without
a selection rule keep every checkpoint weight by default.
"""
return False
def _check_vllm_model_init(model: type[object] | object) -> bool:
@@ -111,13 +116,13 @@ def is_vllm_model(
@runtime_checkable
class VllmModelForTextGeneration(VllmModel[T], Protocol[T]):
class VllmModelForTextGeneration(VllmModel, Protocol):
"""The interface required for all generative models in vLLM."""
def compute_logits(
self,
hidden_states: T,
) -> T | None:
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
"""Return `None` if TP rank > 0."""
...
@@ -138,14 +143,12 @@ def is_text_generation_model(
if not is_vllm_model(model):
return False
if isinstance(model, type):
return isinstance(model, VllmModelForTextGeneration)
return isinstance(model, VllmModelForTextGeneration)
# Optional protocol defaults are not inherited by structural models.
return callable(getattr(model, "compute_logits", None))
@runtime_checkable
class VllmModelForPooling(VllmModel[T_co], Protocol[T_co]):
class VllmModelForPooling(VllmModel, Protocol):
"""The interface required for all pooling models in vLLM."""
is_pooling_model: ClassVar[Literal[True]] = True
-2
View File
@@ -44,7 +44,6 @@ from .interfaces import (
HasInnerState,
IsHybrid,
SupportsLoRA,
SupportsMambaPrefixCaching,
SupportsPP,
)
from .utils import (
@@ -380,7 +379,6 @@ class JambaForCausalLM(
SupportsLoRA,
SupportsPP,
IsHybrid,
SupportsMambaPrefixCaching,
):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={".self_attn.": ".", ".A_log": ".A"},
-19
View File
@@ -255,25 +255,6 @@ class KeyeVisionEmbeddings(nn.Module):
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
return patch_pos_embed
def fetch_position_embedding_lfu_cache(self, embeddings, h, w, max_cache: int = 20):
grid = (h, w)
if grid in self.cache_position_embedding:
self.cache_position_count[grid] += 1
return self.cache_position_embedding[grid]
if len(self.cache_position_embedding) >= max_cache:
min_hit_grid = min(
self.cache_position_count,
key=self.cache_position_count.__getitem__,
)
self.cache_position_count.pop(min_hit_grid)
self.cache_position_embedding.pop(min_hit_grid)
position_embedding = self.interpolate_pos_encoding(embeddings, h, w, True)
self.cache_position_count[grid] = 1
self.cache_position_embedding[grid] = position_embedding
return position_embedding
def forward(
self,
pixel_values: torch.FloatTensor,
+3 -23
View File
@@ -14,7 +14,6 @@ from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_ep_group,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
@@ -36,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
SupportsEagle3,
@@ -354,9 +354,7 @@ class LagunaAttention(nn.Module):
# Attention sinks (learnable per-head bias for SWA layers)
sinks = None
if attention_sink:
self.sink = torch.nn.Parameter(
torch.empty(self.total_num_heads // tp_size, requires_grad=False)
)
self.sink = make_attention_sink(self.total_num_heads // tp_size)
sinks = self.sink
# Resolve rope params per-layer-type. ``config.rope_parameters`` is
@@ -677,22 +675,6 @@ class LagunaModel(nn.Module, EagleModelMixin):
return hidden_states, aux_hidden_states
return hidden_states
def _slice_sink(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
for name, loaded_weight in weights:
# Handle attention sinks (distributed across ranks). Derive the
# per-rank slice from the parameter's own shape so per-layer
# variations in head count are handled correctly.
if "sink" in name:
heads_per_rank = loaded_weight.shape[0] // tp_size
loaded_weight = loaded_weight.narrow(
0, tp_rank * heads_per_rank, heads_per_rank
)
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
@@ -709,9 +691,7 @@ class LagunaModel(nn.Module, EagleModelMixin):
"_input_scale",
],
)
return loader.load_weights(
self._slice_sink(weights), mapper=self.hf_to_vllm_mapper
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
+28 -15
View File
@@ -4,15 +4,14 @@ from collections.abc import Iterable, Mapping, Sequence
import torch
import torch.nn as nn
from transformers import (
BatchFeature,
PixtralVisionConfig,
)
from transformers import BatchFeature
from vllm.config import VllmConfig
from vllm.model_executor.models.mistral3 import (
Mistral3DummyInputsBuilder,
Mistral3ForConditionalGeneration,
Mistral3ImageEmbeddingInputs,
Mistral3ImageInputs,
Mistral3MultiModalProjector,
Mistral3ProcessingInfo,
init_vision_tower_for_mistral3,
@@ -26,12 +25,15 @@ from vllm.model_executor.models.utils import (
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
from vllm.multimodal.parse import (
ImageEmbeddingItems,
ImageProcessorItems,
MultiModalDataItems,
)
from vllm.multimodal.processing import (
BaseMultiModalProcessor,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
@@ -102,18 +104,20 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
hf_config = self.info.get_hf_config()
image_token_id = hf_config.image_token_index
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
encoder_info = PixtralHFEncoderInfo(hf_config)
def replace(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
size = images.get_image_size(item_idx)
ncols, nrows = encoder_info.get_patch_grid_size(
image_width=size.width, image_height=size.height
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
)
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
assert isinstance(images, ImageProcessorItems)
size = images.get_image_size(item_idx)
num_image_tokens = self.info.get_num_image_tokens(
image_width=size.width, image_height=size.height
)
# break/end tokens are not used in LightOnOCR
tokens = [image_token_id] * (ncols * nrows)
return PromptUpdateDetails.select_token_id(tokens, image_token_id)
return [image_token_id] * num_image_tokens
return [
PromptReplacement(
@@ -178,6 +182,15 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
self.language_model.make_empty_intermediate_tensors
)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Mistral3ImageInputs | None:
image_embeds = kwargs.get("image_embeds")
if image_embeds is not None:
return Mistral3ImageEmbeddingInputs(data=image_embeds)
return super()._parse_and_validate_image_input(**kwargs)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-202
View File
@@ -1,202 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .utils import maybe_prefix
class ResidualBlock(nn.Module):
def __init__(self, config: VllmConfig, hidden_size: int, num_layers: int) -> None:
super().__init__()
self.layers = nn.ModuleList(
[
nn.Linear(
hidden_size,
hidden_size,
bias=getattr(config, "medusa_fc_bias", False),
)
for _ in range(num_layers)
]
)
self.act = nn.SiLU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
for layer in self.layers:
x = x + self.act(layer(x))
return x
class Medusa(nn.Module):
"""This class implements the Medusa draft model from the paper: https://arxiv.org/abs/2401.10774
Reference implementation: https://github.com/FasterDecoding/Medusa
Differences from reference implementation:
1. Currently this only supports generating proposals from top-1 tokens.
2. We have an optional token_map which reduces draft vocab to most
frequently used tokens to give some additional speed-up by reducing
sampling overhead. This is disabled unless the checkpoint file has
explicit token_map tensor and config has an optional attribute
truncated_vocab_size < vocab_size. To use this technique, one has to find
the top-k most frequent tokens in target dataset and add that as a tensor
in the draft checkpoint (using key token_map). Also, the draft config
needs to have truncated_vocab_size (=k) as an attribute."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
super().__init__()
self.config = config
self.blocks = nn.ModuleList(
[
ResidualBlock(
config=config,
hidden_size=self.config.hidden_size,
num_layers=self.config.num_hidden_layers,
)
for _ in range(self.config.num_heads)
]
)
self.orig_vocab_size = config.vocab_size
self.truncated_vocab_size = config.truncated_vocab_size
if getattr(config, "original_lm_head", False):
self.lm_head = ParallelLMHead(
self.truncated_vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.lm_heads = [self.lm_head for _ in range(self.config.num_heads)]
else:
self.lm_heads = nn.ModuleList(
[
ParallelLMHead(
config.vocab_size,
config.hidden_size,
prefix=maybe_prefix(prefix, f"lm_heads.{i}"),
)
for i in range(self.config.num_heads)
]
)
logit_scale = getattr(config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
config.vocab_size, self.truncated_vocab_size, logit_scale
)
# Token map is a idx to token mapping to reduce the vocab size for
# the draft model. Using smaller vocab size for draft, containing
# only most frequent tokens reduces the speculation overhead. This
# doesn't affect the acceptance rate much and thus gives more speed
# -up. By default, this is disabled and is only used if the EAGLE
# checkpoint file has token_map tensor.
self.token_map = None
def forward(self, hidden_states: torch.Tensor) -> list[torch.Tensor]:
return [block(hidden_states) for block in self.blocks]
def compute_logits(
self,
hidden_states: list[torch.Tensor],
) -> list[torch.Tensor]:
logits_lst: list[torch.Tensor] = []
for hs, lm_head in zip(hidden_states, self.lm_heads):
_logits = self.logits_processor(lm_head, hs)
if _logits is None:
# _logits should only be None on rank > 0, in which case
# it should remain true for every lm_head
assert len(logits_lst) == 0
continue
if self.token_map is None:
logits_lst.append(_logits)
else:
logits_lst.append(
-torch.inf
* torch.ones(
size=(*_logits.shape[:-1], self.orig_vocab_size),
device=_logits.device,
dtype=_logits.dtype,
)
)
logits_lst[-1][..., self.token_map] = _logits
return logits_lst
@staticmethod
def _remap_old_checkpoint_key(name: str) -> str:
"""Map old FasterDecoding checkpoint keys to vLLM parameter names.
Old format uses bare numeric prefixes:
'{head}.{layer}.linear.weight' -> 'blocks.{head}.layers.{layer}.weight'
'{head}.{layer}.linear.bias' -> 'blocks.{head}.layers.{layer}.bias'
'{head}.{N}.weight' -> 'lm_heads.{head}.weight'
"""
parts = name.split(".")
if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
head, layer = parts[0], parts[1]
rest = parts[2:]
if "linear" in rest:
param = ".".join(rest[rest.index("linear") + 1 :])
return f"blocks.{head}.layers.{layer}.{param}"
else:
return f"lm_heads.{head}.{'.'.join(rest)}"
return name
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
weights_map = {}
for name, loaded_weight in weights:
name = name.replace("medusa_heads.", "")
name = self._remap_old_checkpoint_key(name)
if name == "token_map":
if self.truncated_vocab_size < self.orig_vocab_size:
self.token_map = nn.Parameter(loaded_weight, requires_grad=False)
elif name in params_dict:
weights_map[name] = loaded_weight
elif (
getattr(self.config, "original_lm_head", False)
and name == "lm_heads.0.weight"
):
weights_map["lm_head.weight"] = loaded_weight
for name, loaded_weight in weights_map.items():
if (
"lm_head" in name
and self.token_map is not None
and loaded_weight.shape[0] > self.token_map.shape[0]
):
loaded_weight = loaded_weight[self.token_map]
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
if self.token_map is not None:
self.token_map.to(device=self.lm_heads[0].weight.device)
assert (self.truncated_vocab_size == self.orig_vocab_size) or (
self.token_map is not None
)
return loaded_params
+5 -15
View File
@@ -47,6 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
make_attention_sink,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
@@ -290,9 +291,7 @@ class MiMoV2Attention(nn.Module):
)
self.attention_sink_bias = (
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
if add_swa_attention_sink_bias
else None
make_attention_sink(self.num_heads) if add_swa_attention_sink_bias else None
)
sliding_window = sliding_window_size if sliding_window_size > -1 else None
@@ -848,18 +847,9 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
param = params_dict[name]
if "attention_sink_bias" in name:
total_heads = loaded_weight.shape[0]
heads_per_rank = total_heads // tp_size
head_start = tp_rank * heads_per_rank
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
param.data.copy_(narrow_weight)
loaded_params.add(name)
else:
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
@@ -377,14 +377,6 @@ class MiMoV2MTP(nn.Module):
continue
param = params_dict[name]
# attention_sink_bias is head-parallel; slice by tp
if "attention_sink_bias" in name:
total_heads = loaded_weight.shape[0]
heads_per_rank = total_heads // tp_size
loaded_weight = loaded_weight.narrow(
0, tp_rank * heads_per_rank, heads_per_rank
)
direct_weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
+270 -61
View File
@@ -45,6 +45,7 @@ from .interfaces import (
HasInnerState,
IsHybrid,
MultiModalEmbeddings,
SupportsEagle3,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
if "video_use_vit_merger" in hf_inputs:
fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
"video", keep_on_cpu=True
)
return fields
def _expand_flags(value: object, num_items: int) -> list[bool | None]:
"""Broadcast a batched per-item flag to exactly one entry per item."""
if value is None:
return [None] * num_items
raw: list[object]
if isinstance(value, torch.Tensor):
raw = value.reshape(-1).tolist()
elif isinstance(value, list | tuple):
raw = []
for item in value:
if isinstance(item, torch.Tensor):
raw.extend(item.reshape(-1).tolist())
else:
raw.append(item)
else:
raw = [value]
if len(raw) == num_items:
return [bool(v) for v in raw]
if len(raw) == 1:
return [bool(raw[0])] * num_items
# Unexpected shape: keep one decision for the whole batch instead of
# failing the request.
return [any(bool(v) for v in raw)] * num_items
def _vision_downsample_mode(flag: bool | None) -> str | None:
if flag is None:
return None
return "16x" if flag else "4x"
def _select_vision_items(
image_input: MiniCPMVImagePixelInputs,
item_indices: list[int],
num_slices: list[int],
) -> MiniCPMVImagePixelInputs:
"""Slice a vision input down to the given items, keeping their order."""
offsets = [0]
for n in num_slices:
offsets.append(offsets[-1] + n)
pixel_values: list[torch.Tensor] = []
tgt_sizes: list[torch.Tensor] = []
sizes: list[int] = []
for i in item_indices:
start, end = offsets[i], offsets[i + 1]
pixel_values.extend(image_input["pixel_values"][start:end])
tgt_sizes.append(image_input["tgt_sizes"][start:end])
sizes.append(end - start)
return MiniCPMVImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
tgt_sizes=torch.cat(tgt_sizes, dim=0),
num_slices=torch.tensor(sizes),
)
# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
# to a single modality and must not be forwarded to the processors, which take
# flat arguments only.
_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
def _flat_processor_kwargs(
mm_kwargs: Mapping[str, object],
downsample_mode: str,
) -> dict[str, object]:
"""Drop the modality-scoped nesting and pin the resolved downsample mode.
The placeholder count and the vision tower must read the same mode, so the
value resolved for this modality is passed down explicitly rather than
leaving the processor to fall back to its own default.
"""
kwargs = {
key: value
for key, value in mm_kwargs.items()
if key not in _MODALITY_SCOPED_MM_KWARGS
}
kwargs["downsample_mode"] = downsample_mode
return kwargs
_MODALITY_TO_MM_KWARGS_SCOPE = {
"image": "images_kwargs",
"video": "videos_kwargs",
}
def _scoped_mm_kwarg(
merged: Mapping[str, object],
modality: str,
key: str,
) -> Any:
"""Read a processor kwarg for one modality.
``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
routes a flat key into every scope whose processor declares it, and a value
scoped to the other modality stays there, so it is not visible here.
"""
scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
if isinstance(scoped, Mapping):
return scoped.get(key)
return None
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
def _resolve_downsample_mode(
self,
mm_kwargs: Mapping[str, object],
modality: str = "image",
) -> str:
ds = mm_kwargs.get("downsample_mode")
if ds is not None:
return str(ds)
# A flat `downsample_mode` applies to both modalities; the nested
# `images_kwargs` / `videos_kwargs` forms scope it to one, matching
# MiniCPMV4_6Processor.
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
if downsample_mode is not None:
return str(downsample_mode)
return info._get_downsample_mode()
def _resolve_max_slice_nums(
self,
mm_kwargs: Mapping[str, object],
modality: str = "image",
) -> int | None:
# Per-request max_slice_nums. None keeps the processor default.
# Resolved like `downsample_mode` so the nested per-modality form
# (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
# what `process_images` passes to the image processor: the placeholder
# count and the visual token count are derived from it separately, so
# they diverge (and the engine aborts) if the two sides read different
# values.
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
if max_slice is None:
return None
return int(max_slice)
def get_image_prompt_texts(
self,
image_size,
image_idx: int = 0,
downsample_mode: str | None = None,
max_slice_nums: int | None = None,
) -> str:
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
return info.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
max_slice_nums=max_slice_nums,
downsample_mode=downsample_mode,
)
def _video_local_id_prefix(self, video_idx: int) -> str:
"""`<image_id>{idx}</image_id>` prefix emitted in front of a video placeholder.
MiniCPM-V 4.6 numbers each video with a local id, matching
MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
trained without these ids, so it overrides this to return an empty string.
"""
tokenizer = self.info.get_tokenizer()
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
return f"{id_start}{video_idx}{id_end}"
def get_video_prompt_texts(
self,
image_size,
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_end = getattr(tokenizer, "image_end_token", "</image>")
slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
per_frame = image_start + video_token * source_tokens + image_end
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
per_frame += "\n".join(rows)
body = per_frame * num_frames
return f"{id_start}{video_idx}{id_end}" + body
return self._video_local_id_prefix(video_idx) + body
def process_images(
self,
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
n_images = len(parsed_images)
image_processor = self.info.get_image_processor()
patch_size = image_processor.patch_size
ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
# `_flat_processor_kwargs` drops the nested form, so the value resolved
# from it has to be pinned back for the processor and the prompt text
# to agree on the slice count.
image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
if image_max_slice is not None:
image_proc_kwargs["max_slice_nums"] = image_max_slice
per_image_pixel_values: list[torch.Tensor] = []
per_image_tgt_sizes: list[torch.Tensor] = []
for image in parsed_images:
ip_out = image_processor([image], **mm_kwargs)
ip_out = image_processor([image], **image_proc_kwargs)
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
ts = ip_out["target_sizes"] # (n_slices, 2)
if pv.ndim == 4 and pv.shape[0] == 1:
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
"tgt_sizes": per_image_tgt_sizes,
}
ds_mode = self._resolve_downsample_mode(mm_kwargs)
insert_layer_id = getattr(
self.info.get_hf_config(),
"insert_layer_id",
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_processor = self.info.get_image_processor()
patch_size = image_processor.patch_size
video_max_slice = self.info.get_video_max_slice_num()
video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
video_mm_kwargs = {
**_flat_processor_kwargs(mm_kwargs, video_ds_mode),
"max_slice_nums": video_max_slice,
}
per_video_pixel_values: list[torch.Tensor] = []
per_video_tgt_sizes: list[torch.Tensor] = []
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
if not per_video_pixel_values:
return {}
insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
return {
"video_pixel_values": per_video_pixel_values,
"video_tgt_sizes": per_video_tgt_sizes,
"video_image_sizes": per_video_image_sizes,
"video_use_vit_merger": [
torch.tensor([merger_flag], dtype=torch.bool)
for _ in per_video_pixel_values
],
}
def _get_prompt_updates(
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs,
):
ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
image_ds_mode = self._resolve_downsample_mode(
hf_processor_mm_kwargs, modality="image"
)
video_ds_mode = self._resolve_downsample_mode(
hf_processor_mm_kwargs, modality="video"
)
image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
placeholders = [
("image", self.info.image_pattern),
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_image_prompt_texts(
image_size,
item_idx,
downsample_mode=ds_mode,
downsample_mode=image_ds_mode,
max_slice_nums=image_max_slice,
),
add_special_tokens=False,
),
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
downsample_mode=video_ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
downsample_mode=video_ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
new_update = super()._recompute_cached_prompt_update(
cached_update, new_item_idx
)
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
# (the base class only rewrites the image modality).
# Video placeholders carry an `<image_id>{idx}</image_id>` prefix that
# the base class does not rewrite. 4.7 has no prefix, so this is a
# no-op there.
if cached_update.modality == "video":
tokenizer = self.info.get_tokenizer()
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
old_prefix = self._video_local_id_prefix(cached_update.item_idx)
new_prefix = self._video_local_id_prefix(new_item_idx)
if old_prefix != new_prefix:
tokenizer = self.info.get_tokenizer()
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
text = tokenizer.decode(cached_update.content.full)
prev_item_idx = cached_update.item_idx
text = tokenizer.decode(cached_update.content.full)
new_update = new_update.with_content(
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
new_update = new_update.with_content(
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(old_prefix, new_prefix, 1),
add_special_tokens=False,
),
add_special_tokens=False,
),
cached_encode(tokenizer, video_token, add_special_tokens=False),
cached_encode(tokenizer, video_token, add_special_tokens=False),
)
)
)
return new_update
def _get_mm_fields_config(
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
return source_tokens + grids[0] * grids[1] * patch_tokens
def _stack_vit_merger_qkv(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
"""Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
HF checkpoints store the window-attention projections separately, while
vLLM keeps them fused. The shard_id mechanism on the submodule's
WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
incoming key after calling a submodule's load_weights, so a "k_proj" key
with no matching child raises before the mapping is honoured.
"""
pairs = (
("vit_merger.self_attn.q_proj.", "q"),
("vit_merger.self_attn.k_proj.", "k"),
("vit_merger.self_attn.v_proj.", "v"),
)
for name, tensor in weights:
for src, shard in pairs:
if src in name:
name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
tensor.shard_id = shard
break
yield name, tensor
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
def __init__(
self,
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
SupportsPP,
HasInnerState,
IsHybrid,
SupportsEagle3,
SupportsMRoPE,
):
supports_encoder_tp_data = True
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
apply_encoder_attention_mask=True,
prefix=maybe_prefix(prefix, "vpm"),
)
if config.drop_vision_last_layer:
if self._drop_vision_last_layer():
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
self.language_model.make_empty_intermediate_tensors
)
def _drop_vision_last_layer(self) -> bool:
"""Whether the last vision encoder layer is dropped.
MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
carries the field, so it overrides this.
"""
return self.config.drop_vision_last_layer
# ----- Multimodal parsing -----
def _parse_and_validate_vision_input(
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
if image_input["type"] == "image_embeds":
return image_input["image_embeds"]
downsample_mode = None
if use_vit_merger is not None:
downsample_mode = "16x" if use_vit_merger else "4x"
image_features = self.get_vision_hidden_states(
image_input,
downsample_mode=downsample_mode,
)
num_slices = image_input["num_slices"]
results = []
idx = 0
for n in num_slices.tolist():
group = image_features[idx : idx + n]
results.append(torch.cat(group, dim=0))
idx += n
return results
num_slices = image_input["num_slices"].tolist()
flags = _expand_flags(use_vit_merger, len(num_slices))
# The merger flag is per request, but the model runner may batch items
# from several concurrent requests. Encoding such a mixed batch would
# apply one request's downsample mode to another request's items, so
# group by flag and encode each group separately.
encoded: dict[int, torch.Tensor] = {}
for flag in dict.fromkeys(flags):
item_indices = [i for i, f in enumerate(flags) if f == flag]
group_input = _select_vision_items(image_input, item_indices, num_slices)
group_features = self.get_vision_hidden_states(
group_input,
downsample_mode=_vision_downsample_mode(flag),
)
idx = 0
for i in item_indices:
n = num_slices[i]
encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
idx += n
return [encoded[i] for i in range(len(num_slices))]
# ----- Multimodal embedding interface -----
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
use_vit_merger = None
if use_vit_merger_tensors is not None:
if isinstance(use_vit_merger_tensors, torch.Tensor):
use_vit_merger = bool(use_vit_merger_tensors.any().item())
elif isinstance(use_vit_merger_tensors, list | tuple):
use_vit_merger = any(
bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
for t in use_vit_merger_tensors
)
use_vit_merger = kwargs.pop("use_vit_merger", None)
video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
image_kwargs = {
k: v
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
multimodal_embeddings += tuple(
self._process_vision_input(
video_input,
use_vit_merger=use_vit_merger,
use_vit_merger=video_use_vit_merger,
)
)
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
return loader.load_weights(
_stack_vit_merger_qkv(weights),
mapper=self.hf_to_vllm_mapper,
)
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
+764
View File
@@ -0,0 +1,764 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only MiniCPM-V 4.7 model.
Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.
"""
import math
from collections.abc import Mapping
from typing import Any, NamedTuple
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
from .minicpmv import MiniCPMVDummyInputsBuilder
from .minicpmv4_6 import (
MiniCPMV4_6ForConditionalGeneration,
MiniCPMV4_6MultiModalProcessor,
MiniCPMV4_6ProcessingInfo,
_scoped_mm_kwarg,
)
logger = init_logger(__name__)
class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo):
def _mm_max_slice_nums(self, **kwargs: object) -> int | None:
# No schema resolution here: it would call back into `get_hf_processor`
# below, which resolves this same value.
merged = self.ctx.get_merged_mm_kwargs(kwargs)
max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums")
if max_slice is None:
max_slice = merged.get("max_slice_nums")
if max_slice is None:
return None
return int(max_slice)
def _configured_max_slice_nums(self) -> int | None:
# Configured (server-level) value only: per-request values are passed
# explicitly to the image processor call, so they must not be written
# back to the shared processor instance.
return self._mm_max_slice_nums()
def get_hf_processor(self, **kwargs: object):
hf_processor = super().get_hf_processor(**kwargs)
image_processor = getattr(hf_processor, "image_processor", None)
configured = self._configured_max_slice_nums()
if image_processor is not None and configured is not None:
image_processor.max_slice_nums = configured
return hf_processor
def get_image_max_slice_num(self) -> int:
max_slice = self._configured_max_slice_nums()
if max_slice is not None:
return max_slice
return super().get_image_max_slice_num()
class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor):
def _get_mm_fields_config(
self,
hf_inputs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs))
# Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of
# a prefix-cache-covered item only keeps CPU-side metadata, so without
# this a cached image loses its target sizes, `image_bounds` and
# `target_sizes` disagree, and the whole request silently falls back to
# sequential positions.
for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")):
if key in fields:
fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True)
return fields
def _video_local_id_prefix(self, video_idx: int) -> str:
# MiniCPMV4_7Processor emits no local `<image_id>` in front of video
# placeholders: a video is one temporal sequence and 4.7 was trained
# without them.
return ""
def get_image_prompt_texts(
self,
image_size,
image_idx: int = 0,
downsample_mode: str | None = None,
max_slice_nums: int | None = None,
) -> str:
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
if max_slice_nums is None:
max_slice_nums = info.get_image_max_slice_num()
return info.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
max_slice_nums=max_slice_nums,
downsample_mode=downsample_mode,
)
def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor:
data = tensor.to(torch.long)
if data.numel() == 0:
return data.reshape(0, 2)
if data.shape[-1] != 2:
raise ValueError(f"expected last dim 2, got {tuple(data.shape)}")
return data.reshape(-1, 2)
def _sequential_positions(seq_len: int) -> torch.Tensor:
"""Positions for the fallback where canvas M-RoPE cannot be computed."""
return torch.arange(seq_len).unsqueeze(0).expand(3, -1)
@MULTIMODAL_REGISTRY.register_processor(
MiniCPMV4_7MultiModalProcessor,
info=MiniCPMV4_7ProcessingInfo,
dummy_inputs=MiniCPMVDummyInputsBuilder,
)
class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self._init_canvas_mrope(vllm_config)
def _drop_vision_last_layer(self) -> bool:
# MiniCPMV4_7Config no longer carries this field (transformers marks it
# removed), and 4.7 keeps every vision encoder layer.
return False
def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None:
config = self.config
uses_canvas = bool(getattr(config, "uses_mrope_canvas", False))
if not uses_canvas:
mrope_mode = getattr(config, "mrope_mode", None)
uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower()
self._uses_canvas_mrope = uses_canvas
self._canvas_special_ids: dict[str, int] | None = None
if not uses_canvas:
return
# MiniCPMV4_7Config pins the structural ids in config.json; the
# conversion script resolves them from the tokenizer. Prefer them so the
# canvas matches training, and fall back to the tokenizer for checkpoints
# whose config predates these fields.
configured_ids: dict[str, int | None] = {
"im_start_id": getattr(config, "image_start_id", None),
"im_end_id": getattr(config, "image_end_id", None),
"slice_start_id": getattr(config, "slice_start_id", None),
"slice_end_id": getattr(config, "slice_end_id", None),
"newline_id": getattr(config, "newline_id", None),
}
if all(value is not None for value in configured_ids.values()):
self._canvas_special_ids = {
name: int(value)
for name, value in configured_ids.items()
if value is not None
}
return
from vllm.tokenizers.registry import get_tokenizer
model_config = vllm_config.model_config
tokenizer = get_tokenizer(
model_config.tokenizer,
revision=model_config.tokenizer_revision,
trust_remote_code=model_config.trust_remote_code,
)
self._canvas_special_ids = {
"im_start_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "image_start_token", "<image>")
),
"im_end_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "image_end_token", "</image>")
),
"slice_start_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "slice_start_token", "<slice>")
),
"slice_end_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "slice_end_token", "</slice>")
),
"newline_id": tokenizer.encode("\n", add_special_tokens=False)[0],
}
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
seq_len = len(input_tokens)
if (
not getattr(self, "_uses_canvas_mrope", False)
or self._canvas_special_ids is None
or not mm_features
):
return _sequential_positions(seq_len), 0
input_ids = torch.tensor(input_tokens, dtype=torch.long)
image_bounds = build_image_bounds(input_ids, self._canvas_special_ids)
target_sizes_list: list[torch.Tensor] = []
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
if mm_feature.data is None:
continue
for key in ("tgt_sizes", "video_tgt_sizes"):
tgt = mm_feature.data.get(key)
if tgt is None or not isinstance(tgt.data, torch.Tensor):
continue
try:
target_sizes_list.append(_as_hw_pairs(tgt.data))
except ValueError:
# Not an (N, 2) size table. The bounds/sizes match below
# turns this into a sequential fallback.
continue
if not target_sizes_list or image_bounds.numel() == 0:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE missing inputs: "
"tgt_chunks=%s image_bounds=%s; using sequential positions.",
len(target_sizes_list),
tuple(image_bounds.shape) if image_bounds is not None else None,
)
return _sequential_positions(seq_len), 0
target_sizes = torch.cat(target_sizes_list, dim=0)
if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; "
"using sequential positions.",
tuple(target_sizes.shape),
)
return _sequential_positions(seq_len), 0
if image_bounds.shape[0] != target_sizes.shape[0]:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE size mismatch: "
"image_bounds=%s target_sizes=%s; using sequential positions.",
tuple(image_bounds.shape),
tuple(target_sizes.shape),
)
return _sequential_positions(seq_len), 0
try:
pos3d = _compute_canvas_single(
input_ids,
torch.arange(seq_len, dtype=torch.long),
image_bounds,
target_sizes,
self._canvas_special_ids,
)
except Exception:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE failed; "
"falling back to sequential positions.",
exc_info=True,
)
return _sequential_positions(seq_len), 0
return pos3d, canvas_rope_delta(pos3d, seq_len)
# Canvas 3D M-RoPE: position ids for the slice canvas.
def compute_llm_grid(
n_tokens: int, vision_grid_height: int, vision_grid_width: int
) -> tuple[int, int]:
if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0:
return 0, 0
vit_total = vision_grid_height * vision_grid_width
if vit_total == n_tokens:
return vision_grid_height, vision_grid_width
if vit_total < n_tokens:
return 0, 0
factor = int(round(math.sqrt(vit_total / n_tokens)))
if (
factor > 0
and vision_grid_height % factor == 0
and vision_grid_width % factor == 0
):
height, width = vision_grid_height // factor, vision_grid_width // factor
if height * width == n_tokens:
return height, width
ratio = vision_grid_height / vision_grid_width
width = max(1, int(round(math.sqrt(n_tokens / ratio))))
height = n_tokens // width
if height * width == n_tokens and height > 0:
return height, width
return 0, 0
def build_image_bounds(
input_ids: torch.LongTensor, special_token_ids: dict
) -> torch.LongTensor:
start_ids = [
x
for x in (
special_token_ids.get("im_start_id"),
special_token_ids.get("slice_start_id"),
)
if x is not None
]
end_ids = [
x
for x in (
special_token_ids.get("im_end_id"),
special_token_ids.get("slice_end_id"),
)
if x is not None
]
if not start_ids or not end_ids:
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
start_cond = torch.zeros_like(input_ids, dtype=torch.bool)
end_cond = torch.zeros_like(input_ids, dtype=torch.bool)
for sid in start_ids:
start_cond |= input_ids == sid
for eid in end_ids:
end_cond |= input_ids == eid
starts = torch.where(start_cond)[0] + 1
ends = torch.where(end_cond)[0]
n = min(len(starts), len(ends))
if n == 0:
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
return torch.stack([starts[:n], ends[:n]], dim=-1)
def _compute_canvas_single(
input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids
):
# Inference always handles one unpadded sequence. The packed (cu_seqlens)
# form below is kept only to mirror the transformers reference
# implementation of its canvas position computation.
seq_len = input_ids.shape[0]
cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long)
return _compute_canvas_packed(
position_ids_2d.unsqueeze(0),
cu_seqlens,
image_bound,
target_sizes,
input_ids.unsqueeze(0),
special_token_ids,
)[:, 0, :]
def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
"""Patch-grid height/width recorded for one image or slice."""
return int(target_sizes[index, 0]), int(target_sizes[index, 1])
class _CanvasLayout(NamedTuple):
"""Grid of one canvas measured in LLM tokens.
A canvas is a thumbnail plus the slices cut from it, tiled row-major.
"""
thumb_h: int
thumb_w: int
"""Thumbnail grid."""
slice_h: int
slice_w: int
"""Per-slice grid. Zero when the canvas has no slices."""
cols: int
"""Slices sharing a canvas row. Zero when the canvas has no slices."""
height: int
width: int
"""Assembled canvas."""
def _canvas_layout(
thumbnail: tuple[int, int, int],
slices: list[tuple[int, int, int]],
target_sizes: torch.Tensor,
) -> _CanvasLayout:
"""Measure the canvas for one ``(thumbnail, slices)`` group.
``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
"""
t_bs, t_be, t_gi = thumbnail
thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
if not slices:
return _CanvasLayout(
thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
)
cols = len(slices)
for k in range(len(slices) - 1):
if slices[k + 1][0] - slices[k][1] > 2:
cols = k + 1
break
rows = len(slices) // cols if cols > 0 else 1
if rows * cols != len(slices):
rows, cols = 1, len(slices)
s0_bs, s0_be, s0_gi = slices[0]
slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
return _CanvasLayout(
thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
)
def _assign_canvas_span(
pos3d: torch.Tensor,
*,
thumbnail: tuple[int, int, int],
span_end: int,
slices: list[tuple[int, int, int]],
base: int,
target_sizes: torch.Tensor,
) -> int:
"""Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.
Shared by the video-frame and single-image paths; they differ only in how
the per-canvas base advances, not in the geometry placed here.
Mutates ``pos3d`` in place, as the packed canvas computation has always
done. Returns the distance to the next canvas base, so each caller keeps
only its own base advancement.
"""
t_bs, t_be, _ = thumbnail
device = pos3d.device
span_start = t_bs - 1 # the <im_start> token
n_thumb = t_be - t_bs
layout = _canvas_layout(thumbnail, slices, target_sizes)
cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width
# base coat
pos3d[0, 0, span_start:span_end] = base
pos3d[1, 0, span_start:span_end] = base
pos3d[2, 0, span_start:span_end] = base
# <im_start> -> halo
halo_lo = max(base - 1, 0)
pos3d[1, 0, span_start] = halo_lo
pos3d[2, 0, span_start] = halo_lo
# <im_end> -> halo (right after thumbnail)
if t_be < span_end:
pos3d[1, 0, t_be] = base + canvas_H
pos3d[2, 0, t_be] = base + canvas_W
# thumbnail visual tokens
llm_th, llm_tw = layout.thumb_h, layout.thumb_w
if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
if slices and canvas_H > 0 and canvas_W > 0:
h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
else:
h_c = torch.arange(llm_th, device=device)
w_c = torch.arange(llm_tw, device=device)
h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
pos3d[0, 0, t_bs:t_be] = base
pos3d[1, 0, t_bs:t_be] = h_idx + base
pos3d[2, 0, t_bs:t_be] = w_idx + base
else:
fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
for d in range(3):
pos3d[d, 0, t_bs:t_be] = fallback
# slice visual tokens + slice specials
for k, (s_bs, s_be, s_gi) in enumerate(slices):
n_s = s_be - s_bs
sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
h_off = (k // cols) * layout.slice_h
w_off = (k % cols) * layout.slice_w
slice_start_pos = s_bs - 1
if slice_start_pos >= span_start:
pos3d[1, 0, slice_start_pos] = base + h_off
pos3d[2, 0, slice_start_pos] = base + w_off
slice_end_pos = s_be
if sh_k > 0 and sw_k > 0:
end_h = h_off + sh_k - 1
end_w = w_off + sw_k - 1
else:
end_h = h_off
end_w = w_off
if slice_end_pos < span_end:
pos3d[1, 0, slice_end_pos] = base + end_h
pos3d[2, 0, slice_end_pos] = base + end_w
if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
h_idx = (
torch.arange(sh_k, device=device)
.view(-1, 1)
.expand(-1, sw_k)
.reshape(-1)
)
w_idx = (
torch.arange(sw_k, device=device)
.view(1, -1)
.expand(sh_k, -1)
.reshape(-1)
)
pos3d[0, 0, s_bs:s_be] = base
pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
else:
fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
for d in range(3):
pos3d[d, 0, s_bs:s_be] = fallback
# \n between slice rows -> W = right edge + 1.
# H stays within the ended row (does NOT cross to next row): the \n sits at
# the right-outside of the row that just ended, at H = that row's last row
# and W = one column past the rightmost slice.
for k in range(len(slices) - 1):
gap_s = slices[k][1]
gap_e = slices[k + 1][0]
if gap_e - gap_s <= 2:
continue
boundary_h = ((k // cols) + 1) * layout.slice_h - 1
right_edge_w = cols * layout.slice_w
for nl_pos in range(gap_s + 1, gap_e - 1):
pos3d[1, 0, nl_pos] = base + boundary_h
pos3d[2, 0, nl_pos] = base + right_edge_w
return max(canvas_H, canvas_W) + 1
def _compute_canvas_packed(
position_ids_2d,
cu_seqlens,
image_bound,
target_sizes,
input_ids,
special_token_ids,
):
if position_ids_2d.ndim == 1:
position_ids_2d = position_ids_2d.unsqueeze(0)
if input_ids.ndim == 1:
input_ids = input_ids.unsqueeze(0)
has_images = (
isinstance(image_bound, torch.Tensor)
and image_bound.numel() > 0
and isinstance(target_sizes, torch.Tensor)
and target_sizes.numel() > 0
)
if not has_images:
return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous()
device = position_ids_2d.device
pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone()
ids_flat = input_ids.view(-1)
im_start_id = special_token_ids.get("im_start_id")
im_end_id = special_token_ids.get("im_end_id")
slice_start_id = special_token_ids.get("slice_start_id")
slice_end_id = special_token_ids.get("slice_end_id")
structural_ids = {
v
for v in (im_start_id, im_end_id, slice_start_id, slice_end_id)
if v is not None
}
num_seqs = len(cu_seqlens) - 1
img_ptr = 0
num_imgs = len(image_bound)
newline_id = special_token_ids.get("newline_id")
gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set())
def _group_end_pos(g, s_end):
"""First position after a group's trailing structural tokens.
Matches the transformers reference, which closes a span over every
marker in ``markers`` -- newlines included. The chat template joins
content parts with ``\\n``, so an image followed by text always ends with
one; leaving it out moves that newline out of the canvas and pushes
every later position one step forward.
"""
le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1]
ge = le
while ge < s_end and ids_flat[ge].item() in gap_ok_ids:
ge += 1
return min(ge, s_end)
def _is_tight_gap(g_prev, g_next, s_end):
"""Check if only structural/nl tokens sit between two groups."""
ge = _group_end_pos(g_prev, s_end)
ns = g_next["thumbnail"][0] - 1 # im_start of next group
return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns))
for si in range(num_seqs):
s_start = cu_seqlens[si].item()
s_end = cu_seqlens[si + 1].item()
if s_start >= s_end:
continue
seq_imgs = []
while img_ptr < num_imgs:
bs_ = image_bound[img_ptr, 0].item()
be_ = image_bound[img_ptr, 1].item()
if bs_ >= s_start and be_ <= s_end:
seq_imgs.append((bs_, be_, img_ptr))
img_ptr += 1
else:
break
if not seq_imgs:
continue
# ---- group images: thumbnail + following slices ----
# merge consecutive no-slice frames into one video group
raw_groups = []
cur_group: dict[str, Any] | None = None
for bs_, be_, gi in seq_imgs:
marker_pos = bs_ - 1
is_slice = (
slice_start_id is not None
and marker_pos >= s_start
and ids_flat[marker_pos].item() == slice_start_id
)
if is_slice and cur_group is not None:
cur_group["slices"].append((bs_, be_, gi))
else:
if cur_group is not None:
raw_groups.append(cur_group)
cur_group = {"thumbnail": (bs_, be_, gi), "slices": []}
if cur_group is not None:
raw_groups.append(cur_group)
# merge consecutive tightly-adjacent groups into video groups.
# Two groups are "tightly adjacent" if the gap between them
# contains only structural tokens and \n (no real text).
# This handles both no-slice frames and frames-with-slices.
groups = []
i_g = 0
while i_g < len(raw_groups):
video_frames = [raw_groups[i_g]]
j_g = i_g + 1
while j_g < len(raw_groups) and _is_tight_gap(
raw_groups[j_g - 1], raw_groups[j_g], s_end
):
video_frames.append(raw_groups[j_g])
j_g += 1
if len(video_frames) == 1:
groups.append(raw_groups[i_g])
else:
groups.append({"video_frames": video_frames})
i_g = j_g
# ---- assign 3-D positions ----
pos = 0
cursor = s_start
for group in groups:
is_video_group = "video_frames" in group
if is_video_group:
# === Video group: multiple frames merged ===
# Each frame gets its own base_f so different frames have
# distinct spatial positions. Inter-frame gap tokens (\n
# etc.) are pure 1D text. After each gap, pos advances
# by 1 extra so halo_lo of the next frame does not
# collide with the last gap token.
frames = group["video_frames"]
first_frame = frames[0]
last_frame = frames[-1]
# group_start = <im_start> of first frame
group_start = first_frame["thumbnail"][0] - 1
# group_end = after last structural token of last frame
group_end = _group_end_pos(last_frame, s_end)
# text before video group
text_len = group_start - cursor
if text_len > 0:
t_pos = (
torch.arange(text_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, cursor:group_start] = t_pos
pos += text_len
# process each frame with its own per-frame base
frame_cursor = group_start
for vf in frames:
t_bs = vf["thumbnail"][0]
vf_slices = vf["slices"]
im_start_pos = t_bs - 1
# --- inter-frame gap: pure 1D text T=H=W=pos ---
gap_len = im_start_pos - frame_cursor
if gap_len > 0:
t_pos = (
torch.arange(gap_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, frame_cursor:im_start_pos] = t_pos
pos += gap_len
pos += 1 # buffer: prevents halo_lo collision
# --- per-frame base ---
base_f = pos
frame_end = _group_end_pos(vf, group_end)
pos = base_f + _assign_canvas_span(
pos3d,
thumbnail=vf["thumbnail"],
span_end=frame_end,
slices=vf_slices,
base=base_f,
target_sizes=target_sizes,
)
frame_cursor = frame_end
# trailing tokens after last frame (if any)
if frame_cursor < group_end:
trail_len = group_end - frame_cursor
t_pos = (
torch.arange(trail_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, frame_cursor:group_end] = t_pos
pos += trail_len
cursor = group_end
else:
# === Single image group (thumbnail + optional slices) ===
slices = group["slices"]
group_start = group["thumbnail"][0] - 1
group_end = _group_end_pos(group, s_end)
# text before group
text_len = group_start - cursor
if text_len > 0:
t_pos = (
torch.arange(text_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, cursor:group_start] = t_pos
pos += text_len
base = pos
pos = base + _assign_canvas_span(
pos3d,
thumbnail=group["thumbnail"],
span_end=group_end,
slices=slices,
base=base,
target_sizes=target_sizes,
)
cursor = group_end
# remaining text
rem = s_end - cursor
if rem > 0:
t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos
for d in range(3):
pos3d[d, 0, cursor:s_end] = t_pos
return pos3d
def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
"""Offset applied to the 1-D positions of tokens decoded after the prompt.
The first decoded token sits one past the last prefill position.
"""
return int(position_ids.amax().item()) + 1 - seq_len
+17 -4
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, cast
from typing import Annotated, Any, Literal, TypeAlias, cast
import torch
import torch.nn as nn
@@ -76,6 +76,19 @@ class Mistral3ImagePixelInputs(TensorSchema):
]
class Mistral3ImageEmbeddingInputs(TensorSchema):
"""Projected features with shape (num_images, feature_size, hidden_size)."""
type: Literal["image_embeds"] = "image_embeds"
data: Annotated[
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "ifs", "hs", dynamic_dims={"ifs"}),
]
Mistral3ImageInputs: TypeAlias = Mistral3ImagePixelInputs | Mistral3ImageEmbeddingInputs
class Mistral3PatchMerger(nn.Module):
"""Learned merging of spatial_merge_size ** 2 patches."""
@@ -494,7 +507,7 @@ class Mistral3ForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Mistral3ImagePixelInputs | None:
) -> Mistral3ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -508,8 +521,8 @@ class Mistral3ForConditionalGeneration(
def _process_image_input(
self,
image_input: Mistral3ImagePixelInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
image_input: Mistral3ImageInputs,
) -> MultiModalEmbeddings:
if image_input["type"] == "image_embeds":
return image_input["data"]
@@ -1,182 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .utils import maybe_prefix
class MLPSpeculatorLayerNorm(nn.Module):
"""A L2 normalization implementation
...
Args:
----
normalized_shape : int
Dimensionality of input data (size of final tensor axis)
eps : float
Safety term to prevent division by zero. Make sure the chosen value
fits in the range of your encoding scheme
(i.e. fp16 requires eps >= 6e-8).
elementwise_scale_and_shift : bool
Include a learned scaling and shift term after normalization.
"""
def __init__(
self,
normalized_shape,
eps=1e-06,
elementwise_scale_and_shift=True,
):
super().__init__()
self.elementwise_scale_and_shift = elementwise_scale_and_shift
if self.elementwise_scale_and_shift:
self.weight = nn.Parameter(torch.empty(normalized_shape))
self.bias = nn.Parameter(torch.empty(normalized_shape))
self.eps = eps
def forward(self, x):
xf = x
xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
x = xf.type_as(x)
if self.elementwise_scale_and_shift:
x = self.weight * x
x = x + self.bias
return x
class MLPSpeculator(nn.Module):
"""An implementation of the speculative models introduced in
"Accelerating Production LLMs with Combined Token/Embedding
Speculators"
https://arxiv.org/pdf/2404.19124
Trained speculators of this type are available on HF hub at:
https://huggingface.co/ibm-ai-platform and https://huggingface.co/ibm-granite
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_config
self.n_predict = config.n_predict
self.vocab_size = config.vocab_size
self.emb_dim = config.emb_dim
self.inner_dim = config.inner_dim if config.inner_dim != 0 else config.emb_dim
self.max_speculative_tokens = config.num_lookahead_tokens
self.tie_weights = config.tie_weights
self.scale_input = config.scale_input
if self.tie_weights:
assert self.n_predict > 1, (
"You cannot tie weights between stages when only 1 exists"
)
embedding = VocabParallelEmbedding(
config.vocab_size, self.inner_dim, org_num_embeddings=config.vocab_size
)
self.emb = nn.ModuleList([embedding] * self.max_speculative_tokens)
# the initial projection from the base model may
# have a different size, so that stays separate.
proj_first = nn.Linear(self.emb_dim, self.inner_dim, bias=False)
proj_tied = nn.Linear(self.inner_dim, self.inner_dim, bias=False)
self.proj = nn.ModuleList(
[proj_first] + [proj_tied] * (self.max_speculative_tokens - 1)
)
self.head = nn.ModuleList(
[
ParallelLMHead(
self.vocab_size,
self.inner_dim,
bias=False,
prefix=maybe_prefix(prefix, f"head.{i}"),
)
for i in range(self.max_speculative_tokens)
]
)
ln = MLPSpeculatorLayerNorm(
self.inner_dim, elementwise_scale_and_shift=True
)
self.ln = nn.ModuleList([ln] * self.max_speculative_tokens)
else:
self.emb = nn.ModuleList(
[
VocabParallelEmbedding(
config.vocab_size,
self.inner_dim,
)
for _ in range(self.max_speculative_tokens)
]
)
self.proj = nn.ModuleList(
[
nn.Linear(
(self.emb_dim if i == 0 else self.inner_dim),
self.inner_dim,
bias=False,
)
for i in range(self.max_speculative_tokens)
]
)
self.head = nn.ModuleList(
[
ParallelLMHead(
self.vocab_size,
self.inner_dim,
bias=False,
prefix=maybe_prefix(prefix, f"head.{i}"),
)
for i in range(self.max_speculative_tokens)
]
)
self.ln = nn.ModuleList(
[
MLPSpeculatorLayerNorm(
self.inner_dim, elementwise_scale_and_shift=True
)
for _ in range(self.max_speculative_tokens)
]
)
if self.scale_input:
self.ln0 = MLPSpeculatorLayerNorm(
self.emb_dim, elementwise_scale_and_shift=False
)
self.state_weight = 0.5 ** (0.5 / config.n_predict)
self.emb_weight = math.sqrt((1 - self.state_weight**2) * (self.inner_dim / 2))
self.activation = nn.GELU()
self.config = config
self.logits_processor = LogitsProcessor(
config.vocab_size, config.vocab_size, 1.0
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
name = name.replace("speculator.", "")
param = params_dict.get(name)
if param is not None:
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
+34 -1
View File
@@ -1001,6 +1001,16 @@ class NemotronH_Nano_VL_V2(
vision_projection_hidden_size = config.projector_hidden_size
llm_hidden_size = config.text_config.hidden_size
# Match the HF vision projector for checkpoints trained with MTP.
self.vision_final_layernorm = (
nn.LayerNorm(
vit_hidden_size,
eps=getattr(vision_config, "layer_norm_eps", 1e-6),
).to(llm_dtype)
if (getattr(config.text_config, "num_nextn_predict_layers", 0) or 0) > 0
else None
)
mlp1 = nn.Sequential(
RMSNorm(
hidden_size=vit_hidden_size
@@ -1095,6 +1105,8 @@ class NemotronH_Nano_VL_V2(
"""Dynamic resolution extract_feature for images."""
_, vit_embeds = self.vision_model(pixel_values, imgs_sizes=imgs_sizes)
vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
if self.vision_final_layernorm is not None:
vit_embeds = self.vision_final_layernorm(vit_embeds)
vit_embeds = self.pixel_shuffle_dynamic_res(vit_embeds, imgs_sizes=imgs_sizes)
vit_embeds = self.mlp1(vit_embeds)
return vit_embeds
@@ -1127,6 +1139,8 @@ class NemotronH_Nano_VL_V2(
else:
_, vit_embeds = self.vision_model(chunk)
vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
if self.vision_final_layernorm is not None:
vit_embeds = self.vision_final_layernorm(vit_embeds)
vit_embeds = vit_embeds.reshape(
vit_embeds.shape[0], H_patches, W_patches, -1
)
@@ -1575,7 +1589,7 @@ class NemotronH_Nano_VL_V2(
"""Get the module prefix in multimodal models"""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector=["mlp1", "sound_encoder.projection"],
connector=["mlp1", "sound_encoder.projection", "vision_final_layernorm"],
tower_model=["vision_model", "sound_encoder.encoder"],
)
@@ -1593,6 +1607,11 @@ class NemotronH_Nano_VL_V2(
for modality in ("image", "video", "audio")
)
adapter_dict = dict(self.mlp1.named_parameters())
vision_norm_params = (
dict(self.vision_final_layernorm.named_parameters())
if load_multimodal_weights and self.vision_final_layernorm is not None
else {}
)
def is_llm(name: str) -> bool:
return name.startswith("language_model")
@@ -1606,6 +1625,11 @@ class NemotronH_Nano_VL_V2(
def is_sound_weights(name: str) -> bool:
return name.startswith("sound")
vision_norm_prefix = "vision_projector.vision_final_layernorm."
def is_vision_norm_weights(name: str) -> bool:
return name.startswith(vision_norm_prefix)
# LLM weights (the bulk of the model) are streamed lazily through a
# generator so each tensor is copied into its parameter before the
# iterator advances, avoiding stale-reference corruption with
@@ -1637,6 +1661,15 @@ class NemotronH_Nano_VL_V2(
continue
assert self.sound_encoder is not None
sound_weights.append((name, w.detach().clone()))
elif is_vision_norm_weights(name):
if not load_multimodal_weights:
continue
param_name = name.removeprefix(vision_norm_prefix)
assert param_name in vision_norm_params, (
f"Unexpected vision LayerNorm weight: {name}"
)
with torch.no_grad():
default_weight_loader(vision_norm_params[param_name], w)
# Fully drain the generator so every mm tensor is buffered, even if
# the LLM loader stops iterating early.
+2 -2
View File
@@ -74,7 +74,6 @@ from vllm.model_executor.models.interfaces import (
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsMambaPrefixCaching,
SupportsPP,
SupportsQuant,
SupportsReplaySSM,
@@ -585,6 +584,8 @@ class NemotronHModel(nn.Module, EagleModelMixin):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.embed_tokens",
)
self.has_moe = "E" in config.hybrid_override_pattern
@@ -725,7 +726,6 @@ class NemotronHForCausalLM(
IsHybrid,
SupportsQuant,
MixtureOfExperts,
SupportsMambaPrefixCaching,
SupportsReplaySSM,
):
# Relevant only if self.has_moe is True
+88 -3
View File
@@ -9,11 +9,13 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
from typing import Annotated, Any, Literal, cast
import numpy as np
import torch
import torch.nn as nn
from einops import rearrange
from PIL import Image
from transformers import (
BartConfig,
BatchFeature,
@@ -26,6 +28,7 @@ from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
@@ -401,6 +404,56 @@ class NemotronParseDummyInputsBuilder(
}
def _to_pil_image(image: object) -> Image.Image:
"""Convert a `uint8` array/tensor image item to PIL, since the checkpoint's
image processor calls `.convert("RGB")` on every input."""
if isinstance(image, Image.Image):
return image
array = image.numpy() if isinstance(image, torch.Tensor) else np.asarray(image)
if array.dtype != np.uint8 or array.ndim != 3:
raise ValueError(
"Nemotron-Parse expects image arrays/tensors to be 3-D uint8, "
f"got {array.dtype} {array.shape}"
)
# Same layout heuristic as `ImageProcessorItems.get_image_size`.
if array.shape[-1] not in (1, 3, 4):
array = array.transpose(1, 2, 0)
if array.shape[-1] == 1:
array = array[..., 0]
return Image.fromarray(np.ascontiguousarray(array))
def _build_uint8_canvas(
image_processor: Any, images: Sequence[Image.Image]
) -> torch.Tensor:
"""Reproduce the image processor's pipeline up to, but excluding, its
`ToTensor` rescale, yielding the padded `uint8` canvas as `(N, 3, H, W)`.
Raises `AttributeError`/`TypeError` if the checkpoint's remote code does
not expose that pipeline.
"""
resize = image_processor._resize_with_aspect_ratio
pad = image_processor._pad_to_size
# v1.x pads with albumentations, 2.0 sets `transform` to None and pads
# with `_pad_to_size`; mirror `preprocess` and pick the same branch.
transform = image_processor.transform
canvases = []
for image in images:
array = np.asarray(image.convert("RGB"))
array = resize(array)
array = transform(image=array)["image"] if transform is not None else pad(array)
if array.dtype != np.uint8 or array.ndim != 3 or array.shape[2] != 3:
raise TypeError(
f"expected a uint8 HWC canvas, got {array.dtype} {array.shape}"
)
canvases.append(torch.from_numpy(np.ascontiguousarray(array)).permute(2, 0, 1))
return torch.stack(canvases)
class NemotronParseMultiModalProcessor(
EncDecMultiModalProcessor[NemotronParseProcessingInfo]
):
@@ -411,6 +464,35 @@ class NemotronParseMultiModalProcessor(
) -> list[int]:
return [0]
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
images = [_to_pil_image(i) for i in cast(list, hf_data.get("images") or [])]
if images:
hf_data = {**hf_data, "images": images}
# The checkpoint ships its image processor as remote code, and its
# `preprocess` ignores `do_rescale`/`do_normalize`: it always runs
# `ToTensor` and normalises from `self.do_normalize`. Build the uint8
# canvas here so the device-side norm is not applied on top.
mm_config = self.info.ctx.model_config.get_multimodal_config()
if images and set(hf_data) == {"images"} and mm_config.mm_device_do_normalize:
image_processor = self.info.get_hf_processor().image_processor
try:
pixel_values = _build_uint8_canvas(image_processor, images)
except (AttributeError, TypeError) as exc:
raise RuntimeError(
f"{type(image_processor).__name__} does not expose the "
f"expected pre-rescale pipeline ({exc}), so the uint8 canvas "
"that mm_device_do_normalize requires cannot be built. Pass "
"--no-mm-device-do-normalize to normalise on the host instead."
) from exc
return BatchFeature({"pixel_values": pixel_values})
return super()._call_hf_processor(hf_data, hf_kwargs)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -542,6 +624,8 @@ class RadioWithNeck(nn.Module):
dummy_inputs=NemotronParseDummyInputsBuilder,
)
class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
supports_mm_device_do_normalize = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -550,6 +634,7 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
self.vision_config = config.encoder
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.input_norm = build_mm_input_norm(vllm_config.model_config)
with self._mark_tower_model(vllm_config, "image"):
self.encoder = RadioWithNeck(
@@ -622,8 +707,8 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
assert image_input["type"] == "pixel_values"
pixel_values = image_input["data"]
dtype = next(self.encoder.parameters()).dtype
pixel_values = pixel_values.to(dtype)
return self.encoder(pixel_values)
flat = self.input_norm(pixel_values.flatten(1), dtype)
return self.encoder(flat.view(pixel_values.shape))
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
image_input = self._parse_and_validate_image_input(**kwargs)
+47 -84
View File
@@ -258,28 +258,18 @@ class PaddleOCRVLMultiModalProcessor(
hf_kwargs,
)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if not hf_data:
return processed_data
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
processed_data["pixel_values"] = processed_data["pixel_values"].split(
num_patches_per_image.tolist()
)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
image_grid_sizes = image_grid_thw.prod(-1)
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
pixel_values=MultiModalFieldConfig.flat_from_sizes(
"image", image_grid_sizes
),
image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
)
@@ -347,12 +337,12 @@ class Projector(nn.Module):
) -> torch.Tensor:
m1, m2 = self.merge_kernel_size
if isinstance(image_features, (list, tuple)):
processed_features = list()
for image_feature, image_grid in zip(image_features, image_grid_thw):
image_feature = self.pre_norm(image_feature)
t, h, w = image_grid
image_feature = rearrange(
# Norm and MLP are per-token, so run them once over all images;
# only the 2x2 merge depends on each image's grid.
sizes = [image_feature.shape[0] for image_feature in image_features]
normed_features = self.pre_norm(torch.cat(image_features)).split(sizes)
merged_features = [
rearrange(
image_feature,
"(t h p1 w p2) d -> (t h w) (p1 p2 d)",
t=t,
@@ -361,12 +351,13 @@ class Projector(nn.Module):
w=w // m2,
p2=m2,
)
hidden_states = self.linear_1(image_feature)
hidden_states = self.act(hidden_states)
hidden_states = self.linear_2(hidden_states)
processed_features.append(hidden_states)
for image_feature, (t, h, w) in zip(normed_features, image_grid_thw)
]
hidden_states = self.linear_1(torch.cat(merged_features))
hidden_states = self.act(hidden_states)
hidden_states = self.linear_2(hidden_states)
return processed_features
return hidden_states.split([f.shape[0] for f in merged_features])
dims = image_features.shape[:-1]
dim = image_features.shape[-1]
@@ -383,7 +374,7 @@ class PaddleOCRImagePixelInputs(TensorSchema):
type: Literal["pixel_values"]
pixel_values: Annotated[
torch.Tensor,
TensorShape("bn", "p", 3, "patch_size", "patch_size", dynamic_dims={"p"}),
TensorShape("np", 3, "patch_size", "patch_size"),
]
image_grid_thw: Annotated[
torch.Tensor,
@@ -455,27 +446,6 @@ class SiglipVisionEmbeddings(nn.Module):
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
return patch_pos_embed
def fetch_position_embedding_lfu_cache(
self, embeddings: torch.Tensor, h: int, w: int, max_cache: int = 20
):
grid = (h, w)
if grid in self.cache_position_embedding:
self.cache_position_count[grid] += 1
return self.cache_position_embedding[grid]
if len(self.cache_position_embedding) >= max_cache:
min_hit_grid = min(
self.cache_position_count,
key=self.cache_position_count.__getitem__,
)
self.cache_position_count.pop(min_hit_grid)
self.cache_position_embedding.pop(min_hit_grid)
position_embedding = self.interpolate_pos_encoding(embeddings, h, w, True)
self.cache_position_count[grid] = 1
self.cache_position_embedding[grid] = position_embedding
return position_embedding
def forward(
self,
pixel_values: torch.FloatTensor,
@@ -811,10 +781,10 @@ class SiglipEncoder(nn.Module):
if cu_seqlens is None:
raise ValueError("cu_seqlens cannot be None for SiglipEncoder.")
if not isinstance(cu_seqlens, torch.Tensor):
cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device)
else:
cu_seqlens = cu_seqlens.to(device=device, non_blocking=True)
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32)
# Computed before the H2D copy: the attention wrappers call `.item()`
# on it, which would sync on every layer for a device tensor.
max_seqlen = None
if self.attn_backend in {
AttentionBackendEnum.FLASH_ATTN,
@@ -823,6 +793,11 @@ class SiglipEncoder(nn.Module):
}:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
if cu_seqlens.is_cpu:
cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device)
else:
cu_seqlens = cu_seqlens.to(device=device, non_blocking=True)
hidden_states = inputs_embeds
for encoder_layer in self.layers:
hidden_states = encoder_layer(
@@ -1125,55 +1100,43 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
)
def encode_image(
self, pixel_values: torch.Tensor, image_grid_thw: torch.Tensor
self,
pixel_values: torch.Tensor,
image_grid_thw: list[tuple[int, int, int]],
) -> torch.Tensor:
"""Encode the flat patches of all images in one vision tower call."""
pixel_values = pixel_values.type(self.visual.dtype)
siglip_position_ids = list()
image_grid_hws = list()
cu_seqlens = [0]
seq_lens = [math.prod(grid) for grid in image_grid_thw]
thw_tuple = tuple(image_grid_thw.tolist())
numel = np.prod(thw_tuple)
image_grid_hws.append(thw_tuple)
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
siglip_position_ids.append(image_position_ids)
cu_seqlens.append(cu_seqlens[-1] + numel)
# Both are built on the host; concat straight into a pinned buffer
# so the H2D copy stays non-blocking.
# Built on the host; concat straight into a pinned buffer so the
# H2D copy stays non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
[
torch.arange(seq_len) % (h * w)
for seq_len, (_, h, w) in zip(seq_lens, image_grid_thw)
],
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
out=torch.empty(sum(seq_lens), dtype=torch.int64, pin_memory=PIN_MEMORY),
).to(pixel_values.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values.device
)
# One segment per image so that images never attend to each other.
cu_seqlens = torch.from_numpy(np.cumsum([0, *seq_lens], dtype=np.int32))
vision_outputs = self.visual(
return self.visual(
pixel_values=pixel_values,
image_grid_thw=image_grid_hws,
image_grid_thw=image_grid_thw,
position_ids=siglip_position_ids,
interpolate_pos_encoding=True,
cu_seqlens=cu_seqlens,
)
return vision_outputs
def _process_image_input(
self, image_input: PaddleOCRImagePixelInputs
) -> MultiModalEmbeddings:
pixel_values = image_input.pixel_values
image_grid_thw = image_input.image_grid_thw
vision_outputs = tuple(
self.encode_image(pixel, grid).squeeze(0)
for pixel, grid in zip(pixel_values, image_grid_thw)
)
image_embeds = self.mlp_AR(vision_outputs, image_grid_thw)
return image_embeds
grids = [tuple(grid) for grid in image_grid_thw.tolist()]
vision_outputs = self.encode_image(image_input.pixel_values, grids).squeeze(0)
vision_outputs = vision_outputs.split([math.prod(grid) for grid in grids])
return self.mlp_AR(vision_outputs, image_grid_thw)
def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
@@ -1256,12 +1256,6 @@ class AudioEmbedding(nn.Module):
self.input_embeds = None
self.audio_embed_sizes = None
def set_audio_embeds(self, input_embeds: torch.Tensor) -> None:
self.input_embeds = input_embeds
def set_audio_embed_sizes(self, audio_embed_sizes: torch.Tensor) -> None:
self.audio_embed_sizes = audio_embed_sizes
def get_audio_features(
self,
input_embeds: torch.Tensor,
@@ -132,11 +132,11 @@ def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tenso
def unpad_and_flat_audio_features(
input_audio_features: torch.Tensor,
input_audio_features: torch.Tensor | Sequence[torch.Tensor],
audio_feature_lengths: torch.Tensor,
) -> torch.Tensor:
"""Unpad and flatten batched audio features."""
if len(audio_feature_lengths) != input_audio_features.size(0):
"""Unpad and flatten stacked or ragged per-audio features."""
if len(audio_feature_lengths) != len(input_audio_features):
raise ValueError(
"Length of audio_feature_lengths must match "
"the batch size of input_audio_features."
@@ -920,8 +920,10 @@ class Qwen2_5OmniConditionalGenerationMixin:
# inputs features from rust frontend is batched and padded
# with shape [batch_size, n_mels, padded_seq_len], different
# from python's shape [n_mels, batch_size * seq_len]
if (
# from python's shape [n_mels, batch_size * seq_len].
# Requests with different padded lengths are batched as a
# list of [n_mels, padded_seq_len] tensors.
if isinstance(input_audio_features, (list, tuple)) or (
isinstance(input_audio_features, torch.Tensor)
and input_audio_features.dim() == 3
):
+9 -1
View File
@@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module):
attn_type: str = AttentionType.DECODER,
dual_chunk_attention_config: dict[str, Any] | None = None,
per_layer_sliding_window: int | None = None,
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module):
max_position=max_position,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
mrope_positions_factor=mrope_positions_factor,
)
attn_cls = (
EncoderOnlyAttention
@@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module):
quant_config: QuantizationConfig | None = None,
prefix: str = "",
per_layer_sliding_window: int | None = None,
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
@@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module):
attn_type=attn_type,
dual_chunk_attention_config=dual_chunk_attention_config,
per_layer_sliding_window=per_layer_sliding_window,
mrope_positions_factor=mrope_positions_factor,
)
self.mlp = Qwen3MLP(
hidden_size=self.hidden_size,
@@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = {
}
)
class Qwen3Model(Qwen2Model):
decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(
vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer
vllm_config=vllm_config,
prefix=prefix,
decoder_layer_type=self.decoder_layer_type,
)
+2
View File
@@ -91,6 +91,7 @@ from .qwen3_vl import (
Qwen3VLForConditionalGeneration,
Qwen3VLMultiModalProcessor,
Qwen3VLProcessingInfo,
mrope_positions_factor,
)
from .utils import (
AutoWeightsLoader,
@@ -161,6 +162,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
reduce_results=not self.use_attn_reduce_scatter_for_moe,
mrope_positions_factor=mrope_positions_factor(vllm_config),
)
else:
raise ValueError(f"Invalid layer_type {self.layer_type}")
+66 -13
View File
@@ -29,6 +29,7 @@ import regex as re
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
from transformers.models.qwen3_asr import Qwen3ASRFeatureExtractor, Qwen3ASRProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen3OmniMoeAudioEncoder,
Qwen3OmniMoeThinkerMultiModalProcessor,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -80,6 +80,7 @@ from vllm.multimodal.parse import (
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
@@ -92,9 +93,6 @@ from vllm.transformers_utils.configs.qwen3_asr import (
Qwen3ASRThinkerConfig,
)
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.qwen3_asr import (
Qwen3ASRProcessor,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
@@ -188,18 +186,19 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo):
return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config
def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor:
processor = self.ctx.get_hf_processor(
return self.ctx.get_hf_processor(
Qwen3ASRProcessor,
**kwargs,
)
if not hasattr(processor, "audio_token"):
processor.audio_token = "<|audio_pad|>"
return processor
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
def get_feature_extractor(
self, **kwargs: object
) -> WhisperFeatureExtractor | Qwen3ASRFeatureExtractor:
hf_processor = self.get_hf_processor(**kwargs)
feature_extractor = hf_processor.feature_extractor
assert isinstance(feature_extractor, WhisperFeatureExtractor)
assert isinstance(
feature_extractor, (WhisperFeatureExtractor, Qwen3ASRFeatureExtractor)
)
return feature_extractor
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -275,8 +274,60 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
class Qwen3ASRMultiModalProcessor(
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
BaseMultiModalProcessor[Qwen3ASRProcessingInfo],
):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
audios = hf_data.get("audio")
if not audios:
return BatchFeature()
assert isinstance(audios, list)
if len(audios) == 1:
return super()._call_hf_processor(hf_data, hf_kwargs)
# HF requires one text per audio, and batch padding changes clip features.
# Keep clips independent so partial cache hits cannot change their inputs.
text = self._get_hf_mm_text({"audio": 1})
outputs: dict[str, list[torch.Tensor]] = {
"input_features": [],
"input_features_mask": [],
}
for audio in audios:
processed = super()._call_hf_processor(
dict(hf_data, text=text, audio=audio), hf_kwargs
)
for key, values in outputs.items():
values.append(processed[key][0])
return BatchFeature(outputs)
def _postprocess_hf_mm_data(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
processed_data: BatchFeature,
) -> BatchFeature:
if "input_features" not in processed_data:
return processed_data
features = processed_data.pop("input_features")
masks = processed_data.pop("input_features_mask")
processed_data["input_audio_features"] = torch.cat(
[feature[:, mask.bool()] for feature, mask in zip(features, masks)], dim=1
)
processed_data["feature_attention_mask"] = masks
processed_data["audio_feature_lengths"] = torch.stack(
[mask.sum() for mask in masks]
)
return processed_data
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
@@ -427,8 +478,10 @@ class Qwen3ASRForConditionalGeneration(
# inputs features from rust frontend is batched and padded
# with shape [batch_size, n_mels, padded_seq_len], different
# from python's shape [n_mels, batch_size * seq_len]
if (
# from python's shape [n_mels, batch_size * seq_len].
# Requests with different padded lengths are batched as a
# list of [n_mels, padded_seq_len] tensors.
if isinstance(input_audio_features, (list, tuple)) or (
isinstance(input_audio_features, torch.Tensor)
and input_audio_features.dim() == 3
):
+3 -22
View File
@@ -14,7 +14,6 @@ from vllm import _custom_ops as ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
@@ -33,6 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
from vllm.multimodal.inputs import NestedTensors
from vllm.platforms import current_platform
from vllm.transformers_utils.config import set_default_rope_theta
@@ -251,9 +251,7 @@ class DFlashQwen3Attention(nn.Module):
)
self.attention_sink_bias = (
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
if add_swa_attention_sink_bias
else None
make_attention_sink(self.num_heads) if add_swa_attention_sink_bias else None
)
self.sliding_window = sliding_window
@@ -746,26 +744,9 @@ class DFlashQwen3Model(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def _preprocess(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
for name, loaded_weight in weights:
if "attention_sink_bias" in name:
# Sink bias is per-head; shard it across TP ranks like the
# attention heads themselves.
heads_per_rank = loaded_weight.shape[0] // tp_size
loaded_weight = loaded_weight.narrow(
0, tp_rank * heads_per_rank, heads_per_rank
)
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(
self._preprocess(weights), mapper=self.hf_to_vllm_mapper
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
+4
View File
@@ -256,6 +256,7 @@ class Qwen3MoeAttention(nn.Module):
quant_config: QuantizationConfig | None = None,
prefix: str = "",
dual_chunk_attention_config: dict[str, Any] | None = None,
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -303,6 +304,7 @@ class Qwen3MoeAttention(nn.Module):
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
mrope_positions_factor=mrope_positions_factor,
)
attention_kwargs: dict[str, Any] = {}
if dual_chunk_attention_config:
@@ -351,6 +353,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
vllm_config: VllmConfig,
prefix: str = "",
is_fused_checkpoint_transposed: bool = False,
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
@@ -376,6 +379,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
dual_chunk_attention_config=dual_chunk_attention_config,
mrope_positions_factor=mrope_positions_factor,
)
# `mlp_only_layers` in the config.
+9 -1
View File
@@ -129,7 +129,12 @@ def _should_replicate_misaligned_shared_expert(
class Qwen3NextSparseMoeBlock(nn.Module):
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
reduce_results: bool = True,
):
super().__init__()
config = vllm_config.model_config.hf_text_config
@@ -231,6 +236,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
num_redundant_experts=self.n_redundant_experts,
is_sequence_parallel=self.is_sequence_parallel,
n_shared_experts=1 if self.shared_expert is None else None,
reduce_results=reduce_results,
fuse_shared_experts=self.is_fused_shared_expert_enabled,
shared_expert_prefix=f"{prefix}.shared_expert",
shared_expert_gate=self.shared_expert_gate
@@ -280,6 +286,7 @@ class Qwen3NextAttention(nn.Module):
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
mrope_positions_factor: int = 4,
) -> None:
super().__init__()
self.config = config
@@ -331,6 +338,7 @@ class Qwen3NextAttention(nn.Module):
max_position=config.max_position_embeddings,
rope_parameters=config.rope_parameters,
dual_chunk_attention_config=self.dual_chunk_attention_config,
mrope_positions_factor=mrope_positions_factor,
)
# Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL
@@ -83,6 +83,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.utils import copy_mm_embedding_modality
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -1498,8 +1499,12 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
audio_in_video_item_idx += 1
# Match the fps the processor used, as HF and the mrope positions do.
processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
if second_per_grid_ts:
if isinstance(processed_second_per_grid_ts, torch.Tensor):
video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item()
elif second_per_grid_ts:
video_second_per_grid_t = second_per_grid_ts[item_idx]
else:
video_second_per_grid_t = 2.0
@@ -1904,7 +1909,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
embeddings_main, embeddings_multiscale = torch.split(
embeddings, [visual_dim, multi_dim], dim=-1
)
multimodal_embeddings[index] = embeddings_main
multimodal_embeddings[index] = copy_mm_embedding_modality(
embeddings, embeddings_main
)
multimodal_embeddings_multiscale.append(embeddings_multiscale)
if not is_interleaved:
current_positions = mm_positions[
+88 -3
View File
@@ -52,7 +52,7 @@ from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.multimodal import (
MultiModalConfig,
MultiModalDummyOptions,
@@ -60,7 +60,7 @@ from vllm.config.multimodal import (
VideoPruningMethod,
)
from vllm.distributed import get_pp_group, parallel_state
from vllm.inputs import MultiModalDataDict
from vllm.inputs import MultiModalDataDict, MultiModalHashes
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
from vllm.model_executor.layers.attention.mm_encoder_attention import (
@@ -81,6 +81,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
@@ -94,6 +95,7 @@ from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
@@ -148,7 +150,7 @@ from .qwen2_vl import (
Qwen2VLProcessingInfo,
_create_qwen2vl_field_factory,
)
from .qwen3 import Qwen3ForCausalLM, Qwen3Model
from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
@@ -1342,6 +1344,67 @@ def _replace_video_token_placeholders(
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
def _get_per_video_kwargs(
self, kwargs: Mapping[str, object], num_videos: int | None = None
) -> dict[str, list[Any]]:
merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs)
per_video_kwargs = {
key: value
for key, value in merged.get("videos_kwargs", {}).items()
if key in ("fps", "num_frames") and isinstance(value, list)
}
if num_videos:
for key, values in per_video_kwargs.items():
if len(values) < num_videos:
raise ValueError(
f"{key} must have at least {num_videos} entries "
"(one per video); "
f"got {len(values)}."
)
return per_video_kwargs
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
hashes = dict(super()._get_mm_hashes(inputs))
per_video_kwargs = self._get_per_video_kwargs(
inputs.hf_processor_mm_kwargs, len(hashes.get("video", []))
)
if per_video_kwargs and "video" in hashes:
algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm
hashes["video"] = [
MultiModalHasher.hash_kwargs(
algorithm,
mm_hash=mm_hash,
**{key: values[i] for key, values in per_video_kwargs.items()},
)
for i, mm_hash in enumerate(hashes["video"])
]
return hashes
def _get_cache_missing_processor_kwargs(
self,
inputs: ProcessorInputs,
mm_is_cached: Mapping[str, list[bool]],
) -> Mapping[str, object]:
per_video_kwargs = self._get_per_video_kwargs(
inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", []))
)
if not per_video_kwargs or "video" not in mm_is_cached:
return inputs.hf_processor_mm_kwargs
kwargs = self.info._merge_and_resolve_mm_processor_kwargs(
inputs.hf_processor_mm_kwargs
)
kwargs["videos_kwargs"].update(
{
key: [
value
for value, hit in zip(values, mm_is_cached["video"])
if not hit
]
for key, values in per_video_kwargs.items()
}
)
return kwargs
@staticmethod
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
"""Transformers>=5.10 processors override `replace_video_token`
@@ -1412,6 +1475,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
video_input_ids_lst: list[list[int]] = []
if videos := hf_data.pop("videos", []):
assert isinstance(videos, list)
self._get_per_video_kwargs(hf_kwargs, len(videos))
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
@@ -1544,6 +1608,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
non_video_scoped_kwargs.pop("fps", None)
non_video_scoped_kwargs.pop("num_frames", None)
# The context merges model defaults again when calling HF.
for key in self._get_per_video_kwargs({}):
non_video_scoped_kwargs[key] = None
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
@@ -1748,6 +1815,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
return PromptUpdateDetails.from_seq(all_token_ids)
def mrope_positions_factor(vllm_config: VllmConfig) -> int:
# Video pruning keeps the positions of the tokens it drops. A drafter gets
# the target's positions, so it follows the target.
spec = vllm_config.speculative_config
model_config = (spec and spec.target_model_config) or vllm_config.model_config
mm_config = model_config.multimodal_config
pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled()
return 4 if pruned else 1
class Qwen3VLDecoderLayer(Qwen3DecoderLayer):
def __init__(self, **kwargs) -> None:
factor = mrope_positions_factor(get_current_vllm_config())
super().__init__(**kwargs, mrope_positions_factor=factor)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
@@ -1761,6 +1844,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
}
)
class Qwen3LLMModel(Qwen3Model):
decoder_layer_type = Qwen3VLDecoderLayer
def forward(
self,
input_ids: torch.Tensor | None,
@@ -56,6 +56,7 @@ from .qwen3_vl import (
Qwen3VLForConditionalGeneration,
Qwen3VLMultiModalProcessor,
Qwen3VLProcessingInfo,
mrope_positions_factor,
)
from .utils import StageMissingLayer, maybe_prefix
@@ -73,6 +74,7 @@ class Qwen3VLMoeDecoderLayer(Qwen3MoeDecoderLayer):
vllm_config=vllm_config,
prefix=prefix,
is_fused_checkpoint_transposed=True,
mrope_positions_factor=mrope_positions_factor(vllm_config),
)
+22 -7
View File
@@ -49,7 +49,7 @@ from .interfaces import (
is_attention_free,
is_hybrid,
requires_raw_input_tokens,
supports_mamba_prefix_caching,
supports_hc_sp,
supports_multimodal,
supports_multimodal_encoder_tp_data,
supports_multimodal_raw_input_only,
@@ -71,6 +71,7 @@ logger = init_logger(__name__)
_TEXT_GENERATION_MODELS = {
# [Decoder-only]
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
"AliceAIForCausalLM": ("alice_ai", "AliceAIForCausalLM"),
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
@@ -98,6 +99,7 @@ _TEXT_GENERATION_MODELS = {
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
@@ -285,6 +287,8 @@ _REWARD_MODELS = {
_TOKEN_CLASSIFICATION_MODELS = {
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
"ClefFlashForJointSchema": ("clef_flash", "ClefFlashForJointSchema"),
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
"ModernBertForTokenClassification": (
"modernbert",
"ModernBertForTokenClassification",
@@ -326,6 +330,7 @@ _SEQUENCE_CLASSIFICATION_MODELS = {
"RobertaForSequenceClassification",
),
# [Multimodal]
"D3ForSequenceClassification": ("d3", "D3ForSequenceClassification"),
"JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
"LlamaNemotronVLForSequenceClassification": (
"nemotron_vl",
@@ -513,6 +518,10 @@ _MULTIMODAL_MODELS = {
"minicpmv4_6",
"MiniCPMV4_6ForConditionalGeneration",
),
"MiniCPMV4_7ForConditionalGeneration": (
"minicpmv4_7",
"MiniCPMV4_7ForConditionalGeneration",
),
"Mistral3ForConditionalGeneration": (
"mistral3",
"Mistral3ForConditionalGeneration",
@@ -683,7 +692,6 @@ _SPECULATIVE_DECODING_MODELS = {
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
"GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
"Glm5NextMTPModel": ("vllm.models.glm5next", "Glm5NextMTP"),
"MedusaModel": ("medusa", "Medusa"),
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
@@ -694,9 +702,6 @@ _SPECULATIVE_DECODING_MODELS = {
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
# Temporarily disabled.
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
}
_TRANSFORMERS_SUPPORTED_MODELS = {
@@ -710,6 +715,7 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
@@ -868,11 +874,11 @@ class _ModelInfo:
requires_raw_input_tokens: bool
supports_multimodal_encoder_tp_data: bool
supports_pp: bool
supports_hc_sp: bool
has_inner_state: bool
is_attention_free: bool
is_hybrid: bool
has_noops: bool
supports_mamba_prefix_caching: bool
supports_replayssm: bool
supports_transcription: bool
supports_transcription_only: bool
@@ -898,10 +904,10 @@ class _ModelInfo:
model
),
supports_pp=supports_pp(model),
supports_hc_sp=supports_hc_sp(model),
has_inner_state=has_inner_state(model),
is_attention_free=is_attention_free(model),
is_hybrid=is_hybrid(model),
supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
supports_replayssm=supports_replayssm(model),
supports_transcription=supports_transcription(model),
supports_transcription_only=(
@@ -1458,6 +1464,15 @@ class _ModelRegistry:
model_info, _ = self.inspect_model_cls(architectures, model_config)
return model_info.supports_pp
def is_hc_sp_supported_model(
self,
architectures: str | list[str],
model_config: ModelConfig,
) -> bool:
"""Check support for HC SP without changing MoE parallelism."""
model_info, _ = self.inspect_model_cls(architectures, model_config)
return model_info.supports_hc_sp
def model_has_inner_state(
self,
architectures: str | list[str],
-9
View File
@@ -17,7 +17,6 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
@@ -459,7 +458,6 @@ class Step3p5DecoderLayer(nn.Module):
f"Unsupported attention implementation: {config.att_impl_type}"
)
self.use_moe = False
self.tp_group = get_tp_group()
self.use_fused_all_reduce = (
get_tensor_model_parallel_world_size() > 1
and get_dp_group().world_size == 1
@@ -496,13 +494,6 @@ class Step3p5DecoderLayer(nn.Module):
)
self.prefix = prefix
def add_and_maybe_inplace_all_reduce(
self, in1: torch.Tensor, in2: torch.Tensor
) -> torch.Tensor:
if not self.use_fused_all_reduce:
return in1 + in2
return self.tp_group.all_reduce(in1 + in2)
def forward(
self, positions: torch.Tensor, hidden_states: torch.Tensor
) -> torch.Tensor:
@@ -66,6 +66,7 @@ from vllm.model_executor.models.transformers.fusers.attention import (
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
disable_noop_dynamic_rope,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
@@ -182,6 +183,8 @@ class Base(
self._create_hf_to_vllm_mapper()
# Remove layers not on this pipeline parallel rank
self.pipeline_parallel()
# Skip the device sync of dynamic rope where it can never rescale
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation
@@ -282,6 +285,7 @@ class Base(
dynamic_arg_dims=dynamic_arg_dims,
enable_if=enable_if,
is_encoder=is_encoder,
use_aot_compile=False,
)(cls)
def _decorate_for_torch_compile(self):
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE fuser: route an HF MoE block through `FusedMoE` with vLLM's own routing."""
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
import ast
import functools
import inspect
import textwrap
import types
@@ -11,19 +12,25 @@ from dataclasses import dataclass
import torch
from torch import fx, nn
from typing_extensions import TypeIs
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
is_complete,
is_op,
output_value,
peel,
trace,
)
from vllm.model_executor.models.transformers.utils import named_state
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
logger = init_logger(__name__)
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
@@ -107,6 +114,53 @@ def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
return seen
def _renormalizes(topk: fx.Node) -> bool:
"""Are the top-k weights divided by their own sum?
Every HF router spells renormalization as a division by a sum over the
selection, optionally offset by an epsilon."""
downstream = _reaches(topk, "users")
for node in downstream:
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
continue
denominator = node.args[1]
if not isinstance(denominator, fx.Node):
continue
cone = _reaches(denominator, "all_input_nodes")
if any(is_op(n, "sum") and n in downstream for n in cone):
return True
return False
def _unwrap(node: object, moe_block: nn.Module) -> object:
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
while True:
node = peel(node)
if (
is_op(node, "reshape")
or is_op(node, "view")
or (
isinstance(node, fx.Node)
and node.op == "call_module"
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
)
):
node = node.args[0]
else:
return node
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
"""Is `node` a child module of `moe_block` called on just `x`?"""
return (
isinstance(node, fx.Node)
and node.op == "call_module"
and len(node.args) == 1
and not node.kwargs
and _unwrap(node.args[0], moe_block) is x
)
class SharedExpertMLP(nn.Module):
"""Wraps an HF shared expert, applying the output gating it is paired with."""
@@ -143,16 +197,23 @@ class MoEBlockFuser:
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
gate_name: str
scoring_func: str
scoring_func: str | None
"""`None` if routing stays in the Transformers gate."""
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
renormalize: bool | None = None
@staticmethod
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
def _match_router(
gate: nn.Module,
) -> tuple[str, torch.dtype | None, bool | None] | None:
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
`softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
Returns the scoring function, the dtype the router computes in, and
whether the top-k weights are renormalized (`None` if the trace was cut
short before it could say)."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
@@ -164,99 +225,176 @@ class MoEBlockFuser:
if not topks:
return None
topk = topks[-1]
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
for n in _reaches(topk, "all_input_nodes")
if is_op(n, "softmax") or is_op(n, "sigmoid")
]
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
post_topk = False
# Find all scoring functions that reach the top-k node.
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
if not scorers:
# Find all scoring functions that are users of the top-k node.
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
post_topk = True
if len(scorers) != 1:
return None
scorer = scorers[0]
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
return scoring_func, _forced_dtype(logits_cone)
is_softmax = is_op(scorer, "softmax")
# vLLM always scores every expert, so matching a router that softmaxes
# the selection alone takes renormalizing. Otherwise look for the
# division, reading its absence as "no" only from a complete trace.
renormalize: bool | None
if (post_topk and is_softmax) or _renormalizes(topk):
renormalize = True
else:
renormalize = False if is_complete(graph) else None
scoring_func = "softmax" if is_softmax else "sigmoid"
return scoring_func, _forced_dtype(logits_cone), renormalize
@staticmethod
def _match_experts(
graph: fx.Graph, experts_name: str, moe_block: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
being the block's input.
Returns the experts call, `x` and the gate call."""
experts = find_node(
graph, lambda n: n.op == "call_module" and n.target == experts_name
)
if experts is None:
return None
# Bind by signature so keyword calls match too. Fail closed on any other
# argument: the rewritten forward would drop it.
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
try:
bound = forward.bind(*experts.args, **experts.kwargs)
except TypeError:
return None
if len(bound.arguments) != 3:
return None
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
return None
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
return None
gate = ids.args[0]
if not (
_reads_input(gate, x, moe_block)
and ids.args == (gate, 2)
and weights.args == (gate, 1)
):
return None
return experts, x, gate
@staticmethod
def _match_shared_experts(
graph: fx.Graph, experts: str
) -> tuple[str | None, str | None]:
"""Detects the shared expert and its optional gate by dataflow."""
experts_predicate = lambda n: n.op == "call_module" and n.target == experts
if (experts_node := find_node(graph, experts_predicate)) is None:
return None, None
from_experts = _reaches(experts_node, "users")
for add in graph.nodes:
if not is_op(add, "add"):
node: object, x: fx.Node, moe_block: nn.Module
) -> tuple[str, str | None] | None:
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
Returns the shared expert's name and its optional gate's name."""
if _reads_input(node, x, moe_block):
return node.target, None
if not (is_op(node, "mul") and len(node.args) == 2):
return None
for gated, shared in (node.args, node.args[::-1]):
gated, shared = peel(gated), peel(shared)
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
continue
operands = [a for a in add.args if isinstance(a, fx.Node)]
# Exactly one side is the experts' output; the other is the shared path.
sides = [a in from_experts for a in operands]
if len(operands) != 2 or sides.count(True) != 1:
continue
cone = _reaches(operands[sides.index(False)], "all_input_nodes")
modules = [n for n in cone if n.op == "call_module" and n.target != experts]
# A sigmoid wrapping one of those modules marks the shared-expert gate.
gate = next(
(
src
for n in cone
if is_op(n, "sigmoid")
and isinstance(src := peel(n.args[0]), fx.Node)
and src in modules
),
None,
)
shared = [n for n in modules if n is not gate]
if len(shared) != 1:
return None, None
return shared[0].target, (gate.target if gate is not None else None)
return None, None
gate = peel(gated.args[0])
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
moe_block.get_submodule(gate.target)
):
return shared.target, gate.target
return None
@classmethod
def match(cls, moe_block: nn.Module, experts_name: str) -> "MoEBlockFuser | None":
def match(
cls,
moe_block: nn.Module,
experts_name: str,
divides_shared_output: bool = False,
) -> "MoEBlockFuser | None":
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
reshapes, casts and `nn.Identity`.
Args:
moe_block: The MoE block.
experts_name: The name of the experts in `moe_block`.
divides_shared_output: Whether vLLM routing would divide the shared
expert's output by `routed_scaling_factor` (fp16 overflow
protection), which Transformers decoder layers do not undo.
Returns:
A fuser if more than the experts can run in vLLM, else `None`.
"""
# Standard MoE block returns a single tensor.
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = router_dtype = None
for name, child in moe_block.named_children():
if (
name != experts_name
and (router := cls._match_router(child)) is not None
):
gate_name = name
scoring_func, router_dtype = router
break
if gate_name is None or scoring_func is None:
if (graph := trace(moe_block)) is None:
return None
# Shared expert: a child the block adds to the experts' output.
shared_name = shared_gate_name = None
others = [
n
for n, _ in moe_block.named_children()
if n not in {experts_name, gate_name}
]
if others:
graph = trace(moe_block)
if graph is None:
return None
shared_name, shared_gate_name = cls._match_shared_experts(
graph, experts_name
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
return None
experts, x, gate = matched
unwrap = functools.partial(_unwrap, moe_block=moe_block)
gate_name = gate.target
router = cls._match_router(getattr(moe_block, gate_name))
scoring_func, router_dtype, renormalize = router or (None, None, None)
def is_routed(n: object) -> bool:
return n is experts or (
is_op(n, "mul")
and not isinstance(n.args[1], fx.Node)
and unwrap(n.args[0]) is experts
)
if shared_gate_name is not None and not _is_scalar_gate(
getattr(moe_block, shared_gate_name)
):
# Output: the experts' output, plus the shared expert's if any.
routed = out = unwrap(output_value(graph))
shared = None
if not is_routed(out):
if not (is_op(out, "add") and len(out.args) == 2):
return None
routed, shared = map(unwrap, out.args)
if is_routed(shared):
routed, shared = shared, routed
if not is_routed(routed):
return None
shared_name = shared_gate_name = None
if shared is not None:
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
return None
shared_name, shared_gate_name = match
# Fail closed: `rewrite_forward` runs only the experts and the detected
# shared expert, so any other stateful child would be dropped.
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
if scoring_func is not None and shared_name and divides_shared_output:
logger.warning_once(
"%s could be fused but routing it in vLLM would apply "
"`routed_scaling_factor` by dividing the shared expert output, "
"which only fp16 overflow protection expects the decoder layer to "
"compensate for. Falling back to routing in Transformers; run in "
"bfloat16 to fuse it.",
type(moe_block).__name__,
)
scoring_func = router_dtype = renormalize = None
# Without vLLM routing, only a shared expert can move to vLLM, and only
# if the block does not scale the experts' output (vLLM routing does).
if scoring_func is None and (shared_name is None or routed is not experts):
return None
return cls(
gate_name,
scoring_func,
shared_name,
shared_gate_name,
router_dtype,
renormalize,
)
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
@@ -35,6 +35,21 @@ _MODULE_CALL = nn.Module.__call__
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
`call_module` nodes; meta execution must call modules for real."""
_MODULE_GETATTR = nn.Module.__getattr__
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
recurse until the stack runs out, truncating the graph."""
@contextlib.contextmanager
def _unpatched_modules():
"""Restores the module hooks fx patches, so meta execution records nothing."""
with (
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
):
yield
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
@@ -132,7 +147,8 @@ class _AllLeafTracer(fx.Tracer):
)
return _UNKNOWN
if kind == "get_attr":
value = operator.attrgetter(str(target))(self.root)
with _unpatched_modules():
value = operator.attrgetter(str(target))(self.root)
if isinstance(value, torch.Tensor):
value = torch.empty_like(value, device="meta")
return value
@@ -159,8 +175,6 @@ class _AllLeafTracer(fx.Tracer):
if kind == "call_module":
# Run the child's forward with all its state on "meta", without
# mutating it (at match time params may be meta but buffers real).
# fx patches `nn.Module.__call__` while tracing; restore the real
# one so this execution is not itself recorded.
child = self.root.get_submodule(str(target))
state = {
name: torch.empty_like(tensor, device="meta")
@@ -168,7 +182,7 @@ class _AllLeafTracer(fx.Tracer):
child.named_parameters(), child.named_buffers()
)
}
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
with _unpatched_modules():
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
return _UNKNOWN
@@ -686,6 +700,14 @@ def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
_Replacer().visit(module)
def is_complete(graph: fx.Graph) -> bool:
"""Did the trace run to completion? A partial graph has no `output` node.
Only a complete graph can be read for the *absence* of an op; in a partial
one the tail is exactly what is missing."""
return any(node.op == "output" for node in graph.nodes)
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
"""The first node in `graph` matching `predicate`, or `None`."""
return next((n for n in graph.nodes if predicate(n)), None)
+93 -71
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
@@ -126,6 +127,43 @@ direct_register_custom_op(
)
def _hf_topk_router_precomputed(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
weights arrive as `gating_output`, ids via `moe_state`."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
def _hf_topk_router(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
gate: nn.Module,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
_, topk_weights, topk_ids = gate(hidden_states)
return topk_weights, topk_ids
class TransformersRoutedExperts(RoutedExperts):
def get_expert_mapping(
self,
@@ -180,23 +218,18 @@ class MoEMixin(MixtureOfExperts, Base):
text_config = self.text_config
# Positional arguments
num_experts = self.model_config.get_num_experts()
top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None)
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(text_config, num_experts_names, None)
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
top_k = getattr_iter(text_config, top_k_names, None)
assert top_k is not None
hidden_size = text_config.hidden_size
intermediate_size = getattr_iter(
text_config, ["moe_intermediate_size", "intermediate_size"], None
)
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
assert intermediate_size is not None
num_shared_experts = getattr_iter(
text_config,
[
"n_shared_experts", # DeepSeek, Docs, GLM
"moe_num_shared_experts", # Aria, Ernie
],
0,
)
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
# Common kwargs
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
@@ -275,6 +308,9 @@ class MoEMixin(MixtureOfExperts, Base):
if "bias" in experts_param_name:
has_bias = True
break
# Whether the expert weights are stored in transposed format.
# Comes from Transformers `use_experts_implementation` decorator.
is_transposed = getattr(experts, "is_transposed", False)
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
@@ -308,28 +344,20 @@ class MoEMixin(MixtureOfExperts, Base):
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
is_fused_checkpoint_transposed=is_transposed,
routed_experts_cls=TransformersRoutedExperts,
)
fuser = MoEBlockFuser.match(moe_block, experts_name)
# _maybe_apply_routed_scale_to_output edge case. Transformers
# decoder layers do not compensate for dividing by scaling factor.
reaches_fp16_trick = (
divides_shared_output = (
routed_scaling_factor != 1.0
and apply_routed_scale_to_output
and self.model_config.dtype == torch.float16
and fuser is not None
and fuser.shared_name is not None
)
if reaches_fp16_trick:
logger.warning_once(
"%s could be fused but routing it in vLLM would apply "
"`routed_scaling_factor` by dividing the shared expert "
"output, which only fp16 overflow protection expects the "
"decoder layer to compensate for. Falling back to routing "
"in Transformers; run in bfloat16 to fuse it.",
moe_block_cls,
)
if fuser is not None and not reaches_fp16_trick:
fuser = MoEBlockFuser.match(
moe_block, experts_name, divides_shared_output
)
if fuser is not None:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in MoERunner.
shared_experts = fuser.shared_experts(moe_block, prefix)
@@ -341,34 +369,52 @@ class MoEMixin(MixtureOfExperts, Base):
down_name = glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
if scoring_func := fuser.scoring_func:
# Internal routing kwargs.
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=scoring_func,
renormalize=(
layer_renormalize
if fuser.renormalize is None
else fuser.renormalize
),
gate=gate,
)
if router_dtype is not None:
kwargs["router_logits_dtype"] = router_dtype
if use_grouped_topk:
kwargs |= grouped_topk_routing_kwargs
if routed_scaling_factor != 1.0:
kwargs |= routed_scaling_factor_kwargs
bias = getattr(gate, "e_score_correction_bias", None)
if bias is not None:
kwargs["e_score_correction_bias"] = bias
else:
# External routing kwargs
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
_hf_topk_router,
gate=getattr(moe_block, fuser.gate_name),
),
)
kwargs |= dict(
scoring_func=fuser.scoring_func,
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
gate=gate,
shared_experts=shared_experts,
)
if router_dtype is not None:
kwargs["router_logits_dtype"] = router_dtype
if use_grouped_topk:
kwargs |= grouped_topk_routing_kwargs
if routed_scaling_factor != 1.0:
kwargs |= routed_scaling_factor_kwargs
bias = getattr(gate, "e_score_correction_bias", None)
if bias is not None:
kwargs["e_score_correction_bias"] = bias
fuser.rewrite_forward(moe_block)
routed = "gate + experts"
if fuser.shared_name:
routed += " + shared experts"
logger.info_once(
"Fused: %s (%s) -> MoERunner (internal routing)",
routed,
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
" + shared experts" if fuser.shared_name else "",
moe_block_cls,
"external" if fuser.scoring_func is None else "internal",
)
else:
# MoE block forward is unmodified.
@@ -376,35 +422,11 @@ class MoEMixin(MixtureOfExperts, Base):
# We then smuggle the topk_ids in using a custom op.
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
custom_routing_function, moe_state=moe_state
_hf_topk_router_precomputed, moe_state=moe_state
),
runner_cls=TransformersMoERunner,
runner_args={"moe_state": moe_state},
@@ -357,13 +357,51 @@ def get_feature_request_tip(
return f"{prefix}{tip}"
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
if max_position_embeddings is None:
return False
return vllm_config.model_config.max_model_len <= max_position_embeddings
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
Defaults to `True` but is disabled in the following situations:
- The model uses dynamic rope scaling.
- The model uses dynamic rope scaling that can still rescale within
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
# Dynamic rope scaling is not compatible with torch.compile
return all(rp["rope_type"] != "dynamic" for rp in iter_rope_parameters(text_config))
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
if "dynamic" not in rope_types:
return True
# Layer types using longrope still sync, even once dynamic rope is disabled
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
The update syncs on `torch.max(position_ids)` every forward, which breaks
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
"default" skips it while keeping the `inv_freq` computed at init.
"""
if not dynamic_rope_update_is_noop(vllm_config):
return
for module in model.modules():
rope_type = getattr(module, "rope_type", None)
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
continue
if isinstance(rope_type, dict):
module.rope_type = {
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
}
elif rope_type == "dynamic":
module.rope_type = "default"
+9 -1
View File
@@ -322,7 +322,15 @@ class AutoWeightsLoader:
)
if param.dtype != weight_data.dtype:
logger.warning(
# An exact widening -- an fp32 accumulator reading a bf16
# table, say -- keeps every stored value, so it is a debug
# line. Every other mismatch still warns.
exact_widening = weight_data.dtype in (
torch.float16,
torch.bfloat16,
torch.float32,
) and param.dtype in (torch.float32, torch.float64)
(logger.debug if exact_widening else logger.warning)(
"Attempted to load weight %s with dtype %s into "
"parameter with dtype %s",
weight_qualname,
+2 -6
View File
@@ -45,7 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
)
from vllm.sequence import IntermediateTensors
from .interfaces import HasInnerState, IsHybrid, SupportsMambaPrefixCaching
from .interfaces import HasInnerState, IsHybrid
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
@@ -833,7 +833,7 @@ class Zamba2Model(nn.Module):
return hidden_states
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixCaching):
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid):
"""Zamba2 model with causal language modeling head.
This class wraps the core Zamba2 model and adds:
@@ -911,10 +911,6 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
LoRA and scheduler settings
prefix: Optional prefix for parameter names
Raises:
AssertionError: If prefix caching is enabled
(not supported by Mamba)
"""
config = vllm_config.model_config.hf_config