Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -116,7 +116,6 @@ class AXK1MoE(nn.Module):
|
||||
self.ep_size = self.ep_group.size()
|
||||
assert config.n_routed_experts is not None
|
||||
assert config.num_experts_per_tok is not None
|
||||
assert config.scoring_func is not None
|
||||
assert config.hidden_act is not None
|
||||
self.n_routed_experts: int = config.n_routed_experts
|
||||
self.n_shared_experts: int | None = config.n_shared_experts
|
||||
@@ -134,7 +133,7 @@ class AXK1MoE(nn.Module):
|
||||
config.n_routed_experts,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
if config.topk_method == "noaux_tc":
|
||||
if getattr(config, "topk_method", "noaux_tc") == "noaux_tc":
|
||||
self.gate.e_score_correction_bias = nn.Parameter(
|
||||
torch.empty(config.n_routed_experts, dtype=torch.float32)
|
||||
)
|
||||
@@ -186,7 +185,7 @@ class AXK1MoE(nn.Module):
|
||||
num_expert_group=config.n_group,
|
||||
topk_group=config.topk_group,
|
||||
prefix=f"{prefix}.experts",
|
||||
scoring_func=config.scoring_func,
|
||||
scoring_func=getattr(config, "scoring_func", "sigmoid"),
|
||||
# we do scaling outside, set factor to 1.0 to avoid double mul
|
||||
# aiter applies routed_scaling_factor internally
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
@@ -645,11 +644,10 @@ class AXK1DecoderLayer(nn.Module):
|
||||
self.routed_scaling_factor = config.routed_scaling_factor
|
||||
|
||||
def _is_layer_sparse(self) -> bool:
|
||||
assert self.config.moe_layer_freq is not None
|
||||
return (
|
||||
self.config.n_routed_experts is not None
|
||||
and self.layer_idx >= self.config.first_k_dense_replace
|
||||
and self.layer_idx % self.config.moe_layer_freq == 0
|
||||
and self.layer_idx % getattr(self.config, "moe_layer_freq", 1) == 0
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from .interfaces import (
|
||||
HasInnerState,
|
||||
SupportsHCSP,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -10,6 +11,7 @@ from .interfaces import (
|
||||
SupportsPP,
|
||||
SupportsTranscription,
|
||||
has_inner_state,
|
||||
supports_hc_sp,
|
||||
supports_lora,
|
||||
supports_mrope,
|
||||
supports_multimodal,
|
||||
@@ -33,6 +35,8 @@ __all__ = [
|
||||
"is_text_generation_model",
|
||||
"HasInnerState",
|
||||
"has_inner_state",
|
||||
"SupportsHCSP",
|
||||
"supports_hc_sp",
|
||||
"SupportsLoRA",
|
||||
"supports_lora",
|
||||
"SupportsMultiModal",
|
||||
|
||||
@@ -0,0 +1,733 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only AliceAI model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from copy import copy
|
||||
from itertools import islice
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedConfig, Qwen3NextConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, GateLinear
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
resolve_layer_fused_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||
KimiGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncCalculator,
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
get_quark_ocp_mx_group_size,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MixtureOfExperts,
|
||||
SupportsLoRA,
|
||||
)
|
||||
from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP
|
||||
from vllm.model_executor.models.qwen3_next import (
|
||||
Qwen3NextAttention,
|
||||
_should_replicate_misaligned_shared_expert,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
make_layers,
|
||||
maybe_fuse_shared_experts,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.offloader import (
|
||||
PrefetchOffloader,
|
||||
UVAOffloader,
|
||||
get_offloader,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.ops import attn_res
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
_SPLIT_KDA_CONV_SHARDS = {
|
||||
".linear_attn.q_conv1d": (".linear_attn.conv1d", 0),
|
||||
".linear_attn.k_conv1d": (".linear_attn.conv1d", 1),
|
||||
".linear_attn.v_conv1d": (".linear_attn.conv1d", 2),
|
||||
}
|
||||
|
||||
|
||||
def _load_split_kda_conv_weights(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
params: dict[str, nn.Parameter],
|
||||
loaded: set[str],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Load split AliceAI conv weights through the upstream fused-conv ABI."""
|
||||
for name, weight in weights:
|
||||
for source_name, (target_name, shard_id) in _SPLIT_KDA_CONV_SHARDS.items():
|
||||
if source_name not in name:
|
||||
continue
|
||||
mapped_name = name.replace(source_name, target_name, 1)
|
||||
param = params.get(mapped_name)
|
||||
if param is None:
|
||||
raise ValueError(f"No target parameter for {name!r}: {mapped_name!r}")
|
||||
weight_loader = getattr(param, "weight_loader", None)
|
||||
if weight_loader is None:
|
||||
raise ValueError(
|
||||
f"Target parameter {mapped_name!r} has no weight loader"
|
||||
)
|
||||
weight_loader(param, weight, shard_id)
|
||||
loaded.add(mapped_name)
|
||||
break
|
||||
else:
|
||||
yield name, weight
|
||||
|
||||
|
||||
def get_attn_res_block_size(hf_config: object) -> int:
|
||||
"""Return the architecture-rebase AttnRes grouping, not legacy metadata."""
|
||||
block_size = getattr(hf_config, "block_attn_res_block_size", None)
|
||||
if isinstance(block_size, bool) or not isinstance(block_size, int):
|
||||
raise ValueError(
|
||||
"AliceAI requires a positive integer block_attn_res_block_size"
|
||||
)
|
||||
if block_size <= 0:
|
||||
raise ValueError("block_attn_res_block_size must be positive")
|
||||
return block_size
|
||||
|
||||
|
||||
def _make_kimi_kda_config(config: PreTrainedConfig) -> KimiLinearConfig:
|
||||
"""Adapt the flat AliceAI KDA geometry to the existing upstream Kimi layer."""
|
||||
num_heads = config.linear_num_key_heads
|
||||
num_value_heads = config.linear_num_value_heads
|
||||
head_dim = config.linear_key_head_dim
|
||||
value_head_dim = config.linear_value_head_dim
|
||||
if num_heads != num_value_heads or head_dim != value_head_dim:
|
||||
raise ValueError(
|
||||
"AliceAI KDA requires matching key/value head counts and dimensions"
|
||||
)
|
||||
|
||||
adapted = copy(config)
|
||||
adapted.linear_attn_config = {
|
||||
"head_dim": head_dim,
|
||||
"num_heads": num_heads,
|
||||
"short_conv_kernel_size": config.linear_conv_kernel_dim,
|
||||
"use_full_rank_gate": False,
|
||||
"gate_lower_bound": None,
|
||||
}
|
||||
return cast(KimiLinearConfig, adapted)
|
||||
|
||||
|
||||
def _make_alice_ai_full_attention_config(
|
||||
config: PreTrainedConfig,
|
||||
) -> Qwen3NextConfig:
|
||||
adapted = copy(config)
|
||||
adapted.attn_output_gate = True
|
||||
adapted.qkv_bias = False
|
||||
adapted.dual_chunk_attention_config = None
|
||||
adapted.is_causal = True
|
||||
return cast(Qwen3NextConfig, adapted)
|
||||
|
||||
|
||||
class _AttentionResidualFinal(nn.Module):
|
||||
def __init__(self, hidden_size: int, rms_norm_eps: float, prefix: str) -> None:
|
||||
super().__init__()
|
||||
self.rms_norm_eps = rms_norm_eps
|
||||
self.res_proj = ReplicatedLinear(
|
||||
input_size=hidden_size,
|
||||
output_size=1,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.res_proj",
|
||||
)
|
||||
self.res_norm_weight = nn.Parameter(torch.ones(hidden_size))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
residual_bank: torch.Tensor,
|
||||
num_completed_sources: int,
|
||||
partial: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if partial is None:
|
||||
if num_completed_sources < 1:
|
||||
raise RuntimeError("AttnRes requires at least one residual source")
|
||||
prefix = residual_bank[:, num_completed_sources - 1]
|
||||
num_blocks = num_completed_sources - 1
|
||||
else:
|
||||
prefix = partial
|
||||
num_blocks = num_completed_sources
|
||||
return attn_res(
|
||||
prefix=prefix,
|
||||
delta=None,
|
||||
blocks=residual_bank,
|
||||
norm_weight=self.res_norm_weight,
|
||||
qk_weight=self.res_proj.weight.squeeze(0),
|
||||
output_norm_weight=None,
|
||||
num_blocks=num_blocks,
|
||||
block_write_idx=-1,
|
||||
eps=self.rms_norm_eps,
|
||||
output_norm_eps=0.0,
|
||||
)
|
||||
|
||||
|
||||
class AliceAISparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.ep_size = get_ep_group().device_group.size()
|
||||
self.n_routed_experts = config.num_experts
|
||||
if self.tp_size > self.n_routed_experts:
|
||||
raise ValueError(
|
||||
f"Tensor parallel size {self.tp_size} is greater than "
|
||||
f"the number of experts {self.n_routed_experts}."
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = False
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert(
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
shared_expert_name="shared_expert",
|
||||
)
|
||||
|
||||
self.replicate_shared_expert = False
|
||||
if not self.is_fused_shared_expert_enabled:
|
||||
shared_expert_group_size = get_quark_ocp_mx_group_size(
|
||||
quant_config=quant_config,
|
||||
layer_name=f"{prefix}.shared_expert.down_proj",
|
||||
)
|
||||
self.replicate_shared_expert = _should_replicate_misaligned_shared_expert(
|
||||
intermediate_size=config.shared_expert_intermediate_size,
|
||||
tp_size=self.tp_size,
|
||||
group_size=shared_expert_group_size,
|
||||
enable_expert_parallel=parallel_config.enable_expert_parallel,
|
||||
is_sequence_parallel=False,
|
||||
)
|
||||
|
||||
self.enable_eplb = parallel_config.enable_eplb
|
||||
self.n_logical_experts = self.n_routed_experts
|
||||
self.n_redundant_experts = parallel_config.eplb_config.num_redundant_experts
|
||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
|
||||
self.gate = GateLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=config.num_experts,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
self.gate.e_score_correction_bias = nn.Parameter(
|
||||
torch.empty(config.num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.shared_expert_gate = ReplicatedLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=1,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
if (
|
||||
self.is_fused_shared_expert_enabled
|
||||
or config.shared_expert_intermediate_size <= 0
|
||||
):
|
||||
self.shared_expert = None
|
||||
else:
|
||||
self.shared_expert = Qwen2MoeMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.shared_expert_intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
expert_gate=self.shared_expert_gate,
|
||||
disable_tp=self.replicate_shared_expert,
|
||||
prefix=f"{prefix}.shared_expert",
|
||||
)
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
shared_experts=(
|
||||
None if self.replicate_shared_expert else self.shared_expert
|
||||
),
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=getattr(config, "norm_topk_prob", True),
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_expert",
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
if self.shared_expert is None
|
||||
else None,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
|
||||
shared_output = (
|
||||
self.shared_expert(hidden_states)
|
||||
if self.replicate_shared_expert and self.shared_expert is not None
|
||||
else None
|
||||
)
|
||||
output = self.experts(hidden_states=hidden_states, router_logits=hidden_states)
|
||||
if shared_output is not None:
|
||||
output += shared_output
|
||||
return output.view(orig_shape)
|
||||
|
||||
|
||||
class AliceAIDecoderBlock(nn.Module):
|
||||
"""AliceAI topology with upstream Qwen, Kimi KDA, MoE and AttnRes operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
layer_type: str,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
if config.num_experts <= 0:
|
||||
raise ValueError("AliceAI requires sparse experts")
|
||||
if parallel_config.use_sequence_parallel_moe:
|
||||
raise NotImplementedError("AliceAI does not support sequence-parallel MoE")
|
||||
|
||||
self.layer_type = layer_type
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
self.attn_res_block_size = get_attn_res_block_size(config)
|
||||
self.attn_res_rms_norm_eps = float(config.rms_norm_eps)
|
||||
|
||||
if layer_type == "linear_attention":
|
||||
self.linear_attn = KimiGatedDeltaNetAttention(
|
||||
config=_make_kimi_kda_config(config),
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.linear_attn",
|
||||
)
|
||||
self.linear_attn.o_norm.eps = float(config.rms_norm_eps)
|
||||
elif layer_type == "full_attention":
|
||||
self.self_attn = Qwen3NextAttention(
|
||||
config=_make_alice_ai_full_attention_config(config),
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {layer_type}")
|
||||
|
||||
self.mlp = AliceAISparseMoeBlock(
|
||||
vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = GemmaRMSNorm(
|
||||
hidden_size=config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.post_attention_layernorm = GemmaRMSNorm(
|
||||
hidden_size=config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
if self.layer_idx > 0:
|
||||
self.attn_res_proj = ReplicatedLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=1,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.attn_res_proj",
|
||||
)
|
||||
self.attn_res_norm_weight = nn.Parameter(torch.ones(config.hidden_size))
|
||||
self.mlp_res_proj = ReplicatedLinear(
|
||||
input_size=config.hidden_size,
|
||||
output_size=1,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.mlp_res_proj",
|
||||
)
|
||||
self.mlp_res_norm_weight = nn.Parameter(torch.ones(config.hidden_size))
|
||||
|
||||
def _apply_attention_residual(
|
||||
self,
|
||||
residual_bank: torch.Tensor,
|
||||
num_completed_sources: int,
|
||||
partial: torch.Tensor | None,
|
||||
*,
|
||||
projection: ReplicatedLinear,
|
||||
norm_weight_name: str,
|
||||
) -> torch.Tensor:
|
||||
if partial is None:
|
||||
if num_completed_sources < 1:
|
||||
raise RuntimeError("AttnRes requires at least one residual source")
|
||||
prefix = residual_bank[:, num_completed_sources - 1]
|
||||
num_blocks = num_completed_sources - 1
|
||||
else:
|
||||
prefix = partial
|
||||
num_blocks = num_completed_sources
|
||||
return attn_res(
|
||||
prefix=prefix,
|
||||
delta=None,
|
||||
blocks=residual_bank,
|
||||
norm_weight=getattr(self, norm_weight_name),
|
||||
qk_weight=projection.weight.squeeze(0),
|
||||
output_norm_weight=None,
|
||||
num_blocks=num_blocks,
|
||||
block_write_idx=-1,
|
||||
eps=self.attn_res_rms_norm_eps,
|
||||
output_norm_eps=0.0,
|
||||
)
|
||||
|
||||
def _apply_attention(
|
||||
self, hidden_states: torch.Tensor, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if self.layer_type == "linear_attention":
|
||||
return self.linear_attn(hidden_states=hidden_states, positions=positions)
|
||||
return self.self_attn(hidden_states=hidden_states, positions=positions)
|
||||
|
||||
def forward_attn_res(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
residual_bank: torch.Tensor,
|
||||
num_completed_sources: int,
|
||||
partial: torch.Tensor | None,
|
||||
) -> tuple[int, torch.Tensor]:
|
||||
if self.layer_idx > 0 and self.layer_idx % self.attn_res_block_size == 0:
|
||||
if partial is None:
|
||||
raise RuntimeError("cannot finalize an empty AttnRes block")
|
||||
residual_bank[:, num_completed_sources].copy_(partial)
|
||||
num_completed_sources += 1
|
||||
partial = None
|
||||
|
||||
if self.layer_idx == 0:
|
||||
hidden_states = residual_bank[:, 0]
|
||||
else:
|
||||
hidden_states = self._apply_attention_residual(
|
||||
residual_bank=residual_bank,
|
||||
num_completed_sources=num_completed_sources,
|
||||
partial=partial,
|
||||
projection=self.attn_res_proj,
|
||||
norm_weight_name="attn_res_norm_weight",
|
||||
)
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
attention_output = self._apply_attention(
|
||||
hidden_states=hidden_states, positions=positions
|
||||
)
|
||||
partial = attention_output if partial is None else partial + attention_output
|
||||
|
||||
hidden_states = self._apply_attention_residual(
|
||||
residual_bank=residual_bank,
|
||||
num_completed_sources=num_completed_sources,
|
||||
partial=partial,
|
||||
projection=self.mlp_res_proj,
|
||||
norm_weight_name="mlp_res_norm_weight",
|
||||
)
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
mlp_output = self.mlp(hidden_states)
|
||||
return num_completed_sources, partial + mlp_output
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class AliceAIModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
".linear_attn.a_log_bias": ".linear_attn.A_log",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".linear_attn.q_proj": (".linear_attn.in_proj_qkvgfab", 0),
|
||||
".linear_attn.k_proj": (".linear_attn.in_proj_qkvgfab", 1),
|
||||
".linear_attn.v_proj": (".linear_attn.in_proj_qkvgfab", 2),
|
||||
".linear_attn.b_proj": (".linear_attn.in_proj_qkvgfab", 3),
|
||||
".linear_attn.f_a_proj": (".linear_attn.in_proj_qkvgfab", 4),
|
||||
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
|
||||
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
|
||||
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.config = config
|
||||
self.vocab_size = config.vocab_size
|
||||
self.num_redundant_experts = (
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
self.attn_res_block_size = get_attn_res_block_size(config)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
|
||||
)
|
||||
|
||||
def get_layer(prefix: str) -> AliceAIDecoderBlock:
|
||||
return AliceAIDecoderBlock(
|
||||
vllm_config=vllm_config,
|
||||
layer_type=config.layer_types[extract_layer_index(prefix)],
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
num_hidden_layers=config.num_hidden_layers,
|
||||
layer_fn=get_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
layers=self.layers, moe_cls=AliceAISparseMoeBlock, moe_name="mlp"
|
||||
)
|
||||
self.norm = GemmaRMSNorm(
|
||||
hidden_size=config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
self.attnres_final = _AttentionResidualFinal(
|
||||
hidden_size=config.hidden_size,
|
||||
rms_norm_eps=float(config.rms_norm_eps),
|
||||
prefix=f"{prefix}.attnres_final",
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def _init_residual_bank(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
residual_bank = hidden_states.new_empty(
|
||||
(
|
||||
hidden_states.shape[0],
|
||||
(self.config.num_hidden_layers + self.attn_res_block_size - 1)
|
||||
// self.attn_res_block_size,
|
||||
hidden_states.shape[-1],
|
||||
)
|
||||
)
|
||||
residual_bank[:, 0].copy_(hidden_states)
|
||||
return residual_bank
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
raise NotImplementedError("AliceAI does not support pipeline parallelism")
|
||||
if not get_pp_group().is_first_rank or not get_pp_group().is_last_rank:
|
||||
raise NotImplementedError("AliceAI requires pipeline_parallel_size=1")
|
||||
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
assert input_ids is not None
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual_bank = self._init_residual_bank(hidden_states)
|
||||
|
||||
num_completed_sources = 1
|
||||
partial = None
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
num_completed_sources, partial = layer.forward_attn_res(
|
||||
positions=positions,
|
||||
residual_bank=residual_bank,
|
||||
num_completed_sources=num_completed_sources,
|
||||
partial=partial,
|
||||
)
|
||||
if partial is None:
|
||||
raise RuntimeError("AttnRes produced an empty final block")
|
||||
hidden_states = self.attnres_final(
|
||||
residual_bank=residual_bank,
|
||||
num_completed_sources=num_completed_sources,
|
||||
partial=partial,
|
||||
)
|
||||
return self.norm(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = maybe_fuse_shared_experts(
|
||||
weights=weights,
|
||||
enabled=self.is_fused_shared_expert_enabled,
|
||||
n_routed_experts=self.config.num_experts,
|
||||
n_shared_experts=1,
|
||||
ckpt_prefix="mlp.shared_expert",
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights=weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class AliceAIForCausalLM(
|
||||
nn.Module, HasInnerState, IsHybrid, SupportsLoRA, MixtureOfExperts
|
||||
):
|
||||
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"mtp.": None})
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
"in_proj_qkvgfab": [
|
||||
"q_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
"b_proj",
|
||||
"f_a_proj",
|
||||
],
|
||||
"conv1d": ["q_conv1d", "k_conv1d", "v_conv1d"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
# AttnRes calls bypass the per-layer forward hooks used by offloading.
|
||||
offloader = get_offloader()
|
||||
if isinstance(offloader, PrefetchOffloader):
|
||||
raise NotImplementedError("AliceAI does not support prefetch offloading")
|
||||
if (
|
||||
isinstance(offloader, UVAOffloader)
|
||||
and offloader.cpu_offload_max_bytes > 0
|
||||
and not offloader.uva_offloading
|
||||
):
|
||||
raise NotImplementedError("AliceAI CPU offloading requires UVA")
|
||||
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.model = AliceAIModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
num_embeddings=config.vocab_size,
|
||||
embedding_dim=config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.set_moe_parameters()
|
||||
|
||||
def set_moe_parameters(self) -> None:
|
||||
self.moe_layers = [layer.mlp.experts for layer in self.model.layers]
|
||||
moe = self.model.layers[self.model.start_layer].mlp
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
self.num_expert_groups = 1
|
||||
self.num_shared_experts = 0
|
||||
self.num_logical_experts = moe.n_logical_experts
|
||||
self.num_physical_experts = moe.n_physical_experts
|
||||
self.num_local_physical_experts = moe.n_local_physical_experts
|
||||
self.num_routed_experts = moe.n_routed_experts
|
||||
self.num_redundant_experts = moe.n_redundant_experts
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
num_local_physical_experts: int,
|
||||
) -> None:
|
||||
assert self.num_local_physical_experts == num_local_physical_experts
|
||||
self.num_physical_experts = num_physical_experts
|
||||
self.num_local_physical_experts = num_local_physical_experts
|
||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||
for layer in self.model.layers:
|
||||
moe = layer.mlp
|
||||
moe.n_local_physical_experts = num_local_physical_experts
|
||||
moe.n_physical_experts = num_physical_experts
|
||||
moe.n_redundant_experts = self.num_redundant_experts
|
||||
moe.experts.update_expert_map()
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
return self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
|
||||
return self.logits_processor(lm_head=self.lm_head, hidden_states=hidden_states)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(
|
||||
cls, vllm_config: VllmConfig
|
||||
) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.kda_state_dtype(
|
||||
model_dtype=vllm_config.model_config.dtype,
|
||||
mamba_cache_dtype=vllm_config.cache_config.mamba_cache_dtype,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(
|
||||
cls, vllm_config: VllmConfig
|
||||
) -> tuple[tuple[int, int], tuple[int, int, int]]:
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
num_spec = (
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
if vllm_config.speculative_config
|
||||
else 0
|
||||
)
|
||||
return MambaStateShapeCalculator.kda_state_shape(
|
||||
tp_world_size=vllm_config.parallel_config.tensor_parallel_size,
|
||||
num_heads=config.linear_num_key_heads,
|
||||
head_dim=config.linear_key_head_dim,
|
||||
conv_kernel_size=config.linear_conv_kernel_dim,
|
||||
num_spec=num_spec,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(
|
||||
cls,
|
||||
) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
|
||||
return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
weights = self.hf_to_vllm_mapper.apply(weights)
|
||||
manually_loaded: set[str] = set()
|
||||
weights = _load_split_kda_conv_weights(
|
||||
weights=weights,
|
||||
params=dict(self.named_parameters()),
|
||||
loaded=manually_loaded,
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded = loader.load_weights(weights=weights)
|
||||
loaded.update(manually_loaded)
|
||||
expected_target_weights = {
|
||||
name
|
||||
for name, _ in self.named_parameters()
|
||||
if ".attnres_final." in name
|
||||
or ".attn_res_" in name
|
||||
or ".mlp_res_" in name
|
||||
or name.endswith(".e_score_correction_bias")
|
||||
}
|
||||
if missing := expected_target_weights - loaded:
|
||||
raise ValueError(
|
||||
"AliceAI checkpoint is missing target-specific weights: "
|
||||
+ ", ".join(sorted(missing))
|
||||
)
|
||||
return loaded
|
||||
@@ -20,7 +20,13 @@ from einops import rearrange
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -230,6 +236,12 @@ def _configure_ling_fp8_quant_config(
|
||||
|
||||
quant_config.ignored_layers_match_mode = "suffix"
|
||||
hf_quant_config = getattr(config, "quantization_config", None)
|
||||
if hf_quant_config is None:
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
hf_quant_config = getattr(
|
||||
getattr(model_config, "hf_config", None), "quantization_config", None
|
||||
)
|
||||
if not isinstance(hf_quant_config, dict):
|
||||
return
|
||||
|
||||
@@ -526,6 +538,26 @@ class BailingMoeV3MLAAttention(nn.Module):
|
||||
g_out = self.num_heads * self.v_head_dim
|
||||
else:
|
||||
g_out = 0
|
||||
block_size = (
|
||||
quant_config.weight_block_size
|
||||
if _is_block_fp8_config(quant_config)
|
||||
else None
|
||||
)
|
||||
self.replicate_g_proj = (
|
||||
self.gated_attention_proj_granularity_type == "head_wise"
|
||||
and tp_size > 1
|
||||
and block_size is not None
|
||||
and g_out <= block_size[0]
|
||||
and not _is_fp8_module_excluded(quant_config, f"{prefix}.g_proj")
|
||||
and self.num_local_heads % block_size[0] != 0
|
||||
)
|
||||
# A gate spanning at most one FP8 row block cannot split its scales
|
||||
# across TP ranks. Replicate it and select each rank's heads instead.
|
||||
self.g_proj_start = (
|
||||
get_tensor_model_parallel_rank() * self.num_local_heads
|
||||
if self.replicate_g_proj
|
||||
else 0
|
||||
)
|
||||
self.g_proj = (
|
||||
ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
@@ -533,6 +565,7 @@ class BailingMoeV3MLAAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.g_proj",
|
||||
disable_tp=self.replicate_g_proj,
|
||||
)
|
||||
if g_out > 0
|
||||
else None
|
||||
@@ -581,9 +614,12 @@ class BailingMoeV3MLAAttention(nn.Module):
|
||||
def forward(self, hidden_states: torch.Tensor, positions: torch.Tensor):
|
||||
attn_out = self.mla_attn(positions, hidden_states)
|
||||
if self.g_proj is not None:
|
||||
gate = torch.sigmoid(self.g_proj(hidden_states)[0].float()).to(
|
||||
hidden_states.dtype
|
||||
)
|
||||
gate = self.g_proj(hidden_states)[0]
|
||||
if self.replicate_g_proj:
|
||||
gate = gate[
|
||||
..., self.g_proj_start : self.g_proj_start + self.num_local_heads
|
||||
]
|
||||
gate = torch.sigmoid(gate.float()).to(hidden_states.dtype)
|
||||
if self.gated_attention_proj_granularity_type == "head_wise":
|
||||
attn_out = attn_out.view(-1, self.num_local_heads, self.v_head_dim)
|
||||
attn_out = attn_out * gate.unsqueeze(-1)
|
||||
|
||||
@@ -41,7 +41,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces import SupportsCrossEncoding, SupportsQuant
|
||||
from .interfaces import SupportsCrossEncoding, SupportsLoRA, SupportsQuant
|
||||
from .interfaces_base import attn_type, default_pooling_type
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
@@ -448,7 +448,7 @@ class BertPoolingModel(BertModel):
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS")
|
||||
class BertEmbeddingModel(nn.Module, SupportsQuant):
|
||||
class BertEmbeddingModel(nn.Module, SupportsQuant, SupportsLoRA):
|
||||
"""A model that uses Bert to provide embedding functionalities.
|
||||
|
||||
This class encapsulates the BertModel and provides an interface for
|
||||
@@ -463,9 +463,22 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
|
||||
is_pooling_model = True
|
||||
embedding_class: type[nn.Module] = BertEmbedding
|
||||
|
||||
packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
|
||||
# PEFT adapters name modules relative to the HF backbone (optionally under
|
||||
# `bert.`/`roberta.`), while vLLM nests the backbone under `model.`.
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"bert.": "model.",
|
||||
"roberta.": "model.",
|
||||
"encoder.": "model.encoder.",
|
||||
"embeddings.": "model.embeddings.",
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
@@ -564,9 +577,6 @@ class BertMLMHead(nn.Module):
|
||||
self.layer_norm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
|
||||
self.decoder = nn.Linear(hidden_size, vocab_size, bias=True)
|
||||
|
||||
def tie_weights_with_embeddings(self, embeddings_weight: torch.Tensor):
|
||||
self.decoder.weight = embeddings_weight
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
x = self.dense(hidden_states)
|
||||
x = self.activation(x)
|
||||
|
||||
@@ -129,13 +129,6 @@ class BlipAttention(nn.Module):
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
||||
return (
|
||||
tensor.view(bsz, seq_len, self.num_heads, self.head_dim)
|
||||
.transpose(1, 2)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Clef-Flash: the Qwen3.5 backbone with a joint schema decision head."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .clef_joint_schema import JOINT_HEAD_WEIGHTS, JointSchemaHead, JointSchemaPooler
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
|
||||
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
|
||||
|
||||
|
||||
@default_pooling_type(tok_pooling_type="ALL")
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Qwen3VLMultiModalProcessor,
|
||||
info=Qwen3_5ProcessingInfo,
|
||||
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
||||
)
|
||||
class ClefFlashForJointSchema(Qwen3_5ForConditionalGeneration):
|
||||
is_pooling_model = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
model_config = vllm_config.model_config
|
||||
self.pooler = JointSchemaPooler(
|
||||
JointSchemaHead(**model_config.hf_config.joint_head_config),
|
||||
self._embed_lexical,
|
||||
)
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=model_config.model,
|
||||
revision=model_config.revision,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
|
||||
)
|
||||
]
|
||||
|
||||
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
|
||||
@@ -0,0 +1,343 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared joint schema head for Clef decision models.
|
||||
|
||||
The head reads the final hidden states of the whole prompt, so it runs as a
|
||||
``token_classify`` pooler.
|
||||
|
||||
Each request carries its schema in ``PoolingParams.extra_kwargs`` under
|
||||
``"joint_schema"``:
|
||||
|
||||
- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
|
||||
- ``question_spans``: the ``[start, end)`` of each question's instructions.
|
||||
- ``option_spans``: the ``[start, end)`` of each option, question by question.
|
||||
- ``option_counts``: the number of options of each question.
|
||||
|
||||
Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
|
||||
so they hold after multimodal placeholders are expanded. The output has one
|
||||
value per option, in ``option_spans`` order: its probability within its
|
||||
question, or its logit with ``use_activation=False``.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Set
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
|
||||
NUM_QUESTION_TYPES = 3
|
||||
MAX_LOGIT_SCALE = math.log(100.0)
|
||||
|
||||
|
||||
def segment_mean(
|
||||
values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
|
||||
sums.index_add_(0, segment_ids, values.float())
|
||||
return (sums / lengths[:, None]).to(values.dtype)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaIndices:
|
||||
"""Where one prompt's questions and options are, on the model's device."""
|
||||
|
||||
positions: torch.Tensor
|
||||
"""Every token of every span, questions first."""
|
||||
span_ids: torch.Tensor
|
||||
"""The span of each of ``positions``."""
|
||||
span_lengths: torch.Tensor
|
||||
"""``[Q + O]``."""
|
||||
question_types: torch.Tensor
|
||||
"""``[Q]``."""
|
||||
option_question: torch.Tensor
|
||||
"""The question of each option, ``[O]``."""
|
||||
option_slots: torch.Tensor
|
||||
"""Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
|
||||
option_mask: torch.Tensor
|
||||
"""``[Q, max_options]``, true at the slots of options."""
|
||||
|
||||
|
||||
class EvidenceRoutingLayer(nn.Module):
|
||||
def __init__(self, width: int, heads: int, feedforward: int) -> None:
|
||||
super().__init__()
|
||||
self.query_norm = nn.LayerNorm(width)
|
||||
self.memory_norm = nn.LayerNorm(width)
|
||||
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
||||
self.feedforward_norm = nn.LayerNorm(width)
|
||||
# The Identity takes the place of the checkpoint's dropout.
|
||||
self.feedforward = nn.Sequential(
|
||||
nn.Linear(width, feedforward),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(feedforward, width),
|
||||
)
|
||||
|
||||
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
|
||||
# Separate key and value tensors, like the reference, keep attention's
|
||||
# unpacked input projections.
|
||||
routed, _ = self.attention(
|
||||
self.query_norm(queries),
|
||||
self.memory_norm(memory),
|
||||
self.memory_norm(memory),
|
||||
need_weights=False,
|
||||
)
|
||||
queries = queries + routed
|
||||
return queries + self.feedforward(self.feedforward_norm(queries))
|
||||
|
||||
|
||||
class JointSchemaHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
width: int,
|
||||
routing_layers: int,
|
||||
layers: int,
|
||||
heads: int,
|
||||
feedforward: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = nn.LayerNorm(hidden_size)
|
||||
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
|
||||
self.evidence_layers = nn.ModuleList(
|
||||
EvidenceRoutingLayer(width, heads, feedforward)
|
||||
for _ in range(routing_layers)
|
||||
)
|
||||
self.option_summary_norm = nn.LayerNorm(width)
|
||||
self.layers = nn.ModuleList(
|
||||
nn.TransformerDecoderLayer(
|
||||
d_model=width,
|
||||
nhead=heads,
|
||||
dim_feedforward=feedforward,
|
||||
dropout=0.0,
|
||||
activation="gelu",
|
||||
batch_first=True,
|
||||
norm_first=True,
|
||||
)
|
||||
for _ in range(layers)
|
||||
)
|
||||
self.field_norm = nn.LayerNorm(width)
|
||||
self.option_norm = nn.LayerNorm(width)
|
||||
self.residual_scorer = nn.Sequential(
|
||||
nn.Linear(width * 4, width),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(width, 1),
|
||||
)
|
||||
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.residual_gate = nn.Parameter(torch.zeros(()))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
lexical: torch.Tensor,
|
||||
schema: SchemaIndices,
|
||||
) -> torch.Tensor:
|
||||
"""The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
|
||||
final hidden states and each option's ``[O, hidden]`` mean output
|
||||
embedding."""
|
||||
num_questions = schema.question_types.shape[0]
|
||||
option_question = schema.option_question
|
||||
hidden_states = self.hidden_norm(hidden_states)
|
||||
memory = self.memory_projection(hidden_states).unsqueeze(0)
|
||||
global_vector = hidden_states[-1]
|
||||
means = segment_mean(
|
||||
hidden_states[schema.positions], schema.span_ids, schema.span_lengths
|
||||
)
|
||||
question_vectors, contexts = means[:num_questions], means[num_questions:]
|
||||
|
||||
options = (
|
||||
self.option_context_projection(contexts)
|
||||
+ self.option_lexical_projection(lexical)
|
||||
+ self.option_question_projection(question_vectors)[option_question]
|
||||
).unsqueeze(0)
|
||||
for layer in self.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
options = options[0]
|
||||
|
||||
# Each field summarizes its options, weighted by their match to it.
|
||||
base_fields = self.question_projection(question_vectors)
|
||||
width = options.shape[-1]
|
||||
grid = options.new_zeros(schema.option_mask.numel(), width)
|
||||
grid[schema.option_slots] = options
|
||||
grid = grid.view(*schema.option_mask.shape, width)
|
||||
routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
|
||||
routing = routing.masked_fill(~schema.option_mask, float("-inf"))
|
||||
summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
|
||||
fields = (
|
||||
base_fields
|
||||
+ self.option_summary_norm(summaries)
|
||||
+ self.global_projection(global_vector).unsqueeze(0)
|
||||
+ self.type_embedding(schema.question_types)
|
||||
).unsqueeze(0)
|
||||
for layer in self.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = self.field_norm(fields[0])[option_question]
|
||||
|
||||
anchors = F.normalize(question_vectors + global_vector, dim=-1)
|
||||
prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
prior = prior_scale * (
|
||||
F.normalize(lexical, dim=-1) * anchors[option_question]
|
||||
).sum(dim=-1)
|
||||
options = self.option_norm(options)
|
||||
cosine = F.cosine_similarity(fields, options, dim=-1)
|
||||
features = torch.cat(
|
||||
[fields, options, fields * options, torch.abs(fields - options)], dim=-1
|
||||
)
|
||||
residual = self.residual_scorer(features).squeeze(-1)
|
||||
joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
joint = joint_scale * cosine + residual
|
||||
return prior + torch.sigmoid(self.residual_gate) * joint
|
||||
|
||||
|
||||
def parse_joint_schema(
|
||||
extra_kwargs: dict[str, Any] | None, prompt_len: int
|
||||
) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
|
||||
"""The question types, the absolute spans of the questions then of the
|
||||
options, and the option counts. None if the schema is missing or invalid."""
|
||||
schema = (extra_kwargs or {}).get("joint_schema")
|
||||
if not isinstance(schema, dict):
|
||||
return None
|
||||
try:
|
||||
types = [int(t) for t in schema["question_types"]]
|
||||
counts = [int(c) for c in schema["option_counts"]]
|
||||
question_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["question_spans"]
|
||||
]
|
||||
option_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["option_spans"]
|
||||
]
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
spans = question_spans + option_spans
|
||||
if (
|
||||
not types
|
||||
or len(counts) != len(types)
|
||||
or len(question_spans) != len(types)
|
||||
or len(option_spans) != sum(counts)
|
||||
or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
|
||||
or any(c < 1 for c in counts)
|
||||
or any(not 0 <= start < end <= prompt_len for start, end in spans)
|
||||
):
|
||||
return None
|
||||
return types, spans, counts
|
||||
|
||||
|
||||
class JointSchemaPooler(Pooler):
|
||||
"""Runs the joint schema head on each finished prompt.
|
||||
|
||||
It never raises, because it runs in the engine core: a prompt without a
|
||||
valid schema, such as a profiling dummy, gets an empty output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head: JointSchemaHead,
|
||||
embed_lexical: Callable[[torch.Tensor], torch.Tensor],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pooling = AllPool()
|
||||
self.head = head
|
||||
self.embed_lexical = embed_lexical
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled = self.pooling(hidden_states, pooling_metadata)
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
outputs: list[torch.Tensor | None] = []
|
||||
for hidden, token_ids, params in zip(
|
||||
pooled, prompt_token_ids, pooling_metadata.pooling_params
|
||||
):
|
||||
if hidden is None:
|
||||
outputs.append(None)
|
||||
continue
|
||||
parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
|
||||
if parsed is None:
|
||||
outputs.append(hidden.new_empty(0, dtype=torch.float32))
|
||||
continue
|
||||
schema, lexical_ids, lexical_span_ids = self._indices(
|
||||
*parsed, hidden.device
|
||||
)
|
||||
lexical = segment_mean(
|
||||
self.embed_lexical(
|
||||
async_tensor_h2d(token_ids[lexical_ids], hidden.device)
|
||||
),
|
||||
lexical_span_ids,
|
||||
schema.span_lengths[len(parsed[0]) :],
|
||||
)
|
||||
logits = self.head(hidden, lexical, schema).float()
|
||||
if params.use_activation is not False:
|
||||
mask = schema.option_mask
|
||||
grid = logits.new_full((mask.numel(),), float("-inf"))
|
||||
grid[schema.option_slots] = logits
|
||||
probs = grid.view(mask.shape).softmax(dim=-1)
|
||||
logits = probs.view(-1)[schema.option_slots]
|
||||
outputs.append(logits)
|
||||
return outputs
|
||||
|
||||
@staticmethod
|
||||
def _indices(
|
||||
types: list[int],
|
||||
spans: list[tuple[int, int]],
|
||||
counts: list[int],
|
||||
device: torch.device,
|
||||
) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
|
||||
"""The schema's indices, plus the option tokens' positions (on the CPU)
|
||||
and their option spans, for the lexical embeddings."""
|
||||
num_questions = len(types)
|
||||
lengths = torch.tensor([end - start for start, end in spans])
|
||||
span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
|
||||
positions = torch.cat([torch.arange(start, end) for start, end in spans])
|
||||
num_question_tokens = int(lengths[:num_questions].sum())
|
||||
|
||||
counts_cpu = torch.tensor(counts)
|
||||
max_options = max(counts)
|
||||
option_question = torch.repeat_interleave(
|
||||
torch.arange(num_questions), counts_cpu
|
||||
)
|
||||
option_slots = torch.cat(
|
||||
[torch.arange(count) + q * max_options for q, count in enumerate(counts)]
|
||||
)
|
||||
schema = SchemaIndices(
|
||||
positions=async_tensor_h2d(positions, device),
|
||||
span_ids=async_tensor_h2d(span_ids, device),
|
||||
span_lengths=async_tensor_h2d(lengths, device),
|
||||
question_types=async_tensor_h2d(types, device, dtype=torch.long),
|
||||
option_question=async_tensor_h2d(option_question, device),
|
||||
option_slots=async_tensor_h2d(option_slots, device),
|
||||
option_mask=async_tensor_h2d(
|
||||
torch.arange(max_options) < counts_cpu[:, None], device
|
||||
),
|
||||
)
|
||||
lexical_span_ids = async_tensor_h2d(
|
||||
span_ids[num_question_tokens:] - num_questions, device
|
||||
)
|
||||
return schema, positions[num_question_tokens:], lexical_span_ids
|
||||
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The Qwen3-Omni thinker with a Clef joint schema head."""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .clef_joint_schema import JOINT_HEAD_WEIGHTS, JointSchemaHead, JointSchemaPooler
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerDummyInputsBuilder,
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
Qwen3OmniMoeThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
MAX_FRAME_PIXELS = 256 * 32 * 32
|
||||
|
||||
|
||||
def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
|
||||
"""The reference reads each video frame at at most 256 * 32 * 32 pixels,
|
||||
in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on
|
||||
their device (a CUDA tensor from torchcodec must not round-trip)."""
|
||||
height, width = frames.shape[1:3]
|
||||
if height * width <= MAX_FRAME_PIXELS:
|
||||
return frames
|
||||
scale = math.sqrt(MAX_FRAME_PIXELS / (width * height))
|
||||
new_width = max(2, round(width * scale / 2) * 2)
|
||||
new_height = max(2, round(height * scale / 2) * 2)
|
||||
is_tensor = isinstance(frames, torch.Tensor)
|
||||
tensor = frames if is_tensor else torch.from_numpy(frames)
|
||||
resized = F.interpolate(
|
||||
tensor.permute(0, 3, 1, 2).float(),
|
||||
size=(new_height, new_width),
|
||||
mode="bilinear",
|
||||
)
|
||||
if tensor.dtype == torch.uint8:
|
||||
resized = resized.round()
|
||||
resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous()
|
||||
return resized if is_tensor else resized.numpy()
|
||||
|
||||
|
||||
def _cap_video(item: Any) -> Any:
|
||||
if isinstance(item, tuple):
|
||||
frames, metadata = item
|
||||
return cap_frame_pixels(frames), metadata
|
||||
return cap_frame_pixels(item)
|
||||
|
||||
|
||||
class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor):
|
||||
"""Caps video frames at the reference's per-frame pixel count."""
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if videos := hf_data.get("videos"):
|
||||
assert isinstance(videos, list)
|
||||
hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]}
|
||||
return super()._call_hf_processor(hf_data, hf_kwargs)
|
||||
|
||||
|
||||
@default_pooling_type(tok_pooling_type="ALL")
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
ClefOmniMultiModalProcessor,
|
||||
info=Qwen3OmniMoeThinkerProcessingInfo,
|
||||
dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
|
||||
)
|
||||
class ClefOmniForJointSchema(Qwen3OmniMoeThinkerForConditionalGeneration):
|
||||
is_pooling_model = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
model_config = vllm_config.model_config
|
||||
head_config = model_config.hf_config.joint_head_config
|
||||
self.pooler = JointSchemaPooler(
|
||||
JointSchemaHead(**head_config), self._embed_lexical
|
||||
)
|
||||
# The head's weights ship as their own safetensors file.
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=model_config.model,
|
||||
revision=model_config.revision,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
|
||||
)
|
||||
]
|
||||
|
||||
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
# The head embeds options with the output embeddings. ParallelLMHead
|
||||
# forbids lookups, so use the vocab-parallel embedding's.
|
||||
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
|
||||
@@ -25,6 +25,51 @@ class VerifyAndUpdateConfig:
|
||||
return
|
||||
|
||||
|
||||
class AliceAIForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
hf_config = model_config.hf_config
|
||||
if getattr(hf_config, "kda_allow_negative_eigenvalues", None) is not False:
|
||||
raise NotImplementedError(
|
||||
"AliceAI requires kda_allow_negative_eigenvalues=false"
|
||||
)
|
||||
if getattr(hf_config, "router_score_function", None) != "sigmoid":
|
||||
raise ValueError("AliceAI requires router_score_function='sigmoid'")
|
||||
if getattr(hf_config, "router_bias_correction", None) is not True:
|
||||
raise ValueError("AliceAI requires router_bias_correction=true")
|
||||
mtp_layers = getattr(hf_config, "mtp_num_hidden_layers", 0)
|
||||
if mtp_layers:
|
||||
logger.warning(
|
||||
"AliceAI checkpoint declares %s MTP layer(s), but this "
|
||||
"implementation ignores the MTP configuration and weights; "
|
||||
"MTP speculative decoding is unavailable.",
|
||||
mtp_layers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
if vllm_config.parallel_config.pipeline_parallel_size != 1:
|
||||
raise NotImplementedError(
|
||||
"AliceAI does not support pipeline parallelism; "
|
||||
"pipeline_parallel_size=1 is required"
|
||||
)
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
speculative_method = (
|
||||
speculative_config.method if speculative_config is not None else None
|
||||
)
|
||||
if speculative_method in {
|
||||
"dflash",
|
||||
"dspark",
|
||||
"eagle3",
|
||||
"extract_hidden_states",
|
||||
}:
|
||||
raise NotImplementedError(
|
||||
"AliceAI does not support speculative method "
|
||||
f"{speculative_method!r} because it requires auxiliary hidden states"
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV32ForCausalLM(VerifyAndUpdateConfig):
|
||||
@classmethod
|
||||
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -70,6 +115,32 @@ class GlmMoeDsaForCausalLM(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class Glm5NextModelConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
if (
|
||||
cache_config.cache_dtype != "auto"
|
||||
or not current_platform.is_cuda()
|
||||
or not current_platform.is_device_capability_family(100)
|
||||
or getattr(vllm_config.model_config.hf_text_config, "index_topk", None)
|
||||
is None
|
||||
):
|
||||
return
|
||||
|
||||
cache_config.cache_dtype = "fp8"
|
||||
logger.info_once("Using fp8 KV cache for GLM-5.3-Flash")
|
||||
spec_config = vllm_config.speculative_config
|
||||
if (
|
||||
spec_config is not None
|
||||
and spec_config.method != "mtp"
|
||||
and spec_config.kv_cache_dtype is None
|
||||
):
|
||||
spec_config.kv_cache_dtype = "auto"
|
||||
|
||||
|
||||
class Ernie4_5_VLMoeForConditionalGenerationConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
@@ -692,7 +763,7 @@ class MambaModelConfig(VerifyAndUpdateConfig):
|
||||
model_config.architecture,
|
||||
)
|
||||
assert vllm_config.scheduler_config.enable_chunked_prefill, (
|
||||
"Chunked prefill is required for mamba cache mode 'align'."
|
||||
"Chunked prefill is required for Mamba prefix caching."
|
||||
)
|
||||
# By default, mamba block size will be set to max_model_len (see
|
||||
# below). When enabling prefix caching, we align mamba block size
|
||||
@@ -912,11 +983,21 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config)
|
||||
text_config = vllm_config.model_config.hf_text_config
|
||||
if text_config.hc_count <= 1:
|
||||
raise ValueError("Qwen4Exp requires hc_count > 1")
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# MoE SP also enables HC SP without the explicit HC flag.
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and parallel_config.use_sequence_parallel_moe
|
||||
and parallel_config.pipeline_parallel_size != 1
|
||||
):
|
||||
raise ValueError("Qwen4Exp MoE SP requires PP=1")
|
||||
|
||||
uses_ple_or_qsa = bool(text_config.ple_layer_ids) or (
|
||||
getattr(text_config, "indexer_n_heads", None) is not None
|
||||
)
|
||||
@@ -1006,6 +1087,54 @@ class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||
config.is_causal = is_causal
|
||||
|
||||
|
||||
class D3ForSequenceClassificationConfig(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""Apply the contract declared by a Decision 3.0 `decision_config.json`."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
hf_config = model_config.hf_config
|
||||
decision = getattr(hf_config, "decision_config", None)
|
||||
if not isinstance(decision, dict):
|
||||
raise ValueError("Decision 3.0 checkpoints need a decision_config.json")
|
||||
if decision.get("format_version") != 1:
|
||||
raise ValueError(
|
||||
"unsupported decision_config.json format_version "
|
||||
f"{decision.get('format_version')!r}; vLLM reads version 1"
|
||||
)
|
||||
if decision.get("pooling", "last") != "last":
|
||||
raise ValueError(
|
||||
f"unsupported decision_config.json pooling {decision['pooling']!r}"
|
||||
)
|
||||
attention_mode = decision.get("attention_mode", "causal")
|
||||
if attention_mode not in ("causal", "noncausal_full_attention"):
|
||||
raise ValueError(
|
||||
f"unsupported decision_config.json attention_mode {attention_mode!r}"
|
||||
)
|
||||
codes = decision.get("codes")
|
||||
token_ids = decision.get("token_ids")
|
||||
if not codes or not isinstance(token_ids, list) or len(codes) != len(token_ids):
|
||||
raise ValueError(
|
||||
"decision_config.json must list one token id per answer code"
|
||||
)
|
||||
|
||||
# The full attention layers read the whole prompt in both directions;
|
||||
# the Gated DeltaNet layers stay causal.
|
||||
is_causal = attention_mode == "causal"
|
||||
for config in {
|
||||
id(config): config for config in (hf_config, model_config.hf_text_config)
|
||||
}.values():
|
||||
config.is_causal = is_causal
|
||||
# One readout row per answer code.
|
||||
hf_config.id2label = dict(enumerate(codes))
|
||||
|
||||
# Probabilities only exist over a question's own options, so the head
|
||||
# returns raw code logits unless activation is requested.
|
||||
pooler_config = model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
if pooler_config.use_activation is None:
|
||||
pooler_config.use_activation = False
|
||||
|
||||
|
||||
class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
@@ -1051,8 +1180,10 @@ class LongcatFlashNgramForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
|
||||
|
||||
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"AliceAIForCausalLM": AliceAIForCausalLMConfig,
|
||||
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
|
||||
"ColQwen3_5": ColQwen3_5Config,
|
||||
"D3ForSequenceClassification": D3ForSequenceClassificationConfig,
|
||||
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV4ForConditionalGeneration": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV41ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
@@ -1065,6 +1196,8 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Gemma4ForCausalLM": Gemma4Config,
|
||||
"Gemma4ForConditionalGeneration": Gemma4Config,
|
||||
"Gemma4UnifiedForConditionalGeneration": Gemma4Config,
|
||||
"Glm5NextForCausalLM": Glm5NextModelConfig,
|
||||
"Glm5NextForConditionalGeneration": Glm5NextModelConfig,
|
||||
"GlmMoeDsaForCausalLM": GlmMoeDsaForCausalLM,
|
||||
"GptOssForCausalLM": GptOssForCausalLMConfig,
|
||||
"LongcatFlashNgramForCausalLM": LongcatFlashNgramForCausalLMConfig,
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Decision 3.0 (vllm-sr/d3): a Qwen3.5 backbone with an answer-code readout.
|
||||
|
||||
The checkpoint is a bare `Qwen3_5Model` plus `readout.safetensors`, one row per
|
||||
answer code, and `decision_config.json`. The readout scores the codes at the
|
||||
last prompt token and is served as the `score` head of a classification model.
|
||||
`D3ForSequenceClassificationConfig` applies the checkpoint's attention mode and
|
||||
label space.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from transformers.models.qwen3_vl import Qwen3VLProcessor
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .adapters import as_seq_cls_model
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .qwen3_5 import Qwen3_5ForConditionalGeneration, Qwen3_5ProcessingInfo
|
||||
from .qwen3_vl import Qwen3VLDummyInputsBuilder, Qwen3VLMultiModalProcessor
|
||||
from .utils import WeightsMapper
|
||||
|
||||
READOUT_WEIGHTS = "readout.safetensors"
|
||||
|
||||
# The media budgets of the checkpoint's runtime (`d3_runtime.py`), which differ
|
||||
# from its processor config: images at 65,536 to 1,638,400 pixels, video frames
|
||||
# at up to 200,704 pixels and at most 32 frames per video.
|
||||
IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_638_400}
|
||||
VIDEO_FRAME_MAX_PIXELS = 200_704
|
||||
VIDEO_FRAME_MIN_PIXELS = 4_096
|
||||
VIDEO_MAX_FRAMES = 32
|
||||
|
||||
|
||||
class D3ProcessingInfo(Qwen3_5ProcessingInfo):
|
||||
def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
|
||||
processor = super().get_hf_processor(**kwargs)
|
||||
processor.image_processor.size = dict(IMAGE_SIZE)
|
||||
video = processor.video_processor
|
||||
factor = video.patch_size * video.merge_size
|
||||
video.size = {
|
||||
"shortest_edge": VIDEO_FRAME_MIN_PIXELS,
|
||||
"longest_edge": VIDEO_FRAME_MAX_PIXELS * VIDEO_MAX_FRAMES,
|
||||
}
|
||||
video.cap_pixels_per_frame = True
|
||||
video.max_video_tokens = VIDEO_FRAME_MAX_PIXELS // factor**2
|
||||
video.do_sample_frames = False
|
||||
return processor
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
_D3ForSequenceClassificationBase = Qwen3_5ForConditionalGeneration
|
||||
else:
|
||||
_D3ForSequenceClassificationBase = as_seq_cls_model(Qwen3_5ForConditionalGeneration)
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="LAST")
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Qwen3VLMultiModalProcessor,
|
||||
info=D3ProcessingInfo,
|
||||
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
||||
)
|
||||
class D3ForSequenceClassification(_D3ForSequenceClassificationBase):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"language_model.": "language_model.model.",
|
||||
"mtp.": None,
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
model_config = vllm_config.model_config
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=model_config.model,
|
||||
revision=model_config.revision,
|
||||
prefix="score.",
|
||||
allow_patterns_overrides=[READOUT_WEIGHTS],
|
||||
)
|
||||
]
|
||||
@@ -269,17 +269,6 @@ class DiffusionGemmaForConditionalGeneration(
|
||||
eps=getattr(text_config, "rms_norm_eps", 1e-6),
|
||||
)
|
||||
|
||||
def compute_self_conditioning(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
probs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
embed_weight = self.model.embed_tokens.weight
|
||||
soft_embeds = torch.matmul(
|
||||
probs.to(embed_weight.dtype), embed_weight
|
||||
) * self.model.normalizer.to(inputs_embeds.dtype)
|
||||
return self.self_conditioning(inputs_embeds, soft_embeds)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Multimodal: reuse Gemma4's image parsing, processing & embedding
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@@ -18,6 +18,7 @@ from transformers.models.embedding_gemma2.processing_embedding_gemma2 import (
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
@@ -334,6 +335,7 @@ class EmbeddingGemma2TextPLE(nn.Module):
|
||||
return self.per_layer_projection_norm(proj)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class EmbeddingGemma2TextModel(nn.Module):
|
||||
"""Text embedding backbone for EmbeddingGemma2 with bidirectional attention."""
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.distributed import parallel_state
|
||||
from vllm.distributed import utils as dist_utils
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -272,6 +273,7 @@ class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
|
||||
vision_config: Exaone4_5_VisionConfig,
|
||||
norm_eps: float = 1e-6,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
) -> None:
|
||||
@@ -279,6 +281,7 @@ class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
|
||||
vision_config=vision_config,
|
||||
norm_eps=norm_eps,
|
||||
quant_config=quant_config,
|
||||
input_norm=input_norm,
|
||||
prefix=prefix,
|
||||
)
|
||||
depth = vision_config.depth
|
||||
@@ -335,7 +338,9 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
|
||||
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
@@ -345,6 +350,7 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=self.quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
)
|
||||
|
||||
@@ -75,6 +75,7 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
|
||||
self.layers = nn.ModuleList(
|
||||
Exaone4DecoderLayer(
|
||||
text_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.layers.{idx}",
|
||||
)
|
||||
|
||||
@@ -227,24 +227,31 @@ class ExaoneMoeAttention(nn.Module):
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
|
||||
if config.sliding_windows is not None:
|
||||
self.sliding_window_size = config.sliding_windows[layer_idx]
|
||||
|
||||
if config.layer_types[layer_idx] == "full_attention":
|
||||
self.sliding_window_size = None
|
||||
|
||||
# Configs published before K-EXAONE 2.0 only have the scalar
|
||||
# `sliding_window` and no MTP fields; their MTP layer follows the main
|
||||
# layer with the same index.
|
||||
layer_types = config.layer_types
|
||||
sliding_windows = getattr(config, "sliding_windows", None)
|
||||
if is_mtp:
|
||||
self.sliding_window = (
|
||||
config.mtp_layer_types[layer_idx] == "sliding_attention"
|
||||
layer_types = getattr(config, "mtp_layer_types", None) or layer_types
|
||||
sliding_windows = (
|
||||
getattr(config, "mtp_sliding_windows", None) or sliding_windows
|
||||
)
|
||||
if config.mtp_sliding_windows is not None:
|
||||
self.sliding_window_size = config.mtp_sliding_windows[layer_idx]
|
||||
|
||||
if config.mtp_layer_types[layer_idx] == "full_attention":
|
||||
self.sliding_window_size = None
|
||||
if layer_types[layer_idx] == "full_attention":
|
||||
self.sliding_window_size = None
|
||||
elif sliding_windows is not None:
|
||||
self.sliding_window_size = sliding_windows[layer_idx]
|
||||
else:
|
||||
self.sliding_window_size = config.sliding_window
|
||||
|
||||
# apply rotary embeddings to every layer in full attention models
|
||||
self.apply_rope_all_layers = "sliding_attention" not in config.layer_types
|
||||
# Full-attention layers are NoPE when sliding layers are present, except
|
||||
# the MTP layer, which is trained with RoPE.
|
||||
self.use_rope = (
|
||||
bool(self.sliding_window_size)
|
||||
or is_mtp
|
||||
or "sliding_attention" not in config.layer_types
|
||||
)
|
||||
|
||||
set_default_rope_theta(config, default_theta=1000000)
|
||||
self.rotary_emb = get_rope(
|
||||
@@ -279,7 +286,7 @@ class ExaoneMoeAttention(nn.Module):
|
||||
k = self.k_norm(k)
|
||||
k = k.flatten(-2, -1)
|
||||
|
||||
if self.sliding_window_size or self.apply_rope_all_layers:
|
||||
if self.use_rope:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from functools import wraps
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
APPROX_VALUE_SUFFIX,
|
||||
ConstantExpertSubstitution,
|
||||
decoder_layer_index,
|
||||
get_expert_substitution_spec,
|
||||
)
|
||||
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
|
||||
|
||||
_T = TypeVar("_T", bound=nn.Module)
|
||||
|
||||
|
||||
def _collect_expert_substitutions(
|
||||
model: nn.Module,
|
||||
) -> dict[int, tuple[str, ConstantExpertSubstitution]]:
|
||||
substitutions: dict[int, tuple[str, ConstantExpertSubstitution]] = {}
|
||||
for name, module in model.named_modules():
|
||||
if not isinstance(module, ConstantExpertSubstitution):
|
||||
continue
|
||||
if module.layer_idx in substitutions:
|
||||
raise ValueError(
|
||||
f"layer {module.layer_idx} has more than one MoE with substituted "
|
||||
"experts"
|
||||
)
|
||||
substitutions[module.layer_idx] = (f"{name}.values", module)
|
||||
return substitutions
|
||||
|
||||
|
||||
def _parse_approx_value_name(name: str) -> tuple[int, int] | None:
|
||||
"""Return (layer index, expert ID) of an ``approx_value`` tensor name."""
|
||||
if not name.endswith(APPROX_VALUE_SUFFIX):
|
||||
return None
|
||||
experts_path, _, expert_id = name.removesuffix(APPROX_VALUE_SUFFIX).rpartition(".")
|
||||
layer_idx = decoder_layer_index(experts_path)
|
||||
if layer_idx is None or not expert_id.isdigit():
|
||||
return None
|
||||
return layer_idx, int(expert_id)
|
||||
|
||||
|
||||
def as_expert_substitution_model(model_cls: type[_T], model_config: Any) -> type[_T]:
|
||||
"""Load constant expert values at the ``load_weights`` boundary.
|
||||
|
||||
Initial loading and direct weight updates both go through ``load_weights``.
|
||||
Explicitly named values are always consumed and loaded if their layer is
|
||||
built by this model. ``approx_value`` tensors of layers built elsewhere
|
||||
(other pipeline stages, MTP layers) are passed through, so the model's own
|
||||
loader skips them like any other weight of a layer it does not own.
|
||||
"""
|
||||
spec = get_expert_substitution_spec(model_config)
|
||||
if spec is None:
|
||||
return model_cls
|
||||
value_names = spec.value_names
|
||||
|
||||
class ExpertSubstitutionModel(model_cls): # type: ignore[valid-type, misc]
|
||||
@wraps(model_cls.__init__) # type: ignore[misc]
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._expert_substitutions = _collect_expert_substitutions(self)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
if any(s.values.is_meta for _, s in self._expert_substitutions.values()):
|
||||
# Put the previous weights back so the model stays usable.
|
||||
finalize_layerwise_reload(self, model_config)
|
||||
raise NotImplementedError(
|
||||
"expert substitution does not support layerwise weight reload"
|
||||
)
|
||||
loaded_substitutions: set[str] = set()
|
||||
|
||||
def remaining_weights():
|
||||
for name, loaded_weight in weights:
|
||||
targets = value_names.get(name)
|
||||
if targets is None:
|
||||
key = _parse_approx_value_name(name)
|
||||
if key is None or key[0] not in self._expert_substitutions:
|
||||
yield name, loaded_weight
|
||||
continue
|
||||
targets = (key,)
|
||||
for layer_idx, expert_id in targets:
|
||||
if layer_idx not in self._expert_substitutions:
|
||||
continue
|
||||
param_name, substitution = self._expert_substitutions[layer_idx]
|
||||
param = substitution.values
|
||||
param.weight_loader(param, loaded_weight, expert_id)
|
||||
loaded_substitutions.add(param_name)
|
||||
|
||||
loaded = super().load_weights(remaining_weights())
|
||||
if loaded is not None:
|
||||
loaded.update(loaded_substitutions)
|
||||
return loaded
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
for param_name, substitution in self._expert_substitutions.values():
|
||||
substitution.validate_loaded_values(param_name)
|
||||
finalize = getattr(super(), "process_weights_after_loading", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
ExpertSubstitutionModel.__name__ = model_cls.__name__
|
||||
ExpertSubstitutionModel.__qualname__ = model_cls.__qualname__
|
||||
return ExpertSubstitutionModel
|
||||
@@ -126,10 +126,6 @@ class CacheOnlyAttentionBackend(AttentionBackend):
|
||||
def get_builder_cls() -> type["CacheOnlyAttentionMetadataBuilder"]:
|
||||
return CacheOnlyAttentionMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def use_cascade_attention(*args, **kwargs) -> bool:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
return []
|
||||
@@ -158,11 +154,6 @@ class CacheOnlyAttentionMetadataBuilder(
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> CacheOnlyAttentionMetadata:
|
||||
use_cascade = common_prefix_len > 0
|
||||
if use_cascade:
|
||||
raise NotImplementedError(
|
||||
"Cascade attention not supported by CacheOnlyAttention"
|
||||
)
|
||||
causal = common_attn_metadata.causal
|
||||
if not causal:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -42,7 +42,6 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsLoRA,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
@@ -499,7 +498,6 @@ class FalconH1ForCausalLM(
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
IsHybrid,
|
||||
SupportsMambaPrefixCaching,
|
||||
):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
|
||||
@@ -557,6 +557,10 @@ class Gemma3ForConditionalGeneration(
|
||||
)
|
||||
self.multi_modal_projector = Gemma3MultiModalProjector(config)
|
||||
|
||||
# https://huggingface.co/blog/gemma3, (image tokens attend to each other)
|
||||
# explicit mention here as text side in gemma3 standalone does not need
|
||||
# prefix_lm but multimodal does
|
||||
config.text_config.is_mm_prefix_lm = True
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
|
||||
@@ -98,6 +98,28 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_tower_quant_config(
|
||||
tower_config: Any,
|
||||
quant_config: "QuantizationConfig | None",
|
||||
) -> "QuantizationConfig | None":
|
||||
# BNB/torchao/compressed-tensors handle arbitrary sizes; other methods
|
||||
# (Marlin, FP8, …) require dimensions divisible by 64.
|
||||
# TODO(mgoin): remove this by fixing kernel padding.
|
||||
if quant_config and quant_config.get_name() in [
|
||||
"bitsandbytes",
|
||||
"torchao",
|
||||
"compressed-tensors",
|
||||
]:
|
||||
return quant_config
|
||||
# The audio config has no intermediate_size; treat it as always divisible.
|
||||
intermediate_size = getattr(tower_config, "intermediate_size", None)
|
||||
quantizable = tower_config.hidden_size % 64 == 0 and (
|
||||
intermediate_size is None or intermediate_size % 64 == 0
|
||||
)
|
||||
return quant_config if quantizable else None
|
||||
|
||||
|
||||
# Video constants — match transformers Gemma4VideoProcessor defaults.
|
||||
_SUPPORTED_SOFT_TOKENS = (70, 140, 280, 560, 1120)
|
||||
_VIDEO_MAX_SOFT_TOKENS = 70 # soft tokens per video frame (vs 280 for images)
|
||||
@@ -1071,25 +1093,9 @@ class Gemma4ForConditionalGeneration(
|
||||
lora_config is not None and lora_config.enable_tower_connector_lora
|
||||
)
|
||||
|
||||
# Only quantize towers when the quant method supports their
|
||||
# dimensions. BNB/torchao handle arbitrary sizes; other methods
|
||||
# (Marlin, FP8, …) require dimensions divisible by 64, which
|
||||
# the vision tower (intermediate_size=4304) does not satisfy.
|
||||
# TODO(mgoin): remove this by fixing kernel padding.
|
||||
tower_quant: QuantizationConfig | None
|
||||
if quant_config and quant_config.get_name() in [
|
||||
"bitsandbytes",
|
||||
"torchao",
|
||||
"compressed-tensors",
|
||||
]:
|
||||
tower_quant = quant_config
|
||||
else:
|
||||
vision_cfg = config.vision_config
|
||||
quantizable = (
|
||||
vision_cfg.hidden_size % 64 == 0
|
||||
and vision_cfg.intermediate_size % 64 == 0
|
||||
)
|
||||
tower_quant = quant_config if quantizable else None
|
||||
# Apply the dimension check independently because the vision and audio
|
||||
# towers do not necessarily have the same hidden and intermediate sizes.
|
||||
vision_tower_quant = _get_tower_quant_config(config.vision_config, quant_config)
|
||||
|
||||
# ---- Vision tower (shared by image and video) ----
|
||||
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
||||
@@ -1097,18 +1103,21 @@ class Gemma4ForConditionalGeneration(
|
||||
self.embed_vision = Gemma4MultimodalEmbedder(
|
||||
config.vision_config,
|
||||
config.text_config,
|
||||
quant_config=tower_quant,
|
||||
quant_config=vision_tower_quant,
|
||||
prefix=maybe_prefix(prefix, "embed_vision"),
|
||||
)
|
||||
recursive_replace_linear(
|
||||
self.vision_tower,
|
||||
tower_quant,
|
||||
vision_tower_quant,
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
|
||||
# ---- Audio tower (variants with audio_config) ----
|
||||
self.embed_audio: Gemma4MultimodalEmbedder | None
|
||||
if config.audio_config is not None:
|
||||
audio_tower_quant = _get_tower_quant_config(
|
||||
config.audio_config, quant_config
|
||||
)
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
self.audio_tower = AutoModel.from_config(config=config.audio_config)
|
||||
# AutoModel.from_config does NOT call post_init(),
|
||||
@@ -1119,12 +1128,12 @@ class Gemma4ForConditionalGeneration(
|
||||
self.embed_audio = Gemma4MultimodalEmbedder(
|
||||
config.audio_config,
|
||||
config.text_config,
|
||||
quant_config=tower_quant,
|
||||
quant_config=audio_tower_quant,
|
||||
prefix=maybe_prefix(prefix, "embed_audio"),
|
||||
)
|
||||
recursive_replace_linear(
|
||||
self.audio_tower,
|
||||
tower_quant,
|
||||
audio_tower_quant,
|
||||
prefix=maybe_prefix(prefix, "audio_tower"),
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
make_attention_sink,
|
||||
maybe_remap_kv_scale_name,
|
||||
remap_moe_expert_weights,
|
||||
)
|
||||
@@ -127,9 +128,7 @@ class OAIAttention(nn.Module):
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
|
||||
self.sinks = torch.nn.Parameter(
|
||||
torch.empty(config.num_attention_heads // tp_size, requires_grad=False)
|
||||
)
|
||||
self.sinks = make_attention_sink(config.num_attention_heads // tp_size)
|
||||
|
||||
self.q_size = self.num_attention_heads * self.head_dim // tp_size
|
||||
self.kv_size = self.num_key_value_heads * self.head_dim // tp_size
|
||||
@@ -556,8 +555,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
self,
|
||||
ep_rank_end: int,
|
||||
ep_rank_start: int,
|
||||
heads_per_rank: int,
|
||||
head_start: int,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
stacked_params_mapping: list[tuple[str, str, str]],
|
||||
) -> set[str]:
|
||||
@@ -720,13 +717,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
elif "sinks" in name:
|
||||
# Handle attention sinks (distributed across ranks)
|
||||
param = params_dict[name]
|
||||
narrow_weight = weight.narrow(0, head_start, heads_per_rank)
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
@@ -752,8 +742,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
self,
|
||||
ep_rank_end: int,
|
||||
ep_rank_start: int,
|
||||
heads_per_rank: int,
|
||||
head_start: int,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
stacked_params_mapping: list[tuple[str, str, str]],
|
||||
) -> set[str]:
|
||||
@@ -1082,14 +1070,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
loaded_params.add(fused_name)
|
||||
continue
|
||||
|
||||
elif "sinks" in name:
|
||||
# Handle attention sinks (distributed across ranks)
|
||||
param = params_dict[name]
|
||||
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if weight_name not in name:
|
||||
@@ -1168,8 +1148,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
self,
|
||||
ep_rank_end: int,
|
||||
ep_rank_start: int,
|
||||
heads_per_rank: int,
|
||||
head_start: int,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
stacked_params_mapping: list[tuple[str, str, str]],
|
||||
) -> set[str]:
|
||||
@@ -1256,13 +1234,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
param.copy_(weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
elif "sinks" in name:
|
||||
# Handle attention sinks (distributed across ranks)
|
||||
param = params_dict[name]
|
||||
narrow_weight = weight.narrow(0, head_start, heads_per_rank)
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
@@ -1345,13 +1316,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
]
|
||||
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
|
||||
# Attention heads per rank
|
||||
heads_per_rank = self.config.num_attention_heads // tp_size
|
||||
head_start = tp_rank * heads_per_rank
|
||||
|
||||
ep_size = get_ep_group().world_size
|
||||
ep_rank = get_ep_group().rank_in_group
|
||||
num_experts = self.config.num_local_experts
|
||||
@@ -1383,8 +1347,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
return self._load_weights_mxfp4(
|
||||
ep_rank_end,
|
||||
ep_rank_start,
|
||||
heads_per_rank,
|
||||
head_start,
|
||||
weights,
|
||||
stacked_params_mapping,
|
||||
)
|
||||
@@ -1392,8 +1354,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
return self._load_weights_quark(
|
||||
ep_rank_end,
|
||||
ep_rank_start,
|
||||
heads_per_rank,
|
||||
head_start,
|
||||
weights,
|
||||
stacked_params_mapping,
|
||||
)
|
||||
@@ -1401,8 +1361,6 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
return self._load_weights_other(
|
||||
ep_rank_end,
|
||||
ep_rank_start,
|
||||
heads_per_rank,
|
||||
head_start,
|
||||
weights,
|
||||
stacked_params_mapping,
|
||||
)
|
||||
@@ -1482,6 +1440,9 @@ class GptOssForCausalLM(
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return self.logits_processor(self.lm_head, hidden_states, skip_gather=True)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -60,7 +60,13 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
|
||||
from .interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
@@ -307,7 +313,7 @@ class GraniteDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class GraniteModel(nn.Module):
|
||||
class GraniteModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
@@ -363,7 +369,7 @@ class GraniteModel(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
@@ -375,8 +381,14 @@ class GraniteModel(nn.Module):
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, None
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
@@ -386,6 +398,9 @@ class GraniteModel(nn.Module):
|
||||
)
|
||||
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -393,7 +408,9 @@ class GraniteModel(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
class GraniteForCausalLM(
|
||||
nn.Module, SupportsLoRA, SupportsPP, SupportsQuant, SupportsEagle3
|
||||
):
|
||||
hf_to_vllm_mapper = GraniteModel.hf_to_vllm_mapper
|
||||
# LoRA specific attributes
|
||||
packed_modules_mapping: dict[str, list[str]] = {
|
||||
|
||||
@@ -61,7 +61,7 @@ from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .granite import granite_layer_attn_params
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
@@ -301,7 +301,7 @@ class GraniteMoeDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class GraniteMoeModel(nn.Module):
|
||||
class GraniteMoeModel(nn.Module, EagleModelMixin):
|
||||
hf_to_vllm_mapper: WeightsMapper = WeightsMapper(
|
||||
orig_to_new_suffix={
|
||||
# Legacy names to new names
|
||||
@@ -353,7 +353,7 @@ class GraniteMoeModel(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
@@ -363,8 +363,14 @@ class GraniteMoeModel(nn.Module):
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, None
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{
|
||||
@@ -372,6 +378,9 @@ class GraniteMoeModel(nn.Module):
|
||||
}
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
@@ -379,7 +388,7 @@ class GraniteMoeModel(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -42,7 +42,6 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsLoRA,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
@@ -587,7 +586,6 @@ class GraniteMoeHybridForCausalLM(
|
||||
SupportsPP,
|
||||
IsHybrid,
|
||||
SupportsQuant,
|
||||
SupportsMambaPrefixCaching,
|
||||
):
|
||||
packed_modules_mapping: dict[str, list[str]] = {
|
||||
"qkv_proj": [
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .granitemoe import GraniteMoeAttention, GraniteMoeModel, GraniteMoeMoE
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP
|
||||
from .utils import AutoWeightsLoader, make_layers, maybe_prefix
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ class GraniteMoeSharedDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class GraniteMoeSharedModel(nn.Module):
|
||||
class GraniteMoeSharedModel(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -190,7 +190,7 @@ class GraniteMoeSharedModel(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
@@ -200,8 +200,14 @@ class GraniteMoeSharedModel(nn.Module):
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, None)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, None
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{
|
||||
@@ -209,13 +215,16 @@ class GraniteMoeSharedModel(nn.Module):
|
||||
}
|
||||
)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
hf_to_vllm_mapper = GraniteMoeModel.hf_to_vllm_mapper
|
||||
load_weights = GraniteMoeModel.load_weights
|
||||
|
||||
|
||||
class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
packed_modules_mapping = {
|
||||
|
||||
@@ -142,6 +142,13 @@ _language_model_by_module: "weakref.WeakKeyDictionary[nn.Module, VllmModel]" = (
|
||||
)
|
||||
|
||||
|
||||
def _is_language_model(model: object) -> TypeIs[VllmModel]:
|
||||
# Text towers can use config/position_ids signatures and omit loader hooks.
|
||||
return callable(getattr(model, "embed_input_ids", None)) and callable(
|
||||
getattr(model, "forward", None)
|
||||
)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMultiModalEmbeddings(Protocol):
|
||||
"""The interface for models that can merge external multimodal embeddings."""
|
||||
@@ -285,14 +292,14 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
mod = getattr(mod, attr)
|
||||
|
||||
candidate: object = mod
|
||||
if candidate is not self_module and isinstance(candidate, VllmModel):
|
||||
if candidate is not self_module and _is_language_model(candidate):
|
||||
_language_model_by_module[self_module] = candidate
|
||||
return candidate
|
||||
|
||||
# Fallback
|
||||
for mod in self_module.children():
|
||||
candidate = mod
|
||||
if isinstance(candidate, VllmModel):
|
||||
if _is_language_model(candidate):
|
||||
_language_model_by_module[self_module] = candidate
|
||||
return candidate
|
||||
|
||||
@@ -859,6 +866,27 @@ def _supports_pp_inspect(model: type[object] | object) -> bool:
|
||||
return supports_kw(model_forward, "intermediate_tensors")
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsHCSP(Protocol):
|
||||
"""Support sequence-parallel HC without changing MoE parallelism."""
|
||||
|
||||
supports_hc_sp: ClassVar[Literal[True]] = True
|
||||
|
||||
|
||||
@overload
|
||||
def supports_hc_sp(model: type[object]) -> TypeIs[type[SupportsHCSP]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_hc_sp(model: object) -> TypeIs[SupportsHCSP]: ...
|
||||
|
||||
|
||||
def supports_hc_sp(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsHCSP]] | TypeIs[SupportsHCSP]:
|
||||
return getattr(model, "supports_hc_sp", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class HasInnerState(Protocol):
|
||||
"""The interface required for all models that has inner state."""
|
||||
@@ -1114,13 +1142,10 @@ def has_noops(
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsMambaPrefixCaching(Protocol):
|
||||
"""The interface for models whose mamba layers support prefix caching.
|
||||
|
||||
This is currently experimental.
|
||||
"""The interface for attention-free Mamba models that support prefix
|
||||
caching. Hybrid models get the same methods from `IsHybrid`.
|
||||
"""
|
||||
|
||||
supports_mamba_prefix_caching: ClassVar[Literal[True]] = True
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls) -> tuple["MambaStateCopyFunc", ...]:
|
||||
"""Return copy functions for the model's Mamba states."""
|
||||
@@ -1136,24 +1161,6 @@ class SupportsMambaPrefixCaching(Protocol):
|
||||
return {mamba_type: copy_funcs for mamba_type in mamba_types}
|
||||
|
||||
|
||||
@overload
|
||||
def supports_mamba_prefix_caching(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsMambaPrefixCaching]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_mamba_prefix_caching(
|
||||
model: object,
|
||||
) -> TypeIs[SupportsMambaPrefixCaching]: ...
|
||||
|
||||
|
||||
def supports_mamba_prefix_caching(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsMambaPrefixCaching]] | TypeIs[SupportsMambaPrefixCaching]:
|
||||
return getattr(model, "supports_mamba_prefix_caching", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsReplaySSM(Protocol):
|
||||
"""The interface for models whose recurrent layers support ReplaySSM
|
||||
|
||||
@@ -32,19 +32,13 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# The type of hidden states
|
||||
# Currently, T = torch.Tensor for all models except for Medusa
|
||||
# which has T = list[torch.Tensor]
|
||||
T = TypeVar("T", default=torch.Tensor)
|
||||
T_co = TypeVar("T_co", default=torch.Tensor, covariant=True)
|
||||
|
||||
# NOTE: Unlike those in `interfaces.py`, we don't define `ClassVar` tags
|
||||
# for the base interfaces to avoid breaking OOT registration for existing models
|
||||
# that don't inherit from the base interface classes
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VllmModel(Protocol[T_co]):
|
||||
class VllmModel(Protocol):
|
||||
"""The interface required for all models in vLLM."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None: ...
|
||||
@@ -53,7 +47,18 @@ class VllmModel(Protocol[T_co]):
|
||||
"""Apply token embeddings to `input_ids`."""
|
||||
...
|
||||
|
||||
def forward(self, input_ids: torch.Tensor, positions: torch.Tensor) -> T_co: ...
|
||||
def forward(
|
||||
self, input_ids: torch.Tensor, positions: torch.Tensor
|
||||
) -> torch.Tensor: ...
|
||||
|
||||
def is_unused_checkpoint_weight(self, name: str) -> bool:
|
||||
"""Return True for checkpoint weights this model does not load.
|
||||
|
||||
Defining this method is optional for satisfying this interface.
|
||||
Override with the same rule used in ``load_weights``. Models without
|
||||
a selection rule keep every checkpoint weight by default.
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
def _check_vllm_model_init(model: type[object] | object) -> bool:
|
||||
@@ -111,13 +116,13 @@ def is_vllm_model(
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VllmModelForTextGeneration(VllmModel[T], Protocol[T]):
|
||||
class VllmModelForTextGeneration(VllmModel, Protocol):
|
||||
"""The interface required for all generative models in vLLM."""
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: T,
|
||||
) -> T | None:
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
"""Return `None` if TP rank > 0."""
|
||||
...
|
||||
|
||||
@@ -138,14 +143,12 @@ def is_text_generation_model(
|
||||
if not is_vllm_model(model):
|
||||
return False
|
||||
|
||||
if isinstance(model, type):
|
||||
return isinstance(model, VllmModelForTextGeneration)
|
||||
|
||||
return isinstance(model, VllmModelForTextGeneration)
|
||||
# Optional protocol defaults are not inherited by structural models.
|
||||
return callable(getattr(model, "compute_logits", None))
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VllmModelForPooling(VllmModel[T_co], Protocol[T_co]):
|
||||
class VllmModelForPooling(VllmModel, Protocol):
|
||||
"""The interface required for all pooling models in vLLM."""
|
||||
|
||||
is_pooling_model: ClassVar[Literal[True]] = True
|
||||
|
||||
@@ -44,7 +44,6 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsLoRA,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
@@ -380,7 +379,6 @@ class JambaForCausalLM(
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
IsHybrid,
|
||||
SupportsMambaPrefixCaching,
|
||||
):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={".self_attn.": ".", ".A_log": ".A"},
|
||||
|
||||
@@ -255,25 +255,6 @@ class KeyeVisionEmbeddings(nn.Module):
|
||||
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
|
||||
return patch_pos_embed
|
||||
|
||||
def fetch_position_embedding_lfu_cache(self, embeddings, h, w, max_cache: int = 20):
|
||||
grid = (h, w)
|
||||
if grid in self.cache_position_embedding:
|
||||
self.cache_position_count[grid] += 1
|
||||
return self.cache_position_embedding[grid]
|
||||
|
||||
if len(self.cache_position_embedding) >= max_cache:
|
||||
min_hit_grid = min(
|
||||
self.cache_position_count,
|
||||
key=self.cache_position_count.__getitem__,
|
||||
)
|
||||
self.cache_position_count.pop(min_hit_grid)
|
||||
self.cache_position_embedding.pop(min_hit_grid)
|
||||
|
||||
position_embedding = self.interpolate_pos_encoding(embeddings, h, w, True)
|
||||
self.cache_position_count[grid] = 1
|
||||
self.cache_position_embedding[grid] = position_embedding
|
||||
return position_embedding
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.FloatTensor,
|
||||
|
||||
@@ -14,7 +14,6 @@ from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -36,6 +35,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
EagleModelMixin,
|
||||
SupportsEagle3,
|
||||
@@ -354,9 +354,7 @@ class LagunaAttention(nn.Module):
|
||||
# Attention sinks (learnable per-head bias for SWA layers)
|
||||
sinks = None
|
||||
if attention_sink:
|
||||
self.sink = torch.nn.Parameter(
|
||||
torch.empty(self.total_num_heads // tp_size, requires_grad=False)
|
||||
)
|
||||
self.sink = make_attention_sink(self.total_num_heads // tp_size)
|
||||
sinks = self.sink
|
||||
|
||||
# Resolve rope params per-layer-type. ``config.rope_parameters`` is
|
||||
@@ -677,22 +675,6 @@ class LagunaModel(nn.Module, EagleModelMixin):
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def _slice_sink(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
for name, loaded_weight in weights:
|
||||
# Handle attention sinks (distributed across ranks). Derive the
|
||||
# per-rank slice from the parameter's own shape so per-layer
|
||||
# variations in head count are handled correctly.
|
||||
if "sink" in name:
|
||||
heads_per_rank = loaded_weight.shape[0] // tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, tp_rank * heads_per_rank, heads_per_rank
|
||||
)
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
@@ -709,9 +691,7 @@ class LagunaModel(nn.Module, EagleModelMixin):
|
||||
"_input_scale",
|
||||
],
|
||||
)
|
||||
return loader.load_weights(
|
||||
self._slice_sink(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
|
||||
|
||||
@@ -4,15 +4,14 @@ from collections.abc import Iterable, Mapping, Sequence
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
PixtralVisionConfig,
|
||||
)
|
||||
from transformers import BatchFeature
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.models.mistral3 import (
|
||||
Mistral3DummyInputsBuilder,
|
||||
Mistral3ForConditionalGeneration,
|
||||
Mistral3ImageEmbeddingInputs,
|
||||
Mistral3ImageInputs,
|
||||
Mistral3MultiModalProjector,
|
||||
Mistral3ProcessingInfo,
|
||||
init_vision_tower_for_mistral3,
|
||||
@@ -26,12 +25,15 @@ from vllm.model_executor.models.utils import (
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
||||
from vllm.multimodal.parse import (
|
||||
ImageEmbeddingItems,
|
||||
ImageProcessorItems,
|
||||
MultiModalDataItems,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
|
||||
|
||||
@@ -102,18 +104,20 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing
|
||||
hf_config = self.info.get_hf_config()
|
||||
image_token_id = hf_config.image_token_index
|
||||
|
||||
assert isinstance(hf_config.vision_config, PixtralVisionConfig)
|
||||
encoder_info = PixtralHFEncoderInfo(hf_config)
|
||||
|
||||
def replace(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
ncols, nrows = encoder_info.get_patch_grid_size(
|
||||
image_width=size.width, image_height=size.height
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
assert isinstance(images, ImageProcessorItems)
|
||||
size = images.get_image_size(item_idx)
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
image_width=size.width, image_height=size.height
|
||||
)
|
||||
# break/end tokens are not used in LightOnOCR
|
||||
tokens = [image_token_id] * (ncols * nrows)
|
||||
return PromptUpdateDetails.select_token_id(tokens, image_token_id)
|
||||
return [image_token_id] * num_image_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
@@ -178,6 +182,15 @@ class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration):
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> Mistral3ImageInputs | None:
|
||||
image_embeds = kwargs.get("image_embeds")
|
||||
if image_embeds is not None:
|
||||
return Mistral3ImageEmbeddingInputs(data=image_embeds)
|
||||
|
||||
return super()._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import maybe_prefix
|
||||
|
||||
|
||||
class ResidualBlock(nn.Module):
|
||||
def __init__(self, config: VllmConfig, hidden_size: int, num_layers: int) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
nn.Linear(
|
||||
hidden_size,
|
||||
hidden_size,
|
||||
bias=getattr(config, "medusa_fc_bias", False),
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.act = nn.SiLU()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
for layer in self.layers:
|
||||
x = x + self.act(layer(x))
|
||||
return x
|
||||
|
||||
|
||||
class Medusa(nn.Module):
|
||||
"""This class implements the Medusa draft model from the paper: https://arxiv.org/abs/2401.10774
|
||||
Reference implementation: https://github.com/FasterDecoding/Medusa
|
||||
|
||||
Differences from reference implementation:
|
||||
1. Currently this only supports generating proposals from top-1 tokens.
|
||||
2. We have an optional token_map which reduces draft vocab to most
|
||||
frequently used tokens to give some additional speed-up by reducing
|
||||
sampling overhead. This is disabled unless the checkpoint file has
|
||||
explicit token_map tensor and config has an optional attribute
|
||||
truncated_vocab_size < vocab_size. To use this technique, one has to find
|
||||
the top-k most frequent tokens in target dataset and add that as a tensor
|
||||
in the draft checkpoint (using key token_map). Also, the draft config
|
||||
needs to have truncated_vocab_size (=k) as an attribute."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
config = speculative_config.draft_model_config.hf_config
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
ResidualBlock(
|
||||
config=config,
|
||||
hidden_size=self.config.hidden_size,
|
||||
num_layers=self.config.num_hidden_layers,
|
||||
)
|
||||
for _ in range(self.config.num_heads)
|
||||
]
|
||||
)
|
||||
self.orig_vocab_size = config.vocab_size
|
||||
self.truncated_vocab_size = config.truncated_vocab_size
|
||||
|
||||
if getattr(config, "original_lm_head", False):
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.truncated_vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.lm_heads = [self.lm_head for _ in range(self.config.num_heads)]
|
||||
else:
|
||||
self.lm_heads = nn.ModuleList(
|
||||
[
|
||||
ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
prefix=maybe_prefix(prefix, f"lm_heads.{i}"),
|
||||
)
|
||||
for i in range(self.config.num_heads)
|
||||
]
|
||||
)
|
||||
|
||||
logit_scale = getattr(config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.vocab_size, self.truncated_vocab_size, logit_scale
|
||||
)
|
||||
|
||||
# Token map is a idx to token mapping to reduce the vocab size for
|
||||
# the draft model. Using smaller vocab size for draft, containing
|
||||
# only most frequent tokens reduces the speculation overhead. This
|
||||
# doesn't affect the acceptance rate much and thus gives more speed
|
||||
# -up. By default, this is disabled and is only used if the EAGLE
|
||||
# checkpoint file has token_map tensor.
|
||||
self.token_map = None
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> list[torch.Tensor]:
|
||||
return [block(hidden_states) for block in self.blocks]
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: list[torch.Tensor],
|
||||
) -> list[torch.Tensor]:
|
||||
logits_lst: list[torch.Tensor] = []
|
||||
|
||||
for hs, lm_head in zip(hidden_states, self.lm_heads):
|
||||
_logits = self.logits_processor(lm_head, hs)
|
||||
|
||||
if _logits is None:
|
||||
# _logits should only be None on rank > 0, in which case
|
||||
# it should remain true for every lm_head
|
||||
assert len(logits_lst) == 0
|
||||
continue
|
||||
|
||||
if self.token_map is None:
|
||||
logits_lst.append(_logits)
|
||||
else:
|
||||
logits_lst.append(
|
||||
-torch.inf
|
||||
* torch.ones(
|
||||
size=(*_logits.shape[:-1], self.orig_vocab_size),
|
||||
device=_logits.device,
|
||||
dtype=_logits.dtype,
|
||||
)
|
||||
)
|
||||
|
||||
logits_lst[-1][..., self.token_map] = _logits
|
||||
|
||||
return logits_lst
|
||||
|
||||
@staticmethod
|
||||
def _remap_old_checkpoint_key(name: str) -> str:
|
||||
"""Map old FasterDecoding checkpoint keys to vLLM parameter names.
|
||||
|
||||
Old format uses bare numeric prefixes:
|
||||
'{head}.{layer}.linear.weight' -> 'blocks.{head}.layers.{layer}.weight'
|
||||
'{head}.{layer}.linear.bias' -> 'blocks.{head}.layers.{layer}.bias'
|
||||
'{head}.{N}.weight' -> 'lm_heads.{head}.weight'
|
||||
"""
|
||||
parts = name.split(".")
|
||||
if len(parts) >= 3 and parts[0].isdigit() and parts[1].isdigit():
|
||||
head, layer = parts[0], parts[1]
|
||||
rest = parts[2:]
|
||||
if "linear" in rest:
|
||||
param = ".".join(rest[rest.index("linear") + 1 :])
|
||||
return f"blocks.{head}.layers.{layer}.{param}"
|
||||
else:
|
||||
return f"lm_heads.{head}.{'.'.join(rest)}"
|
||||
return name
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
weights_map = {}
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
name = name.replace("medusa_heads.", "")
|
||||
name = self._remap_old_checkpoint_key(name)
|
||||
|
||||
if name == "token_map":
|
||||
if self.truncated_vocab_size < self.orig_vocab_size:
|
||||
self.token_map = nn.Parameter(loaded_weight, requires_grad=False)
|
||||
elif name in params_dict:
|
||||
weights_map[name] = loaded_weight
|
||||
elif (
|
||||
getattr(self.config, "original_lm_head", False)
|
||||
and name == "lm_heads.0.weight"
|
||||
):
|
||||
weights_map["lm_head.weight"] = loaded_weight
|
||||
|
||||
for name, loaded_weight in weights_map.items():
|
||||
if (
|
||||
"lm_head" in name
|
||||
and self.token_map is not None
|
||||
and loaded_weight.shape[0] > self.token_map.shape[0]
|
||||
):
|
||||
loaded_weight = loaded_weight[self.token_map]
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
if self.token_map is not None:
|
||||
self.token_map.to(device=self.lm_heads[0].weight.device)
|
||||
|
||||
assert (self.truncated_vocab_size == self.orig_vocab_size) or (
|
||||
self.token_map is not None
|
||||
)
|
||||
|
||||
return loaded_params
|
||||
@@ -47,6 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
make_attention_sink,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
@@ -290,9 +291,7 @@ class MiMoV2Attention(nn.Module):
|
||||
)
|
||||
|
||||
self.attention_sink_bias = (
|
||||
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
|
||||
if add_swa_attention_sink_bias
|
||||
else None
|
||||
make_attention_sink(self.num_heads) if add_swa_attention_sink_bias else None
|
||||
)
|
||||
|
||||
sliding_window = sliding_window_size if sliding_window_size > -1 else None
|
||||
@@ -848,18 +847,9 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
|
||||
|
||||
param = params_dict[name]
|
||||
|
||||
if "attention_sink_bias" in name:
|
||||
total_heads = loaded_weight.shape[0]
|
||||
heads_per_rank = total_heads // tp_size
|
||||
head_start = tp_rank * heads_per_rank
|
||||
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
|
||||
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
else:
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
@@ -377,14 +377,6 @@ class MiMoV2MTP(nn.Module):
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
# attention_sink_bias is head-parallel; slice by tp
|
||||
if "attention_sink_bias" in name:
|
||||
total_heads = loaded_weight.shape[0]
|
||||
heads_per_rank = total_heads // tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, tp_rank * heads_per_rank, heads_per_rank
|
||||
)
|
||||
|
||||
direct_weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
|
||||
@@ -45,6 +45,7 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
if "video_use_vit_merger" in hf_inputs:
|
||||
fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"video", keep_on_cpu=True
|
||||
)
|
||||
return fields
|
||||
|
||||
|
||||
def _expand_flags(value: object, num_items: int) -> list[bool | None]:
|
||||
"""Broadcast a batched per-item flag to exactly one entry per item."""
|
||||
if value is None:
|
||||
return [None] * num_items
|
||||
|
||||
raw: list[object]
|
||||
if isinstance(value, torch.Tensor):
|
||||
raw = value.reshape(-1).tolist()
|
||||
elif isinstance(value, list | tuple):
|
||||
raw = []
|
||||
for item in value:
|
||||
if isinstance(item, torch.Tensor):
|
||||
raw.extend(item.reshape(-1).tolist())
|
||||
else:
|
||||
raw.append(item)
|
||||
else:
|
||||
raw = [value]
|
||||
|
||||
if len(raw) == num_items:
|
||||
return [bool(v) for v in raw]
|
||||
if len(raw) == 1:
|
||||
return [bool(raw[0])] * num_items
|
||||
# Unexpected shape: keep one decision for the whole batch instead of
|
||||
# failing the request.
|
||||
return [any(bool(v) for v in raw)] * num_items
|
||||
|
||||
|
||||
def _vision_downsample_mode(flag: bool | None) -> str | None:
|
||||
if flag is None:
|
||||
return None
|
||||
return "16x" if flag else "4x"
|
||||
|
||||
|
||||
def _select_vision_items(
|
||||
image_input: MiniCPMVImagePixelInputs,
|
||||
item_indices: list[int],
|
||||
num_slices: list[int],
|
||||
) -> MiniCPMVImagePixelInputs:
|
||||
"""Slice a vision input down to the given items, keeping their order."""
|
||||
offsets = [0]
|
||||
for n in num_slices:
|
||||
offsets.append(offsets[-1] + n)
|
||||
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
tgt_sizes: list[torch.Tensor] = []
|
||||
sizes: list[int] = []
|
||||
for i in item_indices:
|
||||
start, end = offsets[i], offsets[i + 1]
|
||||
pixel_values.extend(image_input["pixel_values"][start:end])
|
||||
tgt_sizes.append(image_input["tgt_sizes"][start:end])
|
||||
sizes.append(end - start)
|
||||
|
||||
return MiniCPMVImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
tgt_sizes=torch.cat(tgt_sizes, dim=0),
|
||||
num_slices=torch.tensor(sizes),
|
||||
)
|
||||
|
||||
|
||||
# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
|
||||
# to a single modality and must not be forwarded to the processors, which take
|
||||
# flat arguments only.
|
||||
_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
|
||||
|
||||
|
||||
def _flat_processor_kwargs(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
downsample_mode: str,
|
||||
) -> dict[str, object]:
|
||||
"""Drop the modality-scoped nesting and pin the resolved downsample mode.
|
||||
|
||||
The placeholder count and the vision tower must read the same mode, so the
|
||||
value resolved for this modality is passed down explicitly rather than
|
||||
leaving the processor to fall back to its own default.
|
||||
"""
|
||||
kwargs = {
|
||||
key: value
|
||||
for key, value in mm_kwargs.items()
|
||||
if key not in _MODALITY_SCOPED_MM_KWARGS
|
||||
}
|
||||
kwargs["downsample_mode"] = downsample_mode
|
||||
return kwargs
|
||||
|
||||
|
||||
_MODALITY_TO_MM_KWARGS_SCOPE = {
|
||||
"image": "images_kwargs",
|
||||
"video": "videos_kwargs",
|
||||
}
|
||||
|
||||
|
||||
def _scoped_mm_kwarg(
|
||||
merged: Mapping[str, object],
|
||||
modality: str,
|
||||
key: str,
|
||||
) -> Any:
|
||||
"""Read a processor kwarg for one modality.
|
||||
|
||||
``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
|
||||
routes a flat key into every scope whose processor declares it, and a value
|
||||
scoped to the other modality stays there, so it is not visible here.
|
||||
"""
|
||||
scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
|
||||
if isinstance(scoped, Mapping):
|
||||
return scoped.get(key)
|
||||
return None
|
||||
|
||||
|
||||
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
def _resolve_downsample_mode(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> str:
|
||||
ds = mm_kwargs.get("downsample_mode")
|
||||
if ds is not None:
|
||||
return str(ds)
|
||||
# A flat `downsample_mode` applies to both modalities; the nested
|
||||
# `images_kwargs` / `videos_kwargs` forms scope it to one, matching
|
||||
# MiniCPMV4_6Processor.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
|
||||
if downsample_mode is not None:
|
||||
return str(downsample_mode)
|
||||
return info._get_downsample_mode()
|
||||
|
||||
def _resolve_max_slice_nums(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> int | None:
|
||||
# Per-request max_slice_nums. None keeps the processor default.
|
||||
# Resolved like `downsample_mode` so the nested per-modality form
|
||||
# (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
|
||||
# what `process_images` passes to the image processor: the placeholder
|
||||
# count and the visual token count are derived from it separately, so
|
||||
# they diverge (and the engine aborts) if the two sides read different
|
||||
# values.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
"""`<image_id>{idx}</image_id>` prefix emitted in front of a video placeholder.
|
||||
|
||||
MiniCPM-V 4.6 numbers each video with a local id, matching
|
||||
MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
|
||||
trained without these ids, so it overrides this to return an empty string.
|
||||
"""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
return f"{id_start}{video_idx}{id_end}"
|
||||
|
||||
def get_video_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_end = getattr(tokenizer, "image_end_token", "</image>")
|
||||
slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
|
||||
per_frame = image_start + video_token * source_tokens + image_end
|
||||
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
|
||||
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
per_frame += "\n".join(rows)
|
||||
|
||||
body = per_frame * num_frames
|
||||
return f"{id_start}{video_idx}{id_end}" + body
|
||||
return self._video_local_id_prefix(video_idx) + body
|
||||
|
||||
def process_images(
|
||||
self,
|
||||
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
n_images = len(parsed_images)
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
|
||||
image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
|
||||
# `_flat_processor_kwargs` drops the nested form, so the value resolved
|
||||
# from it has to be pinned back for the processor and the prompt text
|
||||
# to agree on the slice count.
|
||||
image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
|
||||
if image_max_slice is not None:
|
||||
image_proc_kwargs["max_slice_nums"] = image_max_slice
|
||||
per_image_pixel_values: list[torch.Tensor] = []
|
||||
per_image_tgt_sizes: list[torch.Tensor] = []
|
||||
for image in parsed_images:
|
||||
ip_out = image_processor([image], **mm_kwargs)
|
||||
ip_out = image_processor([image], **image_proc_kwargs)
|
||||
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
|
||||
ts = ip_out["target_sizes"] # (n_slices, 2)
|
||||
if pv.ndim == 4 and pv.shape[0] == 1:
|
||||
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
"tgt_sizes": per_image_tgt_sizes,
|
||||
}
|
||||
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs)
|
||||
insert_layer_id = getattr(
|
||||
self.info.get_hf_config(),
|
||||
"insert_layer_id",
|
||||
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
video_max_slice = self.info.get_video_max_slice_num()
|
||||
video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
|
||||
video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
|
||||
video_mm_kwargs = {
|
||||
**_flat_processor_kwargs(mm_kwargs, video_ds_mode),
|
||||
"max_slice_nums": video_max_slice,
|
||||
}
|
||||
|
||||
per_video_pixel_values: list[torch.Tensor] = []
|
||||
per_video_tgt_sizes: list[torch.Tensor] = []
|
||||
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
if not per_video_pixel_values:
|
||||
return {}
|
||||
|
||||
insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
|
||||
merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
|
||||
|
||||
return {
|
||||
"video_pixel_values": per_video_pixel_values,
|
||||
"video_tgt_sizes": per_video_tgt_sizes,
|
||||
"video_image_sizes": per_video_image_sizes,
|
||||
"video_use_vit_merger": [
|
||||
torch.tensor([merger_flag], dtype=torch.bool)
|
||||
for _ in per_video_pixel_values
|
||||
],
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs,
|
||||
):
|
||||
ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
|
||||
image_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="image"
|
||||
)
|
||||
video_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="video"
|
||||
)
|
||||
image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
|
||||
|
||||
placeholders = [
|
||||
("image", self.info.image_pattern),
|
||||
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=image_ds_mode,
|
||||
max_slice_nums=image_max_slice,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update, new_item_idx
|
||||
)
|
||||
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
|
||||
# (the base class only rewrites the image modality).
|
||||
# Video placeholders carry an `<image_id>{idx}</image_id>` prefix that
|
||||
# the base class does not rewrite. 4.7 has no prefix, so this is a
|
||||
# no-op there.
|
||||
if cached_update.modality == "video":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
old_prefix = self._video_local_id_prefix(cached_update.item_idx)
|
||||
new_prefix = self._video_local_id_prefix(new_item_idx)
|
||||
if old_prefix != new_prefix:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(old_prefix, new_prefix, 1),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
return source_tokens + grids[0] * grids[1] * patch_tokens
|
||||
|
||||
|
||||
def _stack_vit_merger_qkv(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
|
||||
|
||||
HF checkpoints store the window-attention projections separately, while
|
||||
vLLM keeps them fused. The shard_id mechanism on the submodule's
|
||||
WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
|
||||
incoming key after calling a submodule's load_weights, so a "k_proj" key
|
||||
with no matching child raises before the mapping is honoured.
|
||||
"""
|
||||
pairs = (
|
||||
("vit_merger.self_attn.q_proj.", "q"),
|
||||
("vit_merger.self_attn.k_proj.", "k"),
|
||||
("vit_merger.self_attn.v_proj.", "v"),
|
||||
)
|
||||
for name, tensor in weights:
|
||||
for src, shard in pairs:
|
||||
if src in name:
|
||||
name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
|
||||
tensor.shard_id = shard
|
||||
break
|
||||
yield name, tensor
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
SupportsPP,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsMRoPE,
|
||||
):
|
||||
supports_encoder_tp_data = True
|
||||
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
apply_encoder_attention_mask=True,
|
||||
prefix=maybe_prefix(prefix, "vpm"),
|
||||
)
|
||||
if config.drop_vision_last_layer:
|
||||
if self._drop_vision_last_layer():
|
||||
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
|
||||
|
||||
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
|
||||
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
"""Whether the last vision encoder layer is dropped.
|
||||
|
||||
MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
|
||||
carries the field, so it overrides this.
|
||||
"""
|
||||
return self.config.drop_vision_last_layer
|
||||
|
||||
# ----- Multimodal parsing -----
|
||||
|
||||
def _parse_and_validate_vision_input(
|
||||
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["image_embeds"]
|
||||
|
||||
downsample_mode = None
|
||||
if use_vit_merger is not None:
|
||||
downsample_mode = "16x" if use_vit_merger else "4x"
|
||||
image_features = self.get_vision_hidden_states(
|
||||
image_input,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
num_slices = image_input["num_slices"]
|
||||
results = []
|
||||
idx = 0
|
||||
for n in num_slices.tolist():
|
||||
group = image_features[idx : idx + n]
|
||||
results.append(torch.cat(group, dim=0))
|
||||
idx += n
|
||||
return results
|
||||
num_slices = image_input["num_slices"].tolist()
|
||||
flags = _expand_flags(use_vit_merger, len(num_slices))
|
||||
|
||||
# The merger flag is per request, but the model runner may batch items
|
||||
# from several concurrent requests. Encoding such a mixed batch would
|
||||
# apply one request's downsample mode to another request's items, so
|
||||
# group by flag and encode each group separately.
|
||||
encoded: dict[int, torch.Tensor] = {}
|
||||
for flag in dict.fromkeys(flags):
|
||||
item_indices = [i for i, f in enumerate(flags) if f == flag]
|
||||
group_input = _select_vision_items(image_input, item_indices, num_slices)
|
||||
group_features = self.get_vision_hidden_states(
|
||||
group_input,
|
||||
downsample_mode=_vision_downsample_mode(flag),
|
||||
)
|
||||
|
||||
idx = 0
|
||||
for i in item_indices:
|
||||
n = num_slices[i]
|
||||
encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
|
||||
idx += n
|
||||
|
||||
return [encoded[i] for i in range(len(num_slices))]
|
||||
|
||||
# ----- Multimodal embedding interface -----
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
|
||||
use_vit_merger = None
|
||||
if use_vit_merger_tensors is not None:
|
||||
if isinstance(use_vit_merger_tensors, torch.Tensor):
|
||||
use_vit_merger = bool(use_vit_merger_tensors.any().item())
|
||||
elif isinstance(use_vit_merger_tensors, list | tuple):
|
||||
use_vit_merger = any(
|
||||
bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
|
||||
for t in use_vit_merger_tensors
|
||||
)
|
||||
use_vit_merger = kwargs.pop("use_vit_merger", None)
|
||||
video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
|
||||
|
||||
image_kwargs = {
|
||||
k: v
|
||||
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
multimodal_embeddings += tuple(
|
||||
self._process_vision_input(
|
||||
video_input,
|
||||
use_vit_merger=use_vit_merger,
|
||||
use_vit_merger=video_use_vit_merger,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loader.load_weights(
|
||||
_stack_vit_merger_qkv(weights),
|
||||
mapper=self.hf_to_vllm_mapper,
|
||||
)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
|
||||
@@ -0,0 +1,764 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only MiniCPM-V 4.7 model.
|
||||
|
||||
Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
|
||||
|
||||
from .minicpmv import MiniCPMVDummyInputsBuilder
|
||||
from .minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
MiniCPMV4_6MultiModalProcessor,
|
||||
MiniCPMV4_6ProcessingInfo,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo):
|
||||
def _mm_max_slice_nums(self, **kwargs: object) -> int | None:
|
||||
# No schema resolution here: it would call back into `get_hf_processor`
|
||||
# below, which resolves this same value.
|
||||
merged = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums")
|
||||
if max_slice is None:
|
||||
max_slice = merged.get("max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def _configured_max_slice_nums(self) -> int | None:
|
||||
# Configured (server-level) value only: per-request values are passed
|
||||
# explicitly to the image processor call, so they must not be written
|
||||
# back to the shared processor instance.
|
||||
return self._mm_max_slice_nums()
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
hf_processor = super().get_hf_processor(**kwargs)
|
||||
image_processor = getattr(hf_processor, "image_processor", None)
|
||||
configured = self._configured_max_slice_nums()
|
||||
if image_processor is not None and configured is not None:
|
||||
image_processor.max_slice_nums = configured
|
||||
return hf_processor
|
||||
|
||||
def get_image_max_slice_num(self) -> int:
|
||||
max_slice = self._configured_max_slice_nums()
|
||||
if max_slice is not None:
|
||||
return max_slice
|
||||
return super().get_image_max_slice_num()
|
||||
|
||||
|
||||
class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs))
|
||||
# Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of
|
||||
# a prefix-cache-covered item only keeps CPU-side metadata, so without
|
||||
# this a cached image loses its target sizes, `image_bounds` and
|
||||
# `target_sizes` disagree, and the whole request silently falls back to
|
||||
# sequential positions.
|
||||
for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")):
|
||||
if key in fields:
|
||||
fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True)
|
||||
return fields
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
# MiniCPMV4_7Processor emits no local `<image_id>` in front of video
|
||||
# placeholders: a video is one temporal sequence and 4.7 was trained
|
||||
# without them.
|
||||
return ""
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
if max_slice_nums is None:
|
||||
max_slice_nums = info.get_image_max_slice_num()
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
|
||||
def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor:
|
||||
data = tensor.to(torch.long)
|
||||
if data.numel() == 0:
|
||||
return data.reshape(0, 2)
|
||||
if data.shape[-1] != 2:
|
||||
raise ValueError(f"expected last dim 2, got {tuple(data.shape)}")
|
||||
return data.reshape(-1, 2)
|
||||
|
||||
|
||||
def _sequential_positions(seq_len: int) -> torch.Tensor:
|
||||
"""Positions for the fallback where canvas M-RoPE cannot be computed."""
|
||||
return torch.arange(seq_len).unsqueeze(0).expand(3, -1)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
info=MiniCPMV4_7ProcessingInfo,
|
||||
dummy_inputs=MiniCPMVDummyInputsBuilder,
|
||||
)
|
||||
class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self._init_canvas_mrope(vllm_config)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
# MiniCPMV4_7Config no longer carries this field (transformers marks it
|
||||
# removed), and 4.7 keeps every vision encoder layer.
|
||||
return False
|
||||
|
||||
def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None:
|
||||
config = self.config
|
||||
uses_canvas = bool(getattr(config, "uses_mrope_canvas", False))
|
||||
if not uses_canvas:
|
||||
mrope_mode = getattr(config, "mrope_mode", None)
|
||||
uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower()
|
||||
|
||||
self._uses_canvas_mrope = uses_canvas
|
||||
self._canvas_special_ids: dict[str, int] | None = None
|
||||
if not uses_canvas:
|
||||
return
|
||||
|
||||
# MiniCPMV4_7Config pins the structural ids in config.json; the
|
||||
# conversion script resolves them from the tokenizer. Prefer them so the
|
||||
# canvas matches training, and fall back to the tokenizer for checkpoints
|
||||
# whose config predates these fields.
|
||||
configured_ids: dict[str, int | None] = {
|
||||
"im_start_id": getattr(config, "image_start_id", None),
|
||||
"im_end_id": getattr(config, "image_end_id", None),
|
||||
"slice_start_id": getattr(config, "slice_start_id", None),
|
||||
"slice_end_id": getattr(config, "slice_end_id", None),
|
||||
"newline_id": getattr(config, "newline_id", None),
|
||||
}
|
||||
if all(value is not None for value in configured_ids.values()):
|
||||
self._canvas_special_ids = {
|
||||
name: int(value)
|
||||
for name, value in configured_ids.items()
|
||||
if value is not None
|
||||
}
|
||||
return
|
||||
|
||||
from vllm.tokenizers.registry import get_tokenizer
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
tokenizer = get_tokenizer(
|
||||
model_config.tokenizer,
|
||||
revision=model_config.tokenizer_revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
self._canvas_special_ids = {
|
||||
"im_start_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "image_start_token", "<image>")
|
||||
),
|
||||
"im_end_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "image_end_token", "</image>")
|
||||
),
|
||||
"slice_start_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
),
|
||||
"slice_end_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
),
|
||||
"newline_id": tokenizer.encode("\n", add_special_tokens=False)[0],
|
||||
}
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
seq_len = len(input_tokens)
|
||||
if (
|
||||
not getattr(self, "_uses_canvas_mrope", False)
|
||||
or self._canvas_special_ids is None
|
||||
or not mm_features
|
||||
):
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
input_ids = torch.tensor(input_tokens, dtype=torch.long)
|
||||
image_bounds = build_image_bounds(input_ids, self._canvas_special_ids)
|
||||
|
||||
target_sizes_list: list[torch.Tensor] = []
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
if mm_feature.data is None:
|
||||
continue
|
||||
for key in ("tgt_sizes", "video_tgt_sizes"):
|
||||
tgt = mm_feature.data.get(key)
|
||||
if tgt is None or not isinstance(tgt.data, torch.Tensor):
|
||||
continue
|
||||
try:
|
||||
target_sizes_list.append(_as_hw_pairs(tgt.data))
|
||||
except ValueError:
|
||||
# Not an (N, 2) size table. The bounds/sizes match below
|
||||
# turns this into a sequential fallback.
|
||||
continue
|
||||
|
||||
if not target_sizes_list or image_bounds.numel() == 0:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE missing inputs: "
|
||||
"tgt_chunks=%s image_bounds=%s; using sequential positions.",
|
||||
len(target_sizes_list),
|
||||
tuple(image_bounds.shape) if image_bounds is not None else None,
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
target_sizes = torch.cat(target_sizes_list, dim=0)
|
||||
if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; "
|
||||
"using sequential positions.",
|
||||
tuple(target_sizes.shape),
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
if image_bounds.shape[0] != target_sizes.shape[0]:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE size mismatch: "
|
||||
"image_bounds=%s target_sizes=%s; using sequential positions.",
|
||||
tuple(image_bounds.shape),
|
||||
tuple(target_sizes.shape),
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
try:
|
||||
pos3d = _compute_canvas_single(
|
||||
input_ids,
|
||||
torch.arange(seq_len, dtype=torch.long),
|
||||
image_bounds,
|
||||
target_sizes,
|
||||
self._canvas_special_ids,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE failed; "
|
||||
"falling back to sequential positions.",
|
||||
exc_info=True,
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
return pos3d, canvas_rope_delta(pos3d, seq_len)
|
||||
|
||||
|
||||
# Canvas 3D M-RoPE: position ids for the slice canvas.
|
||||
def compute_llm_grid(
|
||||
n_tokens: int, vision_grid_height: int, vision_grid_width: int
|
||||
) -> tuple[int, int]:
|
||||
if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0:
|
||||
return 0, 0
|
||||
vit_total = vision_grid_height * vision_grid_width
|
||||
if vit_total == n_tokens:
|
||||
return vision_grid_height, vision_grid_width
|
||||
if vit_total < n_tokens:
|
||||
return 0, 0
|
||||
factor = int(round(math.sqrt(vit_total / n_tokens)))
|
||||
if (
|
||||
factor > 0
|
||||
and vision_grid_height % factor == 0
|
||||
and vision_grid_width % factor == 0
|
||||
):
|
||||
height, width = vision_grid_height // factor, vision_grid_width // factor
|
||||
if height * width == n_tokens:
|
||||
return height, width
|
||||
ratio = vision_grid_height / vision_grid_width
|
||||
width = max(1, int(round(math.sqrt(n_tokens / ratio))))
|
||||
height = n_tokens // width
|
||||
if height * width == n_tokens and height > 0:
|
||||
return height, width
|
||||
return 0, 0
|
||||
|
||||
|
||||
def build_image_bounds(
|
||||
input_ids: torch.LongTensor, special_token_ids: dict
|
||||
) -> torch.LongTensor:
|
||||
start_ids = [
|
||||
x
|
||||
for x in (
|
||||
special_token_ids.get("im_start_id"),
|
||||
special_token_ids.get("slice_start_id"),
|
||||
)
|
||||
if x is not None
|
||||
]
|
||||
end_ids = [
|
||||
x
|
||||
for x in (
|
||||
special_token_ids.get("im_end_id"),
|
||||
special_token_ids.get("slice_end_id"),
|
||||
)
|
||||
if x is not None
|
||||
]
|
||||
if not start_ids or not end_ids:
|
||||
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
|
||||
start_cond = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
end_cond = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
for sid in start_ids:
|
||||
start_cond |= input_ids == sid
|
||||
for eid in end_ids:
|
||||
end_cond |= input_ids == eid
|
||||
starts = torch.where(start_cond)[0] + 1
|
||||
ends = torch.where(end_cond)[0]
|
||||
n = min(len(starts), len(ends))
|
||||
if n == 0:
|
||||
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
|
||||
return torch.stack([starts[:n], ends[:n]], dim=-1)
|
||||
|
||||
|
||||
def _compute_canvas_single(
|
||||
input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids
|
||||
):
|
||||
# Inference always handles one unpadded sequence. The packed (cu_seqlens)
|
||||
# form below is kept only to mirror the transformers reference
|
||||
# implementation of its canvas position computation.
|
||||
seq_len = input_ids.shape[0]
|
||||
cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long)
|
||||
return _compute_canvas_packed(
|
||||
position_ids_2d.unsqueeze(0),
|
||||
cu_seqlens,
|
||||
image_bound,
|
||||
target_sizes,
|
||||
input_ids.unsqueeze(0),
|
||||
special_token_ids,
|
||||
)[:, 0, :]
|
||||
|
||||
|
||||
def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
|
||||
"""Patch-grid height/width recorded for one image or slice."""
|
||||
return int(target_sizes[index, 0]), int(target_sizes[index, 1])
|
||||
|
||||
|
||||
class _CanvasLayout(NamedTuple):
|
||||
"""Grid of one canvas measured in LLM tokens.
|
||||
|
||||
A canvas is a thumbnail plus the slices cut from it, tiled row-major.
|
||||
"""
|
||||
|
||||
thumb_h: int
|
||||
thumb_w: int
|
||||
"""Thumbnail grid."""
|
||||
slice_h: int
|
||||
slice_w: int
|
||||
"""Per-slice grid. Zero when the canvas has no slices."""
|
||||
cols: int
|
||||
"""Slices sharing a canvas row. Zero when the canvas has no slices."""
|
||||
height: int
|
||||
width: int
|
||||
"""Assembled canvas."""
|
||||
|
||||
|
||||
def _canvas_layout(
|
||||
thumbnail: tuple[int, int, int],
|
||||
slices: list[tuple[int, int, int]],
|
||||
target_sizes: torch.Tensor,
|
||||
) -> _CanvasLayout:
|
||||
"""Measure the canvas for one ``(thumbnail, slices)`` group.
|
||||
|
||||
``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
|
||||
"""
|
||||
t_bs, t_be, t_gi = thumbnail
|
||||
thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
|
||||
if not slices:
|
||||
return _CanvasLayout(
|
||||
thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
|
||||
)
|
||||
|
||||
cols = len(slices)
|
||||
for k in range(len(slices) - 1):
|
||||
if slices[k + 1][0] - slices[k][1] > 2:
|
||||
cols = k + 1
|
||||
break
|
||||
rows = len(slices) // cols if cols > 0 else 1
|
||||
if rows * cols != len(slices):
|
||||
rows, cols = 1, len(slices)
|
||||
|
||||
s0_bs, s0_be, s0_gi = slices[0]
|
||||
slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
|
||||
return _CanvasLayout(
|
||||
thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
|
||||
)
|
||||
|
||||
|
||||
def _assign_canvas_span(
|
||||
pos3d: torch.Tensor,
|
||||
*,
|
||||
thumbnail: tuple[int, int, int],
|
||||
span_end: int,
|
||||
slices: list[tuple[int, int, int]],
|
||||
base: int,
|
||||
target_sizes: torch.Tensor,
|
||||
) -> int:
|
||||
"""Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.
|
||||
|
||||
Shared by the video-frame and single-image paths; they differ only in how
|
||||
the per-canvas base advances, not in the geometry placed here.
|
||||
|
||||
Mutates ``pos3d`` in place, as the packed canvas computation has always
|
||||
done. Returns the distance to the next canvas base, so each caller keeps
|
||||
only its own base advancement.
|
||||
"""
|
||||
t_bs, t_be, _ = thumbnail
|
||||
device = pos3d.device
|
||||
span_start = t_bs - 1 # the <im_start> token
|
||||
n_thumb = t_be - t_bs
|
||||
layout = _canvas_layout(thumbnail, slices, target_sizes)
|
||||
cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width
|
||||
|
||||
# base coat
|
||||
pos3d[0, 0, span_start:span_end] = base
|
||||
pos3d[1, 0, span_start:span_end] = base
|
||||
pos3d[2, 0, span_start:span_end] = base
|
||||
|
||||
# <im_start> -> halo
|
||||
halo_lo = max(base - 1, 0)
|
||||
pos3d[1, 0, span_start] = halo_lo
|
||||
pos3d[2, 0, span_start] = halo_lo
|
||||
|
||||
# <im_end> -> halo (right after thumbnail)
|
||||
if t_be < span_end:
|
||||
pos3d[1, 0, t_be] = base + canvas_H
|
||||
pos3d[2, 0, t_be] = base + canvas_W
|
||||
|
||||
# thumbnail visual tokens
|
||||
llm_th, llm_tw = layout.thumb_h, layout.thumb_w
|
||||
if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
|
||||
if slices and canvas_H > 0 and canvas_W > 0:
|
||||
h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
|
||||
w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
|
||||
else:
|
||||
h_c = torch.arange(llm_th, device=device)
|
||||
w_c = torch.arange(llm_tw, device=device)
|
||||
h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
|
||||
w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
|
||||
pos3d[0, 0, t_bs:t_be] = base
|
||||
pos3d[1, 0, t_bs:t_be] = h_idx + base
|
||||
pos3d[2, 0, t_bs:t_be] = w_idx + base
|
||||
else:
|
||||
fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
|
||||
for d in range(3):
|
||||
pos3d[d, 0, t_bs:t_be] = fallback
|
||||
|
||||
# slice visual tokens + slice specials
|
||||
for k, (s_bs, s_be, s_gi) in enumerate(slices):
|
||||
n_s = s_be - s_bs
|
||||
sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
|
||||
h_off = (k // cols) * layout.slice_h
|
||||
w_off = (k % cols) * layout.slice_w
|
||||
|
||||
slice_start_pos = s_bs - 1
|
||||
if slice_start_pos >= span_start:
|
||||
pos3d[1, 0, slice_start_pos] = base + h_off
|
||||
pos3d[2, 0, slice_start_pos] = base + w_off
|
||||
|
||||
slice_end_pos = s_be
|
||||
if sh_k > 0 and sw_k > 0:
|
||||
end_h = h_off + sh_k - 1
|
||||
end_w = w_off + sw_k - 1
|
||||
else:
|
||||
end_h = h_off
|
||||
end_w = w_off
|
||||
if slice_end_pos < span_end:
|
||||
pos3d[1, 0, slice_end_pos] = base + end_h
|
||||
pos3d[2, 0, slice_end_pos] = base + end_w
|
||||
|
||||
if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
|
||||
h_idx = (
|
||||
torch.arange(sh_k, device=device)
|
||||
.view(-1, 1)
|
||||
.expand(-1, sw_k)
|
||||
.reshape(-1)
|
||||
)
|
||||
w_idx = (
|
||||
torch.arange(sw_k, device=device)
|
||||
.view(1, -1)
|
||||
.expand(sh_k, -1)
|
||||
.reshape(-1)
|
||||
)
|
||||
pos3d[0, 0, s_bs:s_be] = base
|
||||
pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
|
||||
pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
|
||||
else:
|
||||
fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
|
||||
for d in range(3):
|
||||
pos3d[d, 0, s_bs:s_be] = fallback
|
||||
|
||||
# \n between slice rows -> W = right edge + 1.
|
||||
# H stays within the ended row (does NOT cross to next row): the \n sits at
|
||||
# the right-outside of the row that just ended, at H = that row's last row
|
||||
# and W = one column past the rightmost slice.
|
||||
for k in range(len(slices) - 1):
|
||||
gap_s = slices[k][1]
|
||||
gap_e = slices[k + 1][0]
|
||||
if gap_e - gap_s <= 2:
|
||||
continue
|
||||
boundary_h = ((k // cols) + 1) * layout.slice_h - 1
|
||||
right_edge_w = cols * layout.slice_w
|
||||
for nl_pos in range(gap_s + 1, gap_e - 1):
|
||||
pos3d[1, 0, nl_pos] = base + boundary_h
|
||||
pos3d[2, 0, nl_pos] = base + right_edge_w
|
||||
|
||||
return max(canvas_H, canvas_W) + 1
|
||||
|
||||
|
||||
def _compute_canvas_packed(
|
||||
position_ids_2d,
|
||||
cu_seqlens,
|
||||
image_bound,
|
||||
target_sizes,
|
||||
input_ids,
|
||||
special_token_ids,
|
||||
):
|
||||
if position_ids_2d.ndim == 1:
|
||||
position_ids_2d = position_ids_2d.unsqueeze(0)
|
||||
if input_ids.ndim == 1:
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
|
||||
has_images = (
|
||||
isinstance(image_bound, torch.Tensor)
|
||||
and image_bound.numel() > 0
|
||||
and isinstance(target_sizes, torch.Tensor)
|
||||
and target_sizes.numel() > 0
|
||||
)
|
||||
if not has_images:
|
||||
return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous()
|
||||
|
||||
device = position_ids_2d.device
|
||||
pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone()
|
||||
ids_flat = input_ids.view(-1)
|
||||
|
||||
im_start_id = special_token_ids.get("im_start_id")
|
||||
im_end_id = special_token_ids.get("im_end_id")
|
||||
slice_start_id = special_token_ids.get("slice_start_id")
|
||||
slice_end_id = special_token_ids.get("slice_end_id")
|
||||
|
||||
structural_ids = {
|
||||
v
|
||||
for v in (im_start_id, im_end_id, slice_start_id, slice_end_id)
|
||||
if v is not None
|
||||
}
|
||||
|
||||
num_seqs = len(cu_seqlens) - 1
|
||||
img_ptr = 0
|
||||
num_imgs = len(image_bound)
|
||||
|
||||
newline_id = special_token_ids.get("newline_id")
|
||||
gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set())
|
||||
|
||||
def _group_end_pos(g, s_end):
|
||||
"""First position after a group's trailing structural tokens.
|
||||
|
||||
Matches the transformers reference, which closes a span over every
|
||||
marker in ``markers`` -- newlines included. The chat template joins
|
||||
content parts with ``\\n``, so an image followed by text always ends with
|
||||
one; leaving it out moves that newline out of the canvas and pushes
|
||||
every later position one step forward.
|
||||
"""
|
||||
le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1]
|
||||
ge = le
|
||||
while ge < s_end and ids_flat[ge].item() in gap_ok_ids:
|
||||
ge += 1
|
||||
return min(ge, s_end)
|
||||
|
||||
def _is_tight_gap(g_prev, g_next, s_end):
|
||||
"""Check if only structural/nl tokens sit between two groups."""
|
||||
ge = _group_end_pos(g_prev, s_end)
|
||||
ns = g_next["thumbnail"][0] - 1 # im_start of next group
|
||||
return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns))
|
||||
|
||||
for si in range(num_seqs):
|
||||
s_start = cu_seqlens[si].item()
|
||||
s_end = cu_seqlens[si + 1].item()
|
||||
if s_start >= s_end:
|
||||
continue
|
||||
|
||||
seq_imgs = []
|
||||
while img_ptr < num_imgs:
|
||||
bs_ = image_bound[img_ptr, 0].item()
|
||||
be_ = image_bound[img_ptr, 1].item()
|
||||
if bs_ >= s_start and be_ <= s_end:
|
||||
seq_imgs.append((bs_, be_, img_ptr))
|
||||
img_ptr += 1
|
||||
else:
|
||||
break
|
||||
if not seq_imgs:
|
||||
continue
|
||||
|
||||
# ---- group images: thumbnail + following slices ----
|
||||
# merge consecutive no-slice frames into one video group
|
||||
raw_groups = []
|
||||
cur_group: dict[str, Any] | None = None
|
||||
for bs_, be_, gi in seq_imgs:
|
||||
marker_pos = bs_ - 1
|
||||
is_slice = (
|
||||
slice_start_id is not None
|
||||
and marker_pos >= s_start
|
||||
and ids_flat[marker_pos].item() == slice_start_id
|
||||
)
|
||||
if is_slice and cur_group is not None:
|
||||
cur_group["slices"].append((bs_, be_, gi))
|
||||
else:
|
||||
if cur_group is not None:
|
||||
raw_groups.append(cur_group)
|
||||
cur_group = {"thumbnail": (bs_, be_, gi), "slices": []}
|
||||
if cur_group is not None:
|
||||
raw_groups.append(cur_group)
|
||||
|
||||
# merge consecutive tightly-adjacent groups into video groups.
|
||||
# Two groups are "tightly adjacent" if the gap between them
|
||||
# contains only structural tokens and \n (no real text).
|
||||
# This handles both no-slice frames and frames-with-slices.
|
||||
groups = []
|
||||
i_g = 0
|
||||
while i_g < len(raw_groups):
|
||||
video_frames = [raw_groups[i_g]]
|
||||
j_g = i_g + 1
|
||||
while j_g < len(raw_groups) and _is_tight_gap(
|
||||
raw_groups[j_g - 1], raw_groups[j_g], s_end
|
||||
):
|
||||
video_frames.append(raw_groups[j_g])
|
||||
j_g += 1
|
||||
if len(video_frames) == 1:
|
||||
groups.append(raw_groups[i_g])
|
||||
else:
|
||||
groups.append({"video_frames": video_frames})
|
||||
i_g = j_g
|
||||
|
||||
# ---- assign 3-D positions ----
|
||||
pos = 0
|
||||
cursor = s_start
|
||||
|
||||
for group in groups:
|
||||
is_video_group = "video_frames" in group
|
||||
|
||||
if is_video_group:
|
||||
# === Video group: multiple frames merged ===
|
||||
# Each frame gets its own base_f so different frames have
|
||||
# distinct spatial positions. Inter-frame gap tokens (\n
|
||||
# etc.) are pure 1D text. After each gap, pos advances
|
||||
# by 1 extra so halo_lo of the next frame does not
|
||||
# collide with the last gap token.
|
||||
frames = group["video_frames"]
|
||||
first_frame = frames[0]
|
||||
last_frame = frames[-1]
|
||||
|
||||
# group_start = <im_start> of first frame
|
||||
group_start = first_frame["thumbnail"][0] - 1
|
||||
# group_end = after last structural token of last frame
|
||||
group_end = _group_end_pos(last_frame, s_end)
|
||||
|
||||
# text before video group
|
||||
text_len = group_start - cursor
|
||||
if text_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(text_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:group_start] = t_pos
|
||||
pos += text_len
|
||||
|
||||
# process each frame with its own per-frame base
|
||||
frame_cursor = group_start
|
||||
for vf in frames:
|
||||
t_bs = vf["thumbnail"][0]
|
||||
vf_slices = vf["slices"]
|
||||
im_start_pos = t_bs - 1
|
||||
|
||||
# --- inter-frame gap: pure 1D text T=H=W=pos ---
|
||||
gap_len = im_start_pos - frame_cursor
|
||||
if gap_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(gap_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, frame_cursor:im_start_pos] = t_pos
|
||||
pos += gap_len
|
||||
pos += 1 # buffer: prevents halo_lo collision
|
||||
|
||||
# --- per-frame base ---
|
||||
base_f = pos
|
||||
frame_end = _group_end_pos(vf, group_end)
|
||||
pos = base_f + _assign_canvas_span(
|
||||
pos3d,
|
||||
thumbnail=vf["thumbnail"],
|
||||
span_end=frame_end,
|
||||
slices=vf_slices,
|
||||
base=base_f,
|
||||
target_sizes=target_sizes,
|
||||
)
|
||||
frame_cursor = frame_end
|
||||
|
||||
# trailing tokens after last frame (if any)
|
||||
if frame_cursor < group_end:
|
||||
trail_len = group_end - frame_cursor
|
||||
t_pos = (
|
||||
torch.arange(trail_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, frame_cursor:group_end] = t_pos
|
||||
pos += trail_len
|
||||
|
||||
cursor = group_end
|
||||
|
||||
else:
|
||||
# === Single image group (thumbnail + optional slices) ===
|
||||
slices = group["slices"]
|
||||
|
||||
group_start = group["thumbnail"][0] - 1
|
||||
group_end = _group_end_pos(group, s_end)
|
||||
|
||||
# text before group
|
||||
text_len = group_start - cursor
|
||||
if text_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(text_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:group_start] = t_pos
|
||||
pos += text_len
|
||||
|
||||
base = pos
|
||||
pos = base + _assign_canvas_span(
|
||||
pos3d,
|
||||
thumbnail=group["thumbnail"],
|
||||
span_end=group_end,
|
||||
slices=slices,
|
||||
base=base,
|
||||
target_sizes=target_sizes,
|
||||
)
|
||||
cursor = group_end
|
||||
|
||||
# remaining text
|
||||
rem = s_end - cursor
|
||||
if rem > 0:
|
||||
t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:s_end] = t_pos
|
||||
|
||||
return pos3d
|
||||
|
||||
|
||||
def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
|
||||
"""Offset applied to the 1-D positions of tokens decoded after the prompt.
|
||||
|
||||
The first decoded token sits one past the last prefill position.
|
||||
"""
|
||||
return int(position_ids.amax().item()) + 1 - seq_len
|
||||
@@ -3,7 +3,7 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal, cast
|
||||
from typing import Annotated, Any, Literal, TypeAlias, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -76,6 +76,19 @@ class Mistral3ImagePixelInputs(TensorSchema):
|
||||
]
|
||||
|
||||
|
||||
class Mistral3ImageEmbeddingInputs(TensorSchema):
|
||||
"""Projected features with shape (num_images, feature_size, hidden_size)."""
|
||||
|
||||
type: Literal["image_embeds"] = "image_embeds"
|
||||
data: Annotated[
|
||||
torch.Tensor | list[torch.Tensor],
|
||||
TensorShape("bn", "ifs", "hs", dynamic_dims={"ifs"}),
|
||||
]
|
||||
|
||||
|
||||
Mistral3ImageInputs: TypeAlias = Mistral3ImagePixelInputs | Mistral3ImageEmbeddingInputs
|
||||
|
||||
|
||||
class Mistral3PatchMerger(nn.Module):
|
||||
"""Learned merging of spatial_merge_size ** 2 patches."""
|
||||
|
||||
@@ -494,7 +507,7 @@ class Mistral3ForConditionalGeneration(
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> Mistral3ImagePixelInputs | None:
|
||||
) -> Mistral3ImageInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
image_embeds = kwargs.pop("image_embeds", None)
|
||||
|
||||
@@ -508,8 +521,8 @@ class Mistral3ForConditionalGeneration(
|
||||
|
||||
def _process_image_input(
|
||||
self,
|
||||
image_input: Mistral3ImagePixelInputs,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
image_input: Mistral3ImageInputs,
|
||||
) -> MultiModalEmbeddings:
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["data"]
|
||||
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .utils import maybe_prefix
|
||||
|
||||
|
||||
class MLPSpeculatorLayerNorm(nn.Module):
|
||||
"""A L2 normalization implementation
|
||||
...
|
||||
|
||||
Args:
|
||||
----
|
||||
normalized_shape : int
|
||||
Dimensionality of input data (size of final tensor axis)
|
||||
eps : float
|
||||
Safety term to prevent division by zero. Make sure the chosen value
|
||||
fits in the range of your encoding scheme
|
||||
(i.e. fp16 requires eps >= 6e-8).
|
||||
elementwise_scale_and_shift : bool
|
||||
Include a learned scaling and shift term after normalization.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
normalized_shape,
|
||||
eps=1e-06,
|
||||
elementwise_scale_and_shift=True,
|
||||
):
|
||||
super().__init__()
|
||||
self.elementwise_scale_and_shift = elementwise_scale_and_shift
|
||||
if self.elementwise_scale_and_shift:
|
||||
self.weight = nn.Parameter(torch.empty(normalized_shape))
|
||||
self.bias = nn.Parameter(torch.empty(normalized_shape))
|
||||
self.eps = eps
|
||||
|
||||
def forward(self, x):
|
||||
xf = x
|
||||
xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
|
||||
x = xf.type_as(x)
|
||||
if self.elementwise_scale_and_shift:
|
||||
x = self.weight * x
|
||||
x = x + self.bias
|
||||
return x
|
||||
|
||||
|
||||
class MLPSpeculator(nn.Module):
|
||||
"""An implementation of the speculative models introduced in
|
||||
"Accelerating Production LLMs with Combined Token/Embedding
|
||||
Speculators"
|
||||
https://arxiv.org/pdf/2404.19124
|
||||
|
||||
Trained speculators of this type are available on HF hub at:
|
||||
https://huggingface.co/ibm-ai-platform and https://huggingface.co/ibm-granite
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.n_predict = config.n_predict
|
||||
self.vocab_size = config.vocab_size
|
||||
self.emb_dim = config.emb_dim
|
||||
self.inner_dim = config.inner_dim if config.inner_dim != 0 else config.emb_dim
|
||||
|
||||
self.max_speculative_tokens = config.num_lookahead_tokens
|
||||
|
||||
self.tie_weights = config.tie_weights
|
||||
self.scale_input = config.scale_input
|
||||
|
||||
if self.tie_weights:
|
||||
assert self.n_predict > 1, (
|
||||
"You cannot tie weights between stages when only 1 exists"
|
||||
)
|
||||
embedding = VocabParallelEmbedding(
|
||||
config.vocab_size, self.inner_dim, org_num_embeddings=config.vocab_size
|
||||
)
|
||||
self.emb = nn.ModuleList([embedding] * self.max_speculative_tokens)
|
||||
|
||||
# the initial projection from the base model may
|
||||
# have a different size, so that stays separate.
|
||||
proj_first = nn.Linear(self.emb_dim, self.inner_dim, bias=False)
|
||||
proj_tied = nn.Linear(self.inner_dim, self.inner_dim, bias=False)
|
||||
self.proj = nn.ModuleList(
|
||||
[proj_first] + [proj_tied] * (self.max_speculative_tokens - 1)
|
||||
)
|
||||
|
||||
self.head = nn.ModuleList(
|
||||
[
|
||||
ParallelLMHead(
|
||||
self.vocab_size,
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, f"head.{i}"),
|
||||
)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
|
||||
ln = MLPSpeculatorLayerNorm(
|
||||
self.inner_dim, elementwise_scale_and_shift=True
|
||||
)
|
||||
self.ln = nn.ModuleList([ln] * self.max_speculative_tokens)
|
||||
|
||||
else:
|
||||
self.emb = nn.ModuleList(
|
||||
[
|
||||
VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
self.inner_dim,
|
||||
)
|
||||
for _ in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
|
||||
self.proj = nn.ModuleList(
|
||||
[
|
||||
nn.Linear(
|
||||
(self.emb_dim if i == 0 else self.inner_dim),
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
|
||||
self.head = nn.ModuleList(
|
||||
[
|
||||
ParallelLMHead(
|
||||
self.vocab_size,
|
||||
self.inner_dim,
|
||||
bias=False,
|
||||
prefix=maybe_prefix(prefix, f"head.{i}"),
|
||||
)
|
||||
for i in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
self.ln = nn.ModuleList(
|
||||
[
|
||||
MLPSpeculatorLayerNorm(
|
||||
self.inner_dim, elementwise_scale_and_shift=True
|
||||
)
|
||||
for _ in range(self.max_speculative_tokens)
|
||||
]
|
||||
)
|
||||
if self.scale_input:
|
||||
self.ln0 = MLPSpeculatorLayerNorm(
|
||||
self.emb_dim, elementwise_scale_and_shift=False
|
||||
)
|
||||
|
||||
self.state_weight = 0.5 ** (0.5 / config.n_predict)
|
||||
self.emb_weight = math.sqrt((1 - self.state_weight**2) * (self.inner_dim / 2))
|
||||
self.activation = nn.GELU()
|
||||
self.config = config
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.vocab_size, config.vocab_size, 1.0
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
name = name.replace("speculator.", "")
|
||||
param = params_dict.get(name)
|
||||
if param is not None:
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
@@ -1001,6 +1001,16 @@ class NemotronH_Nano_VL_V2(
|
||||
vision_projection_hidden_size = config.projector_hidden_size
|
||||
llm_hidden_size = config.text_config.hidden_size
|
||||
|
||||
# Match the HF vision projector for checkpoints trained with MTP.
|
||||
self.vision_final_layernorm = (
|
||||
nn.LayerNorm(
|
||||
vit_hidden_size,
|
||||
eps=getattr(vision_config, "layer_norm_eps", 1e-6),
|
||||
).to(llm_dtype)
|
||||
if (getattr(config.text_config, "num_nextn_predict_layers", 0) or 0) > 0
|
||||
else None
|
||||
)
|
||||
|
||||
mlp1 = nn.Sequential(
|
||||
RMSNorm(
|
||||
hidden_size=vit_hidden_size
|
||||
@@ -1095,6 +1105,8 @@ class NemotronH_Nano_VL_V2(
|
||||
"""Dynamic resolution extract_feature for images."""
|
||||
_, vit_embeds = self.vision_model(pixel_values, imgs_sizes=imgs_sizes)
|
||||
vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
|
||||
if self.vision_final_layernorm is not None:
|
||||
vit_embeds = self.vision_final_layernorm(vit_embeds)
|
||||
vit_embeds = self.pixel_shuffle_dynamic_res(vit_embeds, imgs_sizes=imgs_sizes)
|
||||
vit_embeds = self.mlp1(vit_embeds)
|
||||
return vit_embeds
|
||||
@@ -1127,6 +1139,8 @@ class NemotronH_Nano_VL_V2(
|
||||
else:
|
||||
_, vit_embeds = self.vision_model(chunk)
|
||||
vit_embeds = vit_embeds.to(dtype=torch.bfloat16)
|
||||
if self.vision_final_layernorm is not None:
|
||||
vit_embeds = self.vision_final_layernorm(vit_embeds)
|
||||
vit_embeds = vit_embeds.reshape(
|
||||
vit_embeds.shape[0], H_patches, W_patches, -1
|
||||
)
|
||||
@@ -1575,7 +1589,7 @@ class NemotronH_Nano_VL_V2(
|
||||
"""Get the module prefix in multimodal models"""
|
||||
return MultiModelKeys.from_string_field(
|
||||
language_model="language_model",
|
||||
connector=["mlp1", "sound_encoder.projection"],
|
||||
connector=["mlp1", "sound_encoder.projection", "vision_final_layernorm"],
|
||||
tower_model=["vision_model", "sound_encoder.encoder"],
|
||||
)
|
||||
|
||||
@@ -1593,6 +1607,11 @@ class NemotronH_Nano_VL_V2(
|
||||
for modality in ("image", "video", "audio")
|
||||
)
|
||||
adapter_dict = dict(self.mlp1.named_parameters())
|
||||
vision_norm_params = (
|
||||
dict(self.vision_final_layernorm.named_parameters())
|
||||
if load_multimodal_weights and self.vision_final_layernorm is not None
|
||||
else {}
|
||||
)
|
||||
|
||||
def is_llm(name: str) -> bool:
|
||||
return name.startswith("language_model")
|
||||
@@ -1606,6 +1625,11 @@ class NemotronH_Nano_VL_V2(
|
||||
def is_sound_weights(name: str) -> bool:
|
||||
return name.startswith("sound")
|
||||
|
||||
vision_norm_prefix = "vision_projector.vision_final_layernorm."
|
||||
|
||||
def is_vision_norm_weights(name: str) -> bool:
|
||||
return name.startswith(vision_norm_prefix)
|
||||
|
||||
# LLM weights (the bulk of the model) are streamed lazily through a
|
||||
# generator so each tensor is copied into its parameter before the
|
||||
# iterator advances, avoiding stale-reference corruption with
|
||||
@@ -1637,6 +1661,15 @@ class NemotronH_Nano_VL_V2(
|
||||
continue
|
||||
assert self.sound_encoder is not None
|
||||
sound_weights.append((name, w.detach().clone()))
|
||||
elif is_vision_norm_weights(name):
|
||||
if not load_multimodal_weights:
|
||||
continue
|
||||
param_name = name.removeprefix(vision_norm_prefix)
|
||||
assert param_name in vision_norm_params, (
|
||||
f"Unexpected vision LayerNorm weight: {name}"
|
||||
)
|
||||
with torch.no_grad():
|
||||
default_weight_loader(vision_norm_params[param_name], w)
|
||||
|
||||
# Fully drain the generator so every mm tensor is buffered, even if
|
||||
# the LLM loader stops iterating early.
|
||||
|
||||
@@ -74,7 +74,6 @@ from vllm.model_executor.models.interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsReplaySSM,
|
||||
@@ -585,6 +584,8 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
|
||||
self.has_moe = "E" in config.hybrid_override_pattern
|
||||
@@ -725,7 +726,6 @@ class NemotronHForCausalLM(
|
||||
IsHybrid,
|
||||
SupportsQuant,
|
||||
MixtureOfExperts,
|
||||
SupportsMambaPrefixCaching,
|
||||
SupportsReplaySSM,
|
||||
):
|
||||
# Relevant only if self.has_moe is True
|
||||
|
||||
@@ -9,11 +9,13 @@
|
||||
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
from typing import Annotated, Any, Literal, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from PIL import Image
|
||||
from transformers import (
|
||||
BartConfig,
|
||||
BatchFeature,
|
||||
@@ -26,6 +28,7 @@ from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -401,6 +404,56 @@ class NemotronParseDummyInputsBuilder(
|
||||
}
|
||||
|
||||
|
||||
def _to_pil_image(image: object) -> Image.Image:
|
||||
"""Convert a `uint8` array/tensor image item to PIL, since the checkpoint's
|
||||
image processor calls `.convert("RGB")` on every input."""
|
||||
if isinstance(image, Image.Image):
|
||||
return image
|
||||
|
||||
array = image.numpy() if isinstance(image, torch.Tensor) else np.asarray(image)
|
||||
if array.dtype != np.uint8 or array.ndim != 3:
|
||||
raise ValueError(
|
||||
"Nemotron-Parse expects image arrays/tensors to be 3-D uint8, "
|
||||
f"got {array.dtype} {array.shape}"
|
||||
)
|
||||
# Same layout heuristic as `ImageProcessorItems.get_image_size`.
|
||||
if array.shape[-1] not in (1, 3, 4):
|
||||
array = array.transpose(1, 2, 0)
|
||||
if array.shape[-1] == 1:
|
||||
array = array[..., 0]
|
||||
|
||||
return Image.fromarray(np.ascontiguousarray(array))
|
||||
|
||||
|
||||
def _build_uint8_canvas(
|
||||
image_processor: Any, images: Sequence[Image.Image]
|
||||
) -> torch.Tensor:
|
||||
"""Reproduce the image processor's pipeline up to, but excluding, its
|
||||
`ToTensor` rescale, yielding the padded `uint8` canvas as `(N, 3, H, W)`.
|
||||
|
||||
Raises `AttributeError`/`TypeError` if the checkpoint's remote code does
|
||||
not expose that pipeline.
|
||||
"""
|
||||
resize = image_processor._resize_with_aspect_ratio
|
||||
pad = image_processor._pad_to_size
|
||||
# v1.x pads with albumentations, 2.0 sets `transform` to None and pads
|
||||
# with `_pad_to_size`; mirror `preprocess` and pick the same branch.
|
||||
transform = image_processor.transform
|
||||
|
||||
canvases = []
|
||||
for image in images:
|
||||
array = np.asarray(image.convert("RGB"))
|
||||
array = resize(array)
|
||||
array = transform(image=array)["image"] if transform is not None else pad(array)
|
||||
if array.dtype != np.uint8 or array.ndim != 3 or array.shape[2] != 3:
|
||||
raise TypeError(
|
||||
f"expected a uint8 HWC canvas, got {array.dtype} {array.shape}"
|
||||
)
|
||||
canvases.append(torch.from_numpy(np.ascontiguousarray(array)).permute(2, 0, 1))
|
||||
|
||||
return torch.stack(canvases)
|
||||
|
||||
|
||||
class NemotronParseMultiModalProcessor(
|
||||
EncDecMultiModalProcessor[NemotronParseProcessingInfo]
|
||||
):
|
||||
@@ -411,6 +464,35 @@ class NemotronParseMultiModalProcessor(
|
||||
) -> list[int]:
|
||||
return [0]
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
images = [_to_pil_image(i) for i in cast(list, hf_data.get("images") or [])]
|
||||
if images:
|
||||
hf_data = {**hf_data, "images": images}
|
||||
|
||||
# The checkpoint ships its image processor as remote code, and its
|
||||
# `preprocess` ignores `do_rescale`/`do_normalize`: it always runs
|
||||
# `ToTensor` and normalises from `self.do_normalize`. Build the uint8
|
||||
# canvas here so the device-side norm is not applied on top.
|
||||
mm_config = self.info.ctx.model_config.get_multimodal_config()
|
||||
if images and set(hf_data) == {"images"} and mm_config.mm_device_do_normalize:
|
||||
image_processor = self.info.get_hf_processor().image_processor
|
||||
try:
|
||||
pixel_values = _build_uint8_canvas(image_processor, images)
|
||||
except (AttributeError, TypeError) as exc:
|
||||
raise RuntimeError(
|
||||
f"{type(image_processor).__name__} does not expose the "
|
||||
f"expected pre-rescale pipeline ({exc}), so the uint8 canvas "
|
||||
"that mm_device_do_normalize requires cannot be built. Pass "
|
||||
"--no-mm-device-do-normalize to normalise on the host instead."
|
||||
) from exc
|
||||
return BatchFeature({"pixel_values": pixel_values})
|
||||
|
||||
return super()._call_hf_processor(hf_data, hf_kwargs)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
@@ -542,6 +624,8 @@ class RadioWithNeck(nn.Module):
|
||||
dummy_inputs=NemotronParseDummyInputsBuilder,
|
||||
)
|
||||
class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
@@ -550,6 +634,7 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
self.vision_config = config.encoder
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.input_norm = build_mm_input_norm(vllm_config.model_config)
|
||||
|
||||
with self._mark_tower_model(vllm_config, "image"):
|
||||
self.encoder = RadioWithNeck(
|
||||
@@ -622,8 +707,8 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
assert image_input["type"] == "pixel_values"
|
||||
pixel_values = image_input["data"]
|
||||
dtype = next(self.encoder.parameters()).dtype
|
||||
pixel_values = pixel_values.to(dtype)
|
||||
return self.encoder(pixel_values)
|
||||
flat = self.input_norm(pixel_values.flatten(1), dtype)
|
||||
return self.encoder(flat.view(pixel_values.shape))
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
@@ -258,28 +258,18 @@ class PaddleOCRVLMultiModalProcessor(
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if not hf_data:
|
||||
return processed_data
|
||||
|
||||
num_patches_per_image = processed_data["image_grid_thw"].prod(-1)
|
||||
processed_data["pixel_values"] = processed_data["pixel_values"].split(
|
||||
num_patches_per_image.tolist()
|
||||
)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
|
||||
image_grid_sizes = image_grid_thw.prod(-1)
|
||||
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.batched("image"),
|
||||
pixel_values=MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", image_grid_sizes
|
||||
),
|
||||
image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
@@ -347,12 +337,12 @@ class Projector(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
m1, m2 = self.merge_kernel_size
|
||||
if isinstance(image_features, (list, tuple)):
|
||||
processed_features = list()
|
||||
for image_feature, image_grid in zip(image_features, image_grid_thw):
|
||||
image_feature = self.pre_norm(image_feature)
|
||||
t, h, w = image_grid
|
||||
|
||||
image_feature = rearrange(
|
||||
# Norm and MLP are per-token, so run them once over all images;
|
||||
# only the 2x2 merge depends on each image's grid.
|
||||
sizes = [image_feature.shape[0] for image_feature in image_features]
|
||||
normed_features = self.pre_norm(torch.cat(image_features)).split(sizes)
|
||||
merged_features = [
|
||||
rearrange(
|
||||
image_feature,
|
||||
"(t h p1 w p2) d -> (t h w) (p1 p2 d)",
|
||||
t=t,
|
||||
@@ -361,12 +351,13 @@ class Projector(nn.Module):
|
||||
w=w // m2,
|
||||
p2=m2,
|
||||
)
|
||||
hidden_states = self.linear_1(image_feature)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states = self.linear_2(hidden_states)
|
||||
processed_features.append(hidden_states)
|
||||
for image_feature, (t, h, w) in zip(normed_features, image_grid_thw)
|
||||
]
|
||||
hidden_states = self.linear_1(torch.cat(merged_features))
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states = self.linear_2(hidden_states)
|
||||
|
||||
return processed_features
|
||||
return hidden_states.split([f.shape[0] for f in merged_features])
|
||||
|
||||
dims = image_features.shape[:-1]
|
||||
dim = image_features.shape[-1]
|
||||
@@ -383,7 +374,7 @@ class PaddleOCRImagePixelInputs(TensorSchema):
|
||||
type: Literal["pixel_values"]
|
||||
pixel_values: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("bn", "p", 3, "patch_size", "patch_size", dynamic_dims={"p"}),
|
||||
TensorShape("np", 3, "patch_size", "patch_size"),
|
||||
]
|
||||
image_grid_thw: Annotated[
|
||||
torch.Tensor,
|
||||
@@ -455,27 +446,6 @@ class SiglipVisionEmbeddings(nn.Module):
|
||||
patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
|
||||
return patch_pos_embed
|
||||
|
||||
def fetch_position_embedding_lfu_cache(
|
||||
self, embeddings: torch.Tensor, h: int, w: int, max_cache: int = 20
|
||||
):
|
||||
grid = (h, w)
|
||||
if grid in self.cache_position_embedding:
|
||||
self.cache_position_count[grid] += 1
|
||||
return self.cache_position_embedding[grid]
|
||||
|
||||
if len(self.cache_position_embedding) >= max_cache:
|
||||
min_hit_grid = min(
|
||||
self.cache_position_count,
|
||||
key=self.cache_position_count.__getitem__,
|
||||
)
|
||||
self.cache_position_count.pop(min_hit_grid)
|
||||
self.cache_position_embedding.pop(min_hit_grid)
|
||||
|
||||
position_embedding = self.interpolate_pos_encoding(embeddings, h, w, True)
|
||||
self.cache_position_count[grid] = 1
|
||||
self.cache_position_embedding[grid] = position_embedding
|
||||
return position_embedding
|
||||
|
||||
def forward(
|
||||
self,
|
||||
pixel_values: torch.FloatTensor,
|
||||
@@ -811,10 +781,10 @@ class SiglipEncoder(nn.Module):
|
||||
if cu_seqlens is None:
|
||||
raise ValueError("cu_seqlens cannot be None for SiglipEncoder.")
|
||||
if not isinstance(cu_seqlens, torch.Tensor):
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device)
|
||||
else:
|
||||
cu_seqlens = cu_seqlens.to(device=device, non_blocking=True)
|
||||
cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32)
|
||||
|
||||
# Computed before the H2D copy: the attention wrappers call `.item()`
|
||||
# on it, which would sync on every layer for a device tensor.
|
||||
max_seqlen = None
|
||||
if self.attn_backend in {
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
@@ -823,6 +793,11 @@ class SiglipEncoder(nn.Module):
|
||||
}:
|
||||
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
|
||||
|
||||
if cu_seqlens.is_cpu:
|
||||
cu_seqlens = async_tensor_h2d(cu_seqlens, dtype=torch.int32, device=device)
|
||||
else:
|
||||
cu_seqlens = cu_seqlens.to(device=device, non_blocking=True)
|
||||
|
||||
hidden_states = inputs_embeds
|
||||
for encoder_layer in self.layers:
|
||||
hidden_states = encoder_layer(
|
||||
@@ -1125,55 +1100,43 @@ class PaddleOCRVLForConditionalGeneration(nn.Module, SupportsMultiModal, Support
|
||||
)
|
||||
|
||||
def encode_image(
|
||||
self, pixel_values: torch.Tensor, image_grid_thw: torch.Tensor
|
||||
self,
|
||||
pixel_values: torch.Tensor,
|
||||
image_grid_thw: list[tuple[int, int, int]],
|
||||
) -> torch.Tensor:
|
||||
"""Encode the flat patches of all images in one vision tower call."""
|
||||
pixel_values = pixel_values.type(self.visual.dtype)
|
||||
siglip_position_ids = list()
|
||||
image_grid_hws = list()
|
||||
cu_seqlens = [0]
|
||||
seq_lens = [math.prod(grid) for grid in image_grid_thw]
|
||||
|
||||
thw_tuple = tuple(image_grid_thw.tolist())
|
||||
numel = np.prod(thw_tuple)
|
||||
image_grid_hws.append(thw_tuple)
|
||||
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
|
||||
siglip_position_ids.append(image_position_ids)
|
||||
cu_seqlens.append(cu_seqlens[-1] + numel)
|
||||
|
||||
# Both are built on the host; concat straight into a pinned buffer
|
||||
# so the H2D copy stays non-blocking.
|
||||
# Built on the host; concat straight into a pinned buffer so the
|
||||
# H2D copy stays non-blocking.
|
||||
siglip_position_ids = torch.concat(
|
||||
siglip_position_ids,
|
||||
[
|
||||
torch.arange(seq_len) % (h * w)
|
||||
for seq_len, (_, h, w) in zip(seq_lens, image_grid_thw)
|
||||
],
|
||||
dim=0,
|
||||
out=torch.empty(
|
||||
sum(t.numel() for t in siglip_position_ids),
|
||||
dtype=torch.int64,
|
||||
pin_memory=PIN_MEMORY,
|
||||
),
|
||||
out=torch.empty(sum(seq_lens), dtype=torch.int64, pin_memory=PIN_MEMORY),
|
||||
).to(pixel_values.device, non_blocking=True)
|
||||
cu_seqlens = async_tensor_h2d(
|
||||
cu_seqlens, dtype=torch.int32, device=pixel_values.device
|
||||
)
|
||||
# One segment per image so that images never attend to each other.
|
||||
cu_seqlens = torch.from_numpy(np.cumsum([0, *seq_lens], dtype=np.int32))
|
||||
|
||||
vision_outputs = self.visual(
|
||||
return self.visual(
|
||||
pixel_values=pixel_values,
|
||||
image_grid_thw=image_grid_hws,
|
||||
image_grid_thw=image_grid_thw,
|
||||
position_ids=siglip_position_ids,
|
||||
interpolate_pos_encoding=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
return vision_outputs
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: PaddleOCRImagePixelInputs
|
||||
) -> MultiModalEmbeddings:
|
||||
pixel_values = image_input.pixel_values
|
||||
image_grid_thw = image_input.image_grid_thw
|
||||
vision_outputs = tuple(
|
||||
self.encode_image(pixel, grid).squeeze(0)
|
||||
for pixel, grid in zip(pixel_values, image_grid_thw)
|
||||
)
|
||||
image_embeds = self.mlp_AR(vision_outputs, image_grid_thw)
|
||||
return image_embeds
|
||||
grids = [tuple(grid) for grid in image_grid_thw.tolist()]
|
||||
vision_outputs = self.encode_image(image_input.pixel_values, grids).squeeze(0)
|
||||
vision_outputs = vision_outputs.split([math.prod(grid) for grid in grids])
|
||||
return self.mlp_AR(vision_outputs, image_grid_thw)
|
||||
|
||||
def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
@@ -1256,12 +1256,6 @@ class AudioEmbedding(nn.Module):
|
||||
self.input_embeds = None
|
||||
self.audio_embed_sizes = None
|
||||
|
||||
def set_audio_embeds(self, input_embeds: torch.Tensor) -> None:
|
||||
self.input_embeds = input_embeds
|
||||
|
||||
def set_audio_embed_sizes(self, audio_embed_sizes: torch.Tensor) -> None:
|
||||
self.audio_embed_sizes = audio_embed_sizes
|
||||
|
||||
def get_audio_features(
|
||||
self,
|
||||
input_embeds: torch.Tensor,
|
||||
|
||||
@@ -132,11 +132,11 @@ def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tenso
|
||||
|
||||
|
||||
def unpad_and_flat_audio_features(
|
||||
input_audio_features: torch.Tensor,
|
||||
input_audio_features: torch.Tensor | Sequence[torch.Tensor],
|
||||
audio_feature_lengths: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Unpad and flatten batched audio features."""
|
||||
if len(audio_feature_lengths) != input_audio_features.size(0):
|
||||
"""Unpad and flatten stacked or ragged per-audio features."""
|
||||
if len(audio_feature_lengths) != len(input_audio_features):
|
||||
raise ValueError(
|
||||
"Length of audio_feature_lengths must match "
|
||||
"the batch size of input_audio_features."
|
||||
@@ -920,8 +920,10 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
|
||||
# inputs features from rust frontend is batched and padded
|
||||
# with shape [batch_size, n_mels, padded_seq_len], different
|
||||
# from python's shape [n_mels, batch_size * seq_len]
|
||||
if (
|
||||
# from python's shape [n_mels, batch_size * seq_len].
|
||||
# Requests with different padded lengths are batched as a
|
||||
# list of [n_mels, padded_seq_len] tensors.
|
||||
if isinstance(input_audio_features, (list, tuple)) or (
|
||||
isinstance(input_audio_features, torch.Tensor)
|
||||
and input_audio_features.dim() == 3
|
||||
):
|
||||
|
||||
@@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module):
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||
per_layer_sliding_window: int | None = None,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module):
|
||||
max_position=max_position,
|
||||
rope_parameters=rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
attn_cls = (
|
||||
EncoderOnlyAttention
|
||||
@@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
per_layer_sliding_window: int | None = None,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
@@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module):
|
||||
attn_type=attn_type,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
per_layer_sliding_window=per_layer_sliding_window,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
self.mlp = Qwen3MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
@@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = {
|
||||
}
|
||||
)
|
||||
class Qwen3Model(Qwen2Model):
|
||||
decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
decoder_layer_type=self.decoder_layer_type,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -91,6 +91,7 @@ from .qwen3_vl import (
|
||||
Qwen3VLForConditionalGeneration,
|
||||
Qwen3VLMultiModalProcessor,
|
||||
Qwen3VLProcessingInfo,
|
||||
mrope_positions_factor,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -161,6 +162,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
||||
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid layer_type {self.layer_type}")
|
||||
|
||||
@@ -29,6 +29,7 @@ import regex as re
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.models.qwen3_asr import Qwen3ASRFeatureExtractor, Qwen3ASRProcessor
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
@@ -54,7 +55,6 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
|
||||
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeAudioEncoder,
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
@@ -80,6 +80,7 @@ from vllm.multimodal.parse import (
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
@@ -92,9 +93,6 @@ from vllm.transformers_utils.configs.qwen3_asr import (
|
||||
Qwen3ASRThinkerConfig,
|
||||
)
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.processors.qwen3_asr import (
|
||||
Qwen3ASRProcessor,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
@@ -188,18 +186,19 @@ class Qwen3ASRProcessingInfo(BaseProcessingInfo):
|
||||
return self.ctx.get_hf_config(Qwen3ASRConfig).thinker_config
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> Qwen3ASRProcessor:
|
||||
processor = self.ctx.get_hf_processor(
|
||||
return self.ctx.get_hf_processor(
|
||||
Qwen3ASRProcessor,
|
||||
**kwargs,
|
||||
)
|
||||
if not hasattr(processor, "audio_token"):
|
||||
processor.audio_token = "<|audio_pad|>"
|
||||
return processor
|
||||
|
||||
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
|
||||
def get_feature_extractor(
|
||||
self, **kwargs: object
|
||||
) -> WhisperFeatureExtractor | Qwen3ASRFeatureExtractor:
|
||||
hf_processor = self.get_hf_processor(**kwargs)
|
||||
feature_extractor = hf_processor.feature_extractor
|
||||
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
||||
assert isinstance(
|
||||
feature_extractor, (WhisperFeatureExtractor, Qwen3ASRFeatureExtractor)
|
||||
)
|
||||
return feature_extractor
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -275,8 +274,60 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
|
||||
|
||||
|
||||
class Qwen3ASRMultiModalProcessor(
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
|
||||
BaseMultiModalProcessor[Qwen3ASRProcessingInfo],
|
||||
):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
audios = hf_data.get("audio")
|
||||
if not audios:
|
||||
return BatchFeature()
|
||||
assert isinstance(audios, list)
|
||||
|
||||
if len(audios) == 1:
|
||||
return super()._call_hf_processor(hf_data, hf_kwargs)
|
||||
|
||||
# HF requires one text per audio, and batch padding changes clip features.
|
||||
# Keep clips independent so partial cache hits cannot change their inputs.
|
||||
text = self._get_hf_mm_text({"audio": 1})
|
||||
outputs: dict[str, list[torch.Tensor]] = {
|
||||
"input_features": [],
|
||||
"input_features_mask": [],
|
||||
}
|
||||
for audio in audios:
|
||||
processed = super()._call_hf_processor(
|
||||
dict(hf_data, text=text, audio=audio), hf_kwargs
|
||||
)
|
||||
for key, values in outputs.items():
|
||||
values.append(processed[key][0])
|
||||
|
||||
return BatchFeature(outputs)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "input_features" not in processed_data:
|
||||
return processed_data
|
||||
|
||||
features = processed_data.pop("input_features")
|
||||
masks = processed_data.pop("input_features_mask")
|
||||
processed_data["input_audio_features"] = torch.cat(
|
||||
[feature[:, mask.bool()] for feature, mask in zip(features, masks)], dim=1
|
||||
)
|
||||
processed_data["feature_attention_mask"] = masks
|
||||
processed_data["audio_feature_lengths"] = torch.stack(
|
||||
[mask.sum() for mask in masks]
|
||||
)
|
||||
return processed_data
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
@@ -427,8 +478,10 @@ class Qwen3ASRForConditionalGeneration(
|
||||
|
||||
# inputs features from rust frontend is batched and padded
|
||||
# with shape [batch_size, n_mels, padded_seq_len], different
|
||||
# from python's shape [n_mels, batch_size * seq_len]
|
||||
if (
|
||||
# from python's shape [n_mels, batch_size * seq_len].
|
||||
# Requests with different padded lengths are batched as a
|
||||
# list of [n_mels, padded_seq_len] tensors.
|
||||
if isinstance(input_audio_features, (list, tuple)) or (
|
||||
isinstance(input_audio_features, torch.Tensor)
|
||||
and input_audio_features.dim() == 3
|
||||
):
|
||||
|
||||
@@ -14,7 +14,6 @@ from vllm import _custom_ops as ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
@@ -33,6 +32,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import make_attention_sink
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
@@ -251,9 +251,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
)
|
||||
|
||||
self.attention_sink_bias = (
|
||||
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
|
||||
if add_swa_attention_sink_bias
|
||||
else None
|
||||
make_attention_sink(self.num_heads) if add_swa_attention_sink_bias else None
|
||||
)
|
||||
|
||||
self.sliding_window = sliding_window
|
||||
@@ -746,26 +744,9 @@ class DFlashQwen3Model(nn.Module):
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
def _preprocess(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
for name, loaded_weight in weights:
|
||||
if "attention_sink_bias" in name:
|
||||
# Sink bias is per-head; shard it across TP ranks like the
|
||||
# attention heads themselves.
|
||||
heads_per_rank = loaded_weight.shape[0] // tp_size
|
||||
loaded_weight = loaded_weight.narrow(
|
||||
0, tp_rank * heads_per_rank, heads_per_rank
|
||||
)
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self._preprocess(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
|
||||
@@ -256,6 +256,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -303,6 +304,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
attention_kwargs: dict[str, Any] = {}
|
||||
if dual_chunk_attention_config:
|
||||
@@ -351,6 +353,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -376,6 +379,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
|
||||
# `mlp_only_layers` in the config.
|
||||
|
||||
@@ -129,7 +129,12 @@ def _should_replicate_misaligned_shared_expert(
|
||||
|
||||
|
||||
class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
@@ -231,6 +236,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
reduce_results=reduce_results,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_expert",
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
@@ -280,6 +286,7 @@ class Qwen3NextAttention(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -331,6 +338,7 @@ class Qwen3NextAttention(nn.Module):
|
||||
max_position=config.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
dual_chunk_attention_config=self.dual_chunk_attention_config,
|
||||
mrope_positions_factor=mrope_positions_factor,
|
||||
)
|
||||
|
||||
# Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL
|
||||
|
||||
@@ -83,6 +83,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.utils import copy_mm_embedding_modality
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -1498,8 +1499,12 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
|
||||
audio_in_video_item_idx += 1
|
||||
|
||||
# Match the fps the processor used, as HF and the mrope positions do.
|
||||
processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
|
||||
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
|
||||
if second_per_grid_ts:
|
||||
if isinstance(processed_second_per_grid_ts, torch.Tensor):
|
||||
video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item()
|
||||
elif second_per_grid_ts:
|
||||
video_second_per_grid_t = second_per_grid_ts[item_idx]
|
||||
else:
|
||||
video_second_per_grid_t = 2.0
|
||||
@@ -1904,7 +1909,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
embeddings_main, embeddings_multiscale = torch.split(
|
||||
embeddings, [visual_dim, multi_dim], dim=-1
|
||||
)
|
||||
multimodal_embeddings[index] = embeddings_main
|
||||
multimodal_embeddings[index] = copy_mm_embedding_modality(
|
||||
embeddings, embeddings_main
|
||||
)
|
||||
multimodal_embeddings_multiscale.append(embeddings_multiscale)
|
||||
if not is_interleaved:
|
||||
current_positions = mm_positions[
|
||||
|
||||
@@ -52,7 +52,7 @@ from vllm.compilation.decorators import (
|
||||
should_torch_compile_mm_encoder,
|
||||
support_torch_compile,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.multimodal import (
|
||||
MultiModalConfig,
|
||||
MultiModalDummyOptions,
|
||||
@@ -60,7 +60,7 @@ from vllm.config.multimodal import (
|
||||
VideoPruningMethod,
|
||||
)
|
||||
from vllm.distributed import get_pp_group, parallel_state
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalHashes
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
@@ -81,6 +81,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
@@ -94,6 +95,7 @@ from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
@@ -148,7 +150,7 @@ from .qwen2_vl import (
|
||||
Qwen2VLProcessingInfo,
|
||||
_create_qwen2vl_field_factory,
|
||||
)
|
||||
from .qwen3 import Qwen3ForCausalLM, Qwen3Model
|
||||
from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
@@ -1342,6 +1344,67 @@ def _replace_video_token_placeholders(
|
||||
|
||||
|
||||
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
|
||||
def _get_per_video_kwargs(
|
||||
self, kwargs: Mapping[str, object], num_videos: int | None = None
|
||||
) -> dict[str, list[Any]]:
|
||||
merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs)
|
||||
per_video_kwargs = {
|
||||
key: value
|
||||
for key, value in merged.get("videos_kwargs", {}).items()
|
||||
if key in ("fps", "num_frames") and isinstance(value, list)
|
||||
}
|
||||
if num_videos:
|
||||
for key, values in per_video_kwargs.items():
|
||||
if len(values) < num_videos:
|
||||
raise ValueError(
|
||||
f"{key} must have at least {num_videos} entries "
|
||||
"(one per video); "
|
||||
f"got {len(values)}."
|
||||
)
|
||||
return per_video_kwargs
|
||||
|
||||
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
|
||||
hashes = dict(super()._get_mm_hashes(inputs))
|
||||
per_video_kwargs = self._get_per_video_kwargs(
|
||||
inputs.hf_processor_mm_kwargs, len(hashes.get("video", []))
|
||||
)
|
||||
if per_video_kwargs and "video" in hashes:
|
||||
algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm
|
||||
hashes["video"] = [
|
||||
MultiModalHasher.hash_kwargs(
|
||||
algorithm,
|
||||
mm_hash=mm_hash,
|
||||
**{key: values[i] for key, values in per_video_kwargs.items()},
|
||||
)
|
||||
for i, mm_hash in enumerate(hashes["video"])
|
||||
]
|
||||
return hashes
|
||||
|
||||
def _get_cache_missing_processor_kwargs(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
mm_is_cached: Mapping[str, list[bool]],
|
||||
) -> Mapping[str, object]:
|
||||
per_video_kwargs = self._get_per_video_kwargs(
|
||||
inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", []))
|
||||
)
|
||||
if not per_video_kwargs or "video" not in mm_is_cached:
|
||||
return inputs.hf_processor_mm_kwargs
|
||||
kwargs = self.info._merge_and_resolve_mm_processor_kwargs(
|
||||
inputs.hf_processor_mm_kwargs
|
||||
)
|
||||
kwargs["videos_kwargs"].update(
|
||||
{
|
||||
key: [
|
||||
value
|
||||
for value, hit in zip(values, mm_is_cached["video"])
|
||||
if not hit
|
||||
]
|
||||
for key, values in per_video_kwargs.items()
|
||||
}
|
||||
)
|
||||
return kwargs
|
||||
|
||||
@staticmethod
|
||||
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
||||
"""Transformers>=5.10 processors override `replace_video_token`
|
||||
@@ -1412,6 +1475,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
if videos := hf_data.pop("videos", []):
|
||||
assert isinstance(videos, list)
|
||||
self._get_per_video_kwargs(hf_kwargs, len(videos))
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
timestamps_per_video = []
|
||||
@@ -1544,6 +1608,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
|
||||
non_video_scoped_kwargs.pop("fps", None)
|
||||
non_video_scoped_kwargs.pop("num_frames", None)
|
||||
# The context merges model defaults again when calling HF.
|
||||
for key in self._get_per_video_kwargs({}):
|
||||
non_video_scoped_kwargs[key] = None
|
||||
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||
@@ -1748,6 +1815,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
return PromptUpdateDetails.from_seq(all_token_ids)
|
||||
|
||||
|
||||
def mrope_positions_factor(vllm_config: VllmConfig) -> int:
|
||||
# Video pruning keeps the positions of the tokens it drops. A drafter gets
|
||||
# the target's positions, so it follows the target.
|
||||
spec = vllm_config.speculative_config
|
||||
model_config = (spec and spec.target_model_config) or vllm_config.model_config
|
||||
mm_config = model_config.multimodal_config
|
||||
pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled()
|
||||
return 4 if pruned else 1
|
||||
|
||||
|
||||
class Qwen3VLDecoderLayer(Qwen3DecoderLayer):
|
||||
def __init__(self, **kwargs) -> None:
|
||||
factor = mrope_positions_factor(get_current_vllm_config())
|
||||
super().__init__(**kwargs, mrope_positions_factor=factor)
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
@@ -1761,6 +1844,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
}
|
||||
)
|
||||
class Qwen3LLMModel(Qwen3Model):
|
||||
decoder_layer_type = Qwen3VLDecoderLayer
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
|
||||
@@ -56,6 +56,7 @@ from .qwen3_vl import (
|
||||
Qwen3VLForConditionalGeneration,
|
||||
Qwen3VLMultiModalProcessor,
|
||||
Qwen3VLProcessingInfo,
|
||||
mrope_positions_factor,
|
||||
)
|
||||
from .utils import StageMissingLayer, maybe_prefix
|
||||
|
||||
@@ -73,6 +74,7 @@ class Qwen3VLMoeDecoderLayer(Qwen3MoeDecoderLayer):
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
is_fused_checkpoint_transposed=True,
|
||||
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ from .interfaces import (
|
||||
is_attention_free,
|
||||
is_hybrid,
|
||||
requires_raw_input_tokens,
|
||||
supports_mamba_prefix_caching,
|
||||
supports_hc_sp,
|
||||
supports_multimodal,
|
||||
supports_multimodal_encoder_tp_data,
|
||||
supports_multimodal_raw_input_only,
|
||||
@@ -71,6 +71,7 @@ logger = init_logger(__name__)
|
||||
_TEXT_GENERATION_MODELS = {
|
||||
# [Decoder-only]
|
||||
"AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"),
|
||||
"AliceAIForCausalLM": ("alice_ai", "AliceAIForCausalLM"),
|
||||
"ApertusForCausalLM": ("apertus", "ApertusForCausalLM"),
|
||||
"AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"),
|
||||
"BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"),
|
||||
@@ -98,6 +99,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"ExaoneForCausalLM": ("exaone", "ExaoneForCausalLM"),
|
||||
"Exaone4ForCausalLM": ("exaone4", "Exaone4ForCausalLM"),
|
||||
"ExaoneMoeForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"ExaoneMoEForCausalLM": ("exaone_moe", "ExaoneMoeForCausalLM"),
|
||||
"FalconForCausalLM": ("falcon", "FalconForCausalLM"),
|
||||
"FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
|
||||
"FalconH1ForCausalLM": ("falcon_h1", "FalconH1ForCausalLM"),
|
||||
@@ -285,6 +287,8 @@ _REWARD_MODELS = {
|
||||
|
||||
_TOKEN_CLASSIFICATION_MODELS = {
|
||||
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
|
||||
"ClefFlashForJointSchema": ("clef_flash", "ClefFlashForJointSchema"),
|
||||
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
|
||||
"ModernBertForTokenClassification": (
|
||||
"modernbert",
|
||||
"ModernBertForTokenClassification",
|
||||
@@ -326,6 +330,7 @@ _SEQUENCE_CLASSIFICATION_MODELS = {
|
||||
"RobertaForSequenceClassification",
|
||||
),
|
||||
# [Multimodal]
|
||||
"D3ForSequenceClassification": ("d3", "D3ForSequenceClassification"),
|
||||
"JinaVLForRanking": ("jina_vl", "JinaVLForSequenceClassification"),
|
||||
"LlamaNemotronVLForSequenceClassification": (
|
||||
"nemotron_vl",
|
||||
@@ -513,6 +518,10 @@ _MULTIMODAL_MODELS = {
|
||||
"minicpmv4_6",
|
||||
"MiniCPMV4_6ForConditionalGeneration",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": (
|
||||
"minicpmv4_7",
|
||||
"MiniCPMV4_7ForConditionalGeneration",
|
||||
),
|
||||
"Mistral3ForConditionalGeneration": (
|
||||
"mistral3",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
@@ -683,7 +692,6 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Glm4MoeLiteMTPModel": ("glm4_moe_lite_mtp", "Glm4MoeLiteMTP"),
|
||||
"GlmOcrMTPModel": ("glm_ocr_mtp", "GlmOcrMTP"),
|
||||
"Glm5NextMTPModel": ("vllm.models.glm5next", "Glm5NextMTP"),
|
||||
"MedusaModel": ("medusa", "Medusa"),
|
||||
"OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
|
||||
"Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
|
||||
"Qwen4ExpMTP": ("vllm.models.qwen4_exp", "Qwen4ExpMTP"),
|
||||
@@ -694,9 +702,6 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
# Temporarily disabled.
|
||||
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
# "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
|
||||
}
|
||||
|
||||
_TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
@@ -710,6 +715,7 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
@@ -868,11 +874,11 @@ class _ModelInfo:
|
||||
requires_raw_input_tokens: bool
|
||||
supports_multimodal_encoder_tp_data: bool
|
||||
supports_pp: bool
|
||||
supports_hc_sp: bool
|
||||
has_inner_state: bool
|
||||
is_attention_free: bool
|
||||
is_hybrid: bool
|
||||
has_noops: bool
|
||||
supports_mamba_prefix_caching: bool
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
@@ -898,10 +904,10 @@ class _ModelInfo:
|
||||
model
|
||||
),
|
||||
supports_pp=supports_pp(model),
|
||||
supports_hc_sp=supports_hc_sp(model),
|
||||
has_inner_state=has_inner_state(model),
|
||||
is_attention_free=is_attention_free(model),
|
||||
is_hybrid=is_hybrid(model),
|
||||
supports_mamba_prefix_caching=supports_mamba_prefix_caching(model),
|
||||
supports_replayssm=supports_replayssm(model),
|
||||
supports_transcription=supports_transcription(model),
|
||||
supports_transcription_only=(
|
||||
@@ -1458,6 +1464,15 @@ class _ModelRegistry:
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_pp
|
||||
|
||||
def is_hc_sp_supported_model(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
model_config: ModelConfig,
|
||||
) -> bool:
|
||||
"""Check support for HC SP without changing MoE parallelism."""
|
||||
model_info, _ = self.inspect_model_cls(architectures, model_config)
|
||||
return model_info.supports_hc_sp
|
||||
|
||||
def model_has_inner_state(
|
||||
self,
|
||||
architectures: str | list[str],
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul
|
||||
@@ -459,7 +458,6 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
f"Unsupported attention implementation: {config.att_impl_type}"
|
||||
)
|
||||
self.use_moe = False
|
||||
self.tp_group = get_tp_group()
|
||||
self.use_fused_all_reduce = (
|
||||
get_tensor_model_parallel_world_size() > 1
|
||||
and get_dp_group().world_size == 1
|
||||
@@ -496,13 +494,6 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
)
|
||||
self.prefix = prefix
|
||||
|
||||
def add_and_maybe_inplace_all_reduce(
|
||||
self, in1: torch.Tensor, in2: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if not self.use_fused_all_reduce:
|
||||
return in1 + in2
|
||||
return self.tp_group.all_reduce(in1 + in2)
|
||||
|
||||
def forward(
|
||||
self, positions: torch.Tensor, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.model_executor.models.transformers.fusers.attention import (
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
@@ -182,6 +183,8 @@ class Base(
|
||||
self._create_hf_to_vllm_mapper()
|
||||
# Remove layers not on this pipeline parallel rank
|
||||
self.pipeline_parallel()
|
||||
# Skip the device sync of dynamic rope where it can never rescale
|
||||
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
@@ -282,6 +285,7 @@ class Base(
|
||||
dynamic_arg_dims=dynamic_arg_dims,
|
||||
enable_if=enable_if,
|
||||
is_encoder=is_encoder,
|
||||
use_aot_compile=False,
|
||||
)(cls)
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE fuser: route an HF MoE block through `FusedMoE` with vLLM's own routing."""
|
||||
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
|
||||
|
||||
import ast
|
||||
import functools
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
@@ -11,19 +12,25 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
is_complete,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
@@ -107,6 +114,53 @@ def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
|
||||
return seen
|
||||
|
||||
|
||||
def _renormalizes(topk: fx.Node) -> bool:
|
||||
"""Are the top-k weights divided by their own sum?
|
||||
|
||||
Every HF router spells renormalization as a division by a sum over the
|
||||
selection, optionally offset by an epsilon."""
|
||||
downstream = _reaches(topk, "users")
|
||||
for node in downstream:
|
||||
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
|
||||
continue
|
||||
denominator = node.args[1]
|
||||
if not isinstance(denominator, fx.Node):
|
||||
continue
|
||||
cone = _reaches(denominator, "all_input_nodes")
|
||||
if any(is_op(n, "sum") and n in downstream for n in cone):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _unwrap(node: object, moe_block: nn.Module) -> object:
|
||||
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
|
||||
while True:
|
||||
node = peel(node)
|
||||
if (
|
||||
is_op(node, "reshape")
|
||||
or is_op(node, "view")
|
||||
or (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
|
||||
)
|
||||
):
|
||||
node = node.args[0]
|
||||
else:
|
||||
return node
|
||||
|
||||
|
||||
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
|
||||
"""Is `node` a child module of `moe_block` called on just `x`?"""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and _unwrap(node.args[0], moe_block) is x
|
||||
)
|
||||
|
||||
|
||||
class SharedExpertMLP(nn.Module):
|
||||
"""Wraps an HF shared expert, applying the output gating it is paired with."""
|
||||
|
||||
@@ -143,16 +197,23 @@ class MoEBlockFuser:
|
||||
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
|
||||
|
||||
gate_name: str
|
||||
scoring_func: str
|
||||
scoring_func: str | None
|
||||
"""`None` if routing stays in the Transformers gate."""
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
renormalize: bool | None = None
|
||||
|
||||
@staticmethod
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
def _match_router(
|
||||
gate: nn.Module,
|
||||
) -> tuple[str, torch.dtype | None, bool | None] | None:
|
||||
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
|
||||
`softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
Returns the scoring function, the dtype the router computes in, and
|
||||
whether the top-k weights are renormalized (`None` if the trace was cut
|
||||
short before it could say)."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
@@ -164,99 +225,176 @@ class MoEBlockFuser:
|
||||
if not topks:
|
||||
return None
|
||||
topk = topks[-1]
|
||||
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
|
||||
scorers = [
|
||||
n
|
||||
for n in _reaches(topk, "all_input_nodes")
|
||||
if is_op(n, "softmax") or is_op(n, "sigmoid")
|
||||
]
|
||||
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
|
||||
post_topk = False
|
||||
# Find all scoring functions that reach the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
|
||||
if not scorers:
|
||||
# Find all scoring functions that are users of the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
|
||||
post_topk = True
|
||||
if len(scorers) != 1:
|
||||
return None
|
||||
scorer = scorers[0]
|
||||
logits_cone = _reaches(scorer, "all_input_nodes")
|
||||
if not any(is_op(n, "linear") for n in logits_cone):
|
||||
return None
|
||||
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone)
|
||||
is_softmax = is_op(scorer, "softmax")
|
||||
# vLLM always scores every expert, so matching a router that softmaxes
|
||||
# the selection alone takes renormalizing. Otherwise look for the
|
||||
# division, reading its absence as "no" only from a complete trace.
|
||||
renormalize: bool | None
|
||||
if (post_topk and is_softmax) or _renormalizes(topk):
|
||||
renormalize = True
|
||||
else:
|
||||
renormalize = False if is_complete(graph) else None
|
||||
scoring_func = "softmax" if is_softmax else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone), renormalize
|
||||
|
||||
@staticmethod
|
||||
def _match_experts(
|
||||
graph: fx.Graph, experts_name: str, moe_block: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
|
||||
being the block's input.
|
||||
|
||||
Returns the experts call, `x` and the gate call."""
|
||||
experts = find_node(
|
||||
graph, lambda n: n.op == "call_module" and n.target == experts_name
|
||||
)
|
||||
if experts is None:
|
||||
return None
|
||||
# Bind by signature so keyword calls match too. Fail closed on any other
|
||||
# argument: the rewritten forward would drop it.
|
||||
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
|
||||
try:
|
||||
bound = forward.bind(*experts.args, **experts.kwargs)
|
||||
except TypeError:
|
||||
return None
|
||||
if len(bound.arguments) != 3:
|
||||
return None
|
||||
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
|
||||
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
|
||||
return None
|
||||
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
|
||||
return None
|
||||
gate = ids.args[0]
|
||||
if not (
|
||||
_reads_input(gate, x, moe_block)
|
||||
and ids.args == (gate, 2)
|
||||
and weights.args == (gate, 1)
|
||||
):
|
||||
return None
|
||||
return experts, x, gate
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
graph: fx.Graph, experts: str
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Detects the shared expert and its optional gate by dataflow."""
|
||||
experts_predicate = lambda n: n.op == "call_module" and n.target == experts
|
||||
if (experts_node := find_node(graph, experts_predicate)) is None:
|
||||
return None, None
|
||||
from_experts = _reaches(experts_node, "users")
|
||||
for add in graph.nodes:
|
||||
if not is_op(add, "add"):
|
||||
node: object, x: fx.Node, moe_block: nn.Module
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
|
||||
|
||||
Returns the shared expert's name and its optional gate's name."""
|
||||
if _reads_input(node, x, moe_block):
|
||||
return node.target, None
|
||||
if not (is_op(node, "mul") and len(node.args) == 2):
|
||||
return None
|
||||
for gated, shared in (node.args, node.args[::-1]):
|
||||
gated, shared = peel(gated), peel(shared)
|
||||
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
|
||||
continue
|
||||
operands = [a for a in add.args if isinstance(a, fx.Node)]
|
||||
# Exactly one side is the experts' output; the other is the shared path.
|
||||
sides = [a in from_experts for a in operands]
|
||||
if len(operands) != 2 or sides.count(True) != 1:
|
||||
continue
|
||||
cone = _reaches(operands[sides.index(False)], "all_input_nodes")
|
||||
modules = [n for n in cone if n.op == "call_module" and n.target != experts]
|
||||
# A sigmoid wrapping one of those modules marks the shared-expert gate.
|
||||
gate = next(
|
||||
(
|
||||
src
|
||||
for n in cone
|
||||
if is_op(n, "sigmoid")
|
||||
and isinstance(src := peel(n.args[0]), fx.Node)
|
||||
and src in modules
|
||||
),
|
||||
None,
|
||||
)
|
||||
shared = [n for n in modules if n is not gate]
|
||||
if len(shared) != 1:
|
||||
return None, None
|
||||
return shared[0].target, (gate.target if gate is not None else None)
|
||||
return None, None
|
||||
gate = peel(gated.args[0])
|
||||
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
|
||||
moe_block.get_submodule(gate.target)
|
||||
):
|
||||
return shared.target, gate.target
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def match(cls, moe_block: nn.Module, experts_name: str) -> "MoEBlockFuser | None":
|
||||
def match(
|
||||
cls,
|
||||
moe_block: nn.Module,
|
||||
experts_name: str,
|
||||
divides_shared_output: bool = False,
|
||||
) -> "MoEBlockFuser | None":
|
||||
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
|
||||
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
|
||||
reshapes, casts and `nn.Identity`.
|
||||
|
||||
Args:
|
||||
moe_block: The MoE block.
|
||||
experts_name: The name of the experts in `moe_block`.
|
||||
divides_shared_output: Whether vLLM routing would divide the shared
|
||||
expert's output by `routed_scaling_factor` (fp16 overflow
|
||||
protection), which Transformers decoder layers do not undo.
|
||||
|
||||
Returns:
|
||||
A fuser if more than the experts can run in vLLM, else `None`.
|
||||
|
||||
"""
|
||||
# Standard MoE block returns a single tensor.
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
# Router: the child that scores + top-k selects.
|
||||
gate_name = scoring_func = router_dtype = None
|
||||
for name, child in moe_block.named_children():
|
||||
if (
|
||||
name != experts_name
|
||||
and (router := cls._match_router(child)) is not None
|
||||
):
|
||||
gate_name = name
|
||||
scoring_func, router_dtype = router
|
||||
break
|
||||
if gate_name is None or scoring_func is None:
|
||||
if (graph := trace(moe_block)) is None:
|
||||
return None
|
||||
# Shared expert: a child the block adds to the experts' output.
|
||||
shared_name = shared_gate_name = None
|
||||
others = [
|
||||
n
|
||||
for n, _ in moe_block.named_children()
|
||||
if n not in {experts_name, gate_name}
|
||||
]
|
||||
if others:
|
||||
graph = trace(moe_block)
|
||||
if graph is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = cls._match_shared_experts(
|
||||
graph, experts_name
|
||||
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
|
||||
return None
|
||||
experts, x, gate = matched
|
||||
unwrap = functools.partial(_unwrap, moe_block=moe_block)
|
||||
gate_name = gate.target
|
||||
router = cls._match_router(getattr(moe_block, gate_name))
|
||||
scoring_func, router_dtype, renormalize = router or (None, None, None)
|
||||
|
||||
def is_routed(n: object) -> bool:
|
||||
return n is experts or (
|
||||
is_op(n, "mul")
|
||||
and not isinstance(n.args[1], fx.Node)
|
||||
and unwrap(n.args[0]) is experts
|
||||
)
|
||||
if shared_gate_name is not None and not _is_scalar_gate(
|
||||
getattr(moe_block, shared_gate_name)
|
||||
):
|
||||
|
||||
# Output: the experts' output, plus the shared expert's if any.
|
||||
routed = out = unwrap(output_value(graph))
|
||||
shared = None
|
||||
if not is_routed(out):
|
||||
if not (is_op(out, "add") and len(out.args) == 2):
|
||||
return None
|
||||
routed, shared = map(unwrap, out.args)
|
||||
if is_routed(shared):
|
||||
routed, shared = shared, routed
|
||||
if not is_routed(routed):
|
||||
return None
|
||||
shared_name = shared_gate_name = None
|
||||
if shared is not None:
|
||||
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = match
|
||||
# Fail closed: `rewrite_forward` runs only the experts and the detected
|
||||
# shared expert, so any other stateful child would be dropped.
|
||||
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
|
||||
if scoring_func is not None and shared_name and divides_shared_output:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert output, "
|
||||
"which only fp16 overflow protection expects the decoder layer to "
|
||||
"compensate for. Falling back to routing in Transformers; run in "
|
||||
"bfloat16 to fuse it.",
|
||||
type(moe_block).__name__,
|
||||
)
|
||||
scoring_func = router_dtype = renormalize = None
|
||||
# Without vLLM routing, only a shared expert can move to vLLM, and only
|
||||
# if the block does not scale the experts' output (vLLM routing does).
|
||||
if scoring_func is None and (shared_name is None or routed is not experts):
|
||||
return None
|
||||
return cls(
|
||||
gate_name,
|
||||
scoring_func,
|
||||
shared_name,
|
||||
shared_gate_name,
|
||||
router_dtype,
|
||||
renormalize,
|
||||
)
|
||||
|
||||
def gate(
|
||||
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
|
||||
|
||||
@@ -35,6 +35,21 @@ _MODULE_CALL = nn.Module.__call__
|
||||
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
|
||||
`call_module` nodes; meta execution must call modules for real."""
|
||||
|
||||
_MODULE_GETATTR = nn.Module.__getattr__
|
||||
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
|
||||
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
|
||||
recurse until the stack runs out, truncating the graph."""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _unpatched_modules():
|
||||
"""Restores the module hooks fx patches, so meta execution records nothing."""
|
||||
with (
|
||||
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
|
||||
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
|
||||
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
|
||||
@@ -132,7 +147,8 @@ class _AllLeafTracer(fx.Tracer):
|
||||
)
|
||||
return _UNKNOWN
|
||||
if kind == "get_attr":
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
with _unpatched_modules():
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = torch.empty_like(value, device="meta")
|
||||
return value
|
||||
@@ -159,8 +175,6 @@ class _AllLeafTracer(fx.Tracer):
|
||||
if kind == "call_module":
|
||||
# Run the child's forward with all its state on "meta", without
|
||||
# mutating it (at match time params may be meta but buffers real).
|
||||
# fx patches `nn.Module.__call__` while tracing; restore the real
|
||||
# one so this execution is not itself recorded.
|
||||
child = self.root.get_submodule(str(target))
|
||||
state = {
|
||||
name: torch.empty_like(tensor, device="meta")
|
||||
@@ -168,7 +182,7 @@ class _AllLeafTracer(fx.Tracer):
|
||||
child.named_parameters(), child.named_buffers()
|
||||
)
|
||||
}
|
||||
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
|
||||
with _unpatched_modules():
|
||||
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
|
||||
return _UNKNOWN
|
||||
|
||||
@@ -686,6 +700,14 @@ def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
|
||||
_Replacer().visit(module)
|
||||
|
||||
|
||||
def is_complete(graph: fx.Graph) -> bool:
|
||||
"""Did the trace run to completion? A partial graph has no `output` node.
|
||||
|
||||
Only a complete graph can be read for the *absence* of an op; in a partial
|
||||
one the tail is exactly what is missing."""
|
||||
return any(node.op == "output" for node in graph.nodes)
|
||||
|
||||
|
||||
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
|
||||
"""The first node in `graph` matching `predicate`, or `None`."""
|
||||
return next((n for n in graph.nodes if predicate(n)), None)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .base import Base
|
||||
@@ -126,6 +127,43 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def _hf_topk_router_precomputed(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
|
||||
weights arrive as `gating_output`, ids via `moe_state`."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
def _hf_topk_router(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
gate: nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
|
||||
_, topk_weights, topk_ids = gate(hidden_states)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
class TransformersRoutedExperts(RoutedExperts):
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
@@ -180,23 +218,18 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
text_config = self.text_config
|
||||
|
||||
# Positional arguments
|
||||
num_experts = self.model_config.get_num_experts()
|
||||
top_k = getattr_iter(text_config, ["num_experts_per_tok", "top_k"], None)
|
||||
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
|
||||
num_experts = getattr_iter(text_config, num_experts_names, None)
|
||||
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
|
||||
top_k = getattr_iter(text_config, top_k_names, None)
|
||||
assert top_k is not None
|
||||
hidden_size = text_config.hidden_size
|
||||
intermediate_size = getattr_iter(
|
||||
text_config, ["moe_intermediate_size", "intermediate_size"], None
|
||||
)
|
||||
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
|
||||
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
|
||||
assert intermediate_size is not None
|
||||
|
||||
num_shared_experts = getattr_iter(
|
||||
text_config,
|
||||
[
|
||||
"n_shared_experts", # DeepSeek, Docs, GLM
|
||||
"moe_num_shared_experts", # Aria, Ernie
|
||||
],
|
||||
0,
|
||||
)
|
||||
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
|
||||
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
|
||||
|
||||
# Common kwargs
|
||||
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
|
||||
@@ -275,6 +308,9 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
if "bias" in experts_param_name:
|
||||
has_bias = True
|
||||
break
|
||||
# Whether the expert weights are stored in transposed format.
|
||||
# Comes from Transformers `use_experts_implementation` decorator.
|
||||
is_transposed = getattr(experts, "is_transposed", False)
|
||||
# If the config does not specify num_shared_experts, but
|
||||
# the model has shared experts, we assume there is one.
|
||||
if self.num_shared_experts == 0:
|
||||
@@ -308,28 +344,20 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
enable_eplb=enable_eplb,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
is_fused_checkpoint_transposed=is_transposed,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
fuser = MoEBlockFuser.match(moe_block, experts_name)
|
||||
# _maybe_apply_routed_scale_to_output edge case. Transformers
|
||||
# decoder layers do not compensate for dividing by scaling factor.
|
||||
reaches_fp16_trick = (
|
||||
divides_shared_output = (
|
||||
routed_scaling_factor != 1.0
|
||||
and apply_routed_scale_to_output
|
||||
and self.model_config.dtype == torch.float16
|
||||
and fuser is not None
|
||||
and fuser.shared_name is not None
|
||||
)
|
||||
if reaches_fp16_trick:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert "
|
||||
"output, which only fp16 overflow protection expects the "
|
||||
"decoder layer to compensate for. Falling back to routing "
|
||||
"in Transformers; run in bfloat16 to fuse it.",
|
||||
moe_block_cls,
|
||||
)
|
||||
if fuser is not None and not reaches_fp16_trick:
|
||||
fuser = MoEBlockFuser.match(
|
||||
moe_block, experts_name, divides_shared_output
|
||||
)
|
||||
if fuser is not None:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in MoERunner.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
@@ -341,34 +369,52 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
down_name = glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
if scoring_func := fuser.scoring_func:
|
||||
# Internal routing kwargs.
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=scoring_func,
|
||||
renormalize=(
|
||||
layer_renormalize
|
||||
if fuser.renormalize is None
|
||||
else fuser.renormalize
|
||||
),
|
||||
gate=gate,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
else:
|
||||
# External routing kwargs
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
_hf_topk_router,
|
||||
gate=getattr(moe_block, fuser.gate_name),
|
||||
),
|
||||
)
|
||||
|
||||
kwargs |= dict(
|
||||
scoring_func=fuser.scoring_func,
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
gate=gate,
|
||||
shared_experts=shared_experts,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
fuser.rewrite_forward(moe_block)
|
||||
routed = "gate + experts"
|
||||
if fuser.shared_name:
|
||||
routed += " + shared experts"
|
||||
logger.info_once(
|
||||
"Fused: %s (%s) -> MoERunner (internal routing)",
|
||||
routed,
|
||||
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
|
||||
" + shared experts" if fuser.shared_name else "",
|
||||
moe_block_cls,
|
||||
"external" if fuser.scoring_func is None else "internal",
|
||||
)
|
||||
else:
|
||||
# MoE block forward is unmodified.
|
||||
@@ -376,35 +422,11 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
# We then smuggle the topk_ids in using a custom op.
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
def custom_routing_function(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function, moe_state=moe_state
|
||||
_hf_topk_router_precomputed, moe_state=moe_state
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
runner_args={"moe_state": moe_state},
|
||||
|
||||
@@ -357,13 +357,51 @@ def get_feature_request_tip(
|
||||
return f"{prefix}{tip}"
|
||||
|
||||
|
||||
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
|
||||
|
||||
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
|
||||
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
|
||||
if max_position_embeddings is None:
|
||||
return False
|
||||
return vllm_config.model_config.max_model_len <= max_position_embeddings
|
||||
|
||||
|
||||
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
|
||||
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
|
||||
|
||||
Defaults to `True` but is disabled in the following situations:
|
||||
|
||||
- The model uses dynamic rope scaling.
|
||||
- The model uses dynamic rope scaling that can still rescale within
|
||||
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
# Dynamic rope scaling is not compatible with torch.compile
|
||||
return all(rp["rope_type"] != "dynamic" for rp in iter_rope_parameters(text_config))
|
||||
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
|
||||
if "dynamic" not in rope_types:
|
||||
return True
|
||||
# Layer types using longrope still sync, even once dynamic rope is disabled
|
||||
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
|
||||
|
||||
|
||||
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
|
||||
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
|
||||
|
||||
The update syncs on `torch.max(position_ids)` every forward, which breaks
|
||||
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
|
||||
"default" skips it while keeping the `inv_freq` computed at init.
|
||||
"""
|
||||
if not dynamic_rope_update_is_noop(vllm_config):
|
||||
return
|
||||
for module in model.modules():
|
||||
rope_type = getattr(module, "rope_type", None)
|
||||
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
|
||||
continue
|
||||
if isinstance(rope_type, dict):
|
||||
module.rope_type = {
|
||||
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
|
||||
}
|
||||
elif rope_type == "dynamic":
|
||||
module.rope_type = "default"
|
||||
|
||||
@@ -322,7 +322,15 @@ class AutoWeightsLoader:
|
||||
)
|
||||
|
||||
if param.dtype != weight_data.dtype:
|
||||
logger.warning(
|
||||
# An exact widening -- an fp32 accumulator reading a bf16
|
||||
# table, say -- keeps every stored value, so it is a debug
|
||||
# line. Every other mismatch still warns.
|
||||
exact_widening = weight_data.dtype in (
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
) and param.dtype in (torch.float32, torch.float64)
|
||||
(logger.debug if exact_widening else logger.warning)(
|
||||
"Attempted to load weight %s with dtype %s into "
|
||||
"parameter with dtype %s",
|
||||
weight_qualname,
|
||||
|
||||
@@ -45,7 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsMambaPrefixCaching
|
||||
from .interfaces import HasInnerState, IsHybrid
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
@@ -833,7 +833,7 @@ class Zamba2Model(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixCaching):
|
||||
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid):
|
||||
"""Zamba2 model with causal language modeling head.
|
||||
|
||||
This class wraps the core Zamba2 model and adds:
|
||||
@@ -911,10 +911,6 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
|
||||
LoRA and scheduler settings
|
||||
prefix: Optional prefix for parameter names
|
||||
|
||||
Raises:
|
||||
AssertionError: If prefix caching is enabled
|
||||
(not supported by Mamba)
|
||||
|
||||
"""
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
|
||||
Reference in New Issue
Block a user