diff --git a/vllm/config/model.py b/vllm/config/model.py index cfbb870e78..502f1bbd8d 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -508,6 +508,10 @@ class ModelConfig: # here early. if self.multimodal_config: factors["language_model_only"] = self.multimodal_config.language_model_only + # Sizes Qwen3.5's M-RoPE cache. + factors["video_pruning"] = ( + self.multimodal_config.is_multimodal_pruning_enabled() + ) return hash_factors(factors) def _update_nested( diff --git a/vllm/model_executor/layers/rotary_embedding/__init__.py b/vllm/model_executor/layers/rotary_embedding/__init__.py index e23a5128b8..85d4cb63e5 100644 --- a/vllm/model_executor/layers/rotary_embedding/__init__.py +++ b/vllm/model_executor/layers/rotary_embedding/__init__.py @@ -36,6 +36,7 @@ def get_rope( rope_parameters: dict[str, Any] | None = None, dtype: torch.dtype | None = None, dual_chunk_attention_config: dict[str, Any] | None = None, + mrope_positions_factor: int = 4, ) -> RotaryEmbedding: if dtype is None: dtype = torch.get_default_dtype() @@ -78,6 +79,7 @@ def get_rope( rope_parameters_args, dual_chunk_attention_args, dtype, + mrope_positions_factor, ) if key in _ROPE_DICT: return _ROPE_DICT[key] @@ -118,6 +120,7 @@ def get_rope( dtype, mrope_section=rope_parameters["mrope_section"], mrope_interleaved=rope_parameters.get("mrope_interleaved", False), + positions_factor=mrope_positions_factor, ) elif "use_fope" in rope_parameters and rope_parameters["use_fope"]: extra_kwargs = { @@ -263,6 +266,7 @@ def get_rope( dtype, mrope_section=rope_parameters["mrope_section"], mrope_interleaved=rope_parameters.get("mrope_interleaved", False), + positions_factor=mrope_positions_factor, scaling_factor=scaling_factor, **extra_kwargs, ) diff --git a/vllm/model_executor/layers/rotary_embedding/mrope.py b/vllm/model_executor/layers/rotary_embedding/mrope.py index 11882856e0..9bcc5a95e1 100644 --- a/vllm/model_executor/layers/rotary_embedding/mrope.py +++ b/vllm/model_executor/layers/rotary_embedding/mrope.py @@ -315,8 +315,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase): dtype: torch.dtype, mrope_section: list[int] | None = None, mrope_interleaved: bool = False, - # YaRN parameters. *, + positions_factor: int = 4, + # YaRN parameters. scaling_factor: float | None = None, beta_fast: int = 32, beta_slow: int = 1, @@ -345,8 +346,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase): # In Qwen2.5-VL, the maximum index value is related to the duration of # the input video. We enlarge max_position_embeddings to 4 times to get - # a larger the cos and sin cache. - self.cache_max_position_num = max_position_embeddings * 4 + # a larger the cos and sin cache. Models whose positions never exceed + # the token count pass `positions_factor=1`. + self.cache_max_position_num = max_position_embeddings * positions_factor super().__init__( head_size, rotary_dim, diff --git a/vllm/model_executor/models/qwen3.py b/vllm/model_executor/models/qwen3.py index 7a1d76234f..31d1a094eb 100644 --- a/vllm/model_executor/models/qwen3.py +++ b/vllm/model_executor/models/qwen3.py @@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module): attn_type: str = AttentionType.DECODER, dual_chunk_attention_config: dict[str, Any] | None = None, per_layer_sliding_window: int | None = None, + mrope_positions_factor: int = 4, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module): max_position=max_position, rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, + mrope_positions_factor=mrope_positions_factor, ) attn_cls = ( EncoderOnlyAttention @@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module): quant_config: QuantizationConfig | None = None, prefix: str = "", per_layer_sliding_window: int | None = None, + mrope_positions_factor: int = 4, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module): attn_type=attn_type, dual_chunk_attention_config=dual_chunk_attention_config, per_layer_sliding_window=per_layer_sliding_window, + mrope_positions_factor=mrope_positions_factor, ) self.mlp = Qwen3MLP( hidden_size=self.hidden_size, @@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = { } ) class Qwen3Model(Qwen2Model): + decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__( - vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer + vllm_config=vllm_config, + prefix=prefix, + decoder_layer_type=self.decoder_layer_type, ) diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 0d110dd58b..f105948f92 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -91,6 +91,7 @@ from .qwen3_vl import ( Qwen3VLForConditionalGeneration, Qwen3VLMultiModalProcessor, Qwen3VLProcessingInfo, + mrope_positions_factor, ) from .utils import ( AutoWeightsLoader, @@ -161,6 +162,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer): quant_config=quant_config, prefix=f"{prefix}.self_attn", reduce_results=not self.use_attn_reduce_scatter_for_moe, + mrope_positions_factor=mrope_positions_factor(vllm_config), ) else: raise ValueError(f"Invalid layer_type {self.layer_type}") diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index 3f293550ac..870a082d3a 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -256,6 +256,7 @@ class Qwen3MoeAttention(nn.Module): quant_config: QuantizationConfig | None = None, prefix: str = "", dual_chunk_attention_config: dict[str, Any] | None = None, + mrope_positions_factor: int = 4, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -303,6 +304,7 @@ class Qwen3MoeAttention(nn.Module): max_position=max_position_embeddings, rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, + mrope_positions_factor=mrope_positions_factor, ) attention_kwargs: dict[str, Any] = {} if dual_chunk_attention_config: @@ -351,6 +353,7 @@ class Qwen3MoeDecoderLayer(nn.Module): vllm_config: VllmConfig, prefix: str = "", is_fused_checkpoint_transposed: bool = False, + mrope_positions_factor: int = 4, ) -> None: super().__init__() @@ -376,6 +379,7 @@ class Qwen3MoeDecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.self_attn", dual_chunk_attention_config=dual_chunk_attention_config, + mrope_positions_factor=mrope_positions_factor, ) # `mlp_only_layers` in the config. diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index b0bf0fecd1..04d20c5741 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -280,6 +280,7 @@ class Qwen3NextAttention(nn.Module): quant_config: QuantizationConfig | None = None, reduce_results: bool = True, prefix: str = "", + mrope_positions_factor: int = 4, ) -> None: super().__init__() self.config = config @@ -331,6 +332,7 @@ class Qwen3NextAttention(nn.Module): max_position=config.max_position_embeddings, rope_parameters=config.rope_parameters, dual_chunk_attention_config=self.dual_chunk_attention_config, + mrope_positions_factor=mrope_positions_factor, ) # Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 6648949c73..77780b3d1d 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -52,7 +52,7 @@ from vllm.compilation.decorators import ( should_torch_compile_mm_encoder, support_torch_compile, ) -from vllm.config import VllmConfig +from vllm.config import VllmConfig, get_current_vllm_config from vllm.config.multimodal import ( MultiModalConfig, MultiModalDummyOptions, @@ -148,7 +148,7 @@ from .qwen2_vl import ( Qwen2VLProcessingInfo, _create_qwen2vl_field_factory, ) -from .qwen3 import Qwen3ForCausalLM, Qwen3Model +from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model from .utils import ( AutoWeightsLoader, PPMissingLayer, @@ -1748,6 +1748,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) return PromptUpdateDetails.from_seq(all_token_ids) +def mrope_positions_factor(vllm_config: VllmConfig) -> int: + # Video pruning keeps the positions of the tokens it drops. A drafter gets + # the target's positions, so it follows the target. + spec = vllm_config.speculative_config + model_config = (spec and spec.target_model_config) or vllm_config.model_config + mm_config = model_config.multimodal_config + pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled() + return 4 if pruned else 1 + + +class Qwen3VLDecoderLayer(Qwen3DecoderLayer): + def __init__(self, **kwargs) -> None: + factor = mrope_positions_factor(get_current_vllm_config()) + super().__init__(**kwargs, mrope_positions_factor=factor) + + @support_torch_compile( dynamic_arg_dims={ "input_ids": 0, @@ -1761,6 +1777,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) } ) class Qwen3LLMModel(Qwen3Model): + decoder_layer_type = Qwen3VLDecoderLayer + def forward( self, input_ids: torch.Tensor | None, diff --git a/vllm/model_executor/models/qwen3_vl_moe.py b/vllm/model_executor/models/qwen3_vl_moe.py index 4d0c7788a6..9a1d771ec4 100644 --- a/vllm/model_executor/models/qwen3_vl_moe.py +++ b/vllm/model_executor/models/qwen3_vl_moe.py @@ -56,6 +56,7 @@ from .qwen3_vl import ( Qwen3VLForConditionalGeneration, Qwen3VLMultiModalProcessor, Qwen3VLProcessingInfo, + mrope_positions_factor, ) from .utils import StageMissingLayer, maybe_prefix @@ -73,6 +74,7 @@ class Qwen3VLMoeDecoderLayer(Qwen3MoeDecoderLayer): vllm_config=vllm_config, prefix=prefix, is_fused_checkpoint_transposed=True, + mrope_positions_factor=mrope_positions_factor(vllm_config), ) diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py index ff01490bb1..709fd0b08e 100644 --- a/vllm/models/qwen4_exp/amd/qsa.py +++ b/vllm/models/qwen4_exp/amd/qsa.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLine from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.models.qwen3_next import Qwen3NextAttention +from vllm.model_executor.models.qwen3_vl import mrope_positions_factor from vllm.platforms import current_platform from vllm.utils.torch_utils import ( LayerNameType, @@ -248,6 +249,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): head_size=self.head_dim, max_position=config.max_position_embeddings, rope_parameters=config.rope_parameters, + mrope_positions_factor=mrope_positions_factor(vllm_config), ) self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index 0246db8094..9f0934d5a0 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope from vllm.model_executor.models.qwen3_next import Qwen3NextAttention +from vllm.model_executor.models.qwen3_vl import mrope_positions_factor from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, @@ -427,6 +428,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): head_size=self.head_dim, max_position=config.max_position_embeddings, rope_parameters=config.rope_parameters, + mrope_positions_factor=mrope_positions_factor(vllm_config), ) self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)