[Model] Drop the 4x M-RoPE cache headroom for Qwen3-VL, Qwen3.5 and Qwen3.8 (#59853)

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
Signed-off-by: lesj0610 <lesj0610@gmail.com>
Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>

vllm/config/model.py

@@ -508,6 +508,10 @@ class ModelConfig:
         # here early.
         if self.multimodal_config:
             factors["language_model_only"] = self.multimodal_config.language_model_only
+            # Sizes Qwen3.5's M-RoPE cache.
+            factors["video_pruning"] = (
+                self.multimodal_config.is_multimodal_pruning_enabled()
+            )
         return hash_factors(factors)
 
     def _update_nested(

vllm/model_executor/layers/rotary_embedding/__init__.py

@@ -36,6 +36,7 @@ def get_rope(
     rope_parameters: dict[str, Any] | None = None,
     dtype: torch.dtype | None = None,
     dual_chunk_attention_config: dict[str, Any] | None = None,
+    mrope_positions_factor: int = 4,
 ) -> RotaryEmbedding:
     if dtype is None:
         dtype = torch.get_default_dtype()
@@ -78,6 +79,7 @@ def get_rope(
         rope_parameters_args,
         dual_chunk_attention_args,
         dtype,
+        mrope_positions_factor,
     )
     if key in _ROPE_DICT:
         return _ROPE_DICT[key]
@@ -118,6 +120,7 @@ def get_rope(
                 dtype,
                 mrope_section=rope_parameters["mrope_section"],
                 mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
+                positions_factor=mrope_positions_factor,
             )
         elif "use_fope" in rope_parameters and rope_parameters["use_fope"]:
             extra_kwargs = {
@@ -263,6 +266,7 @@ def get_rope(
                 dtype,
                 mrope_section=rope_parameters["mrope_section"],
                 mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
+                positions_factor=mrope_positions_factor,
                 scaling_factor=scaling_factor,
                 **extra_kwargs,
             )

vllm/model_executor/layers/rotary_embedding/mrope.py

@@ -315,8 +315,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
         dtype: torch.dtype,
         mrope_section: list[int] | None = None,
         mrope_interleaved: bool = False,
-        # YaRN parameters.
         *,
+        positions_factor: int = 4,
+        # YaRN parameters.
         scaling_factor: float | None = None,
         beta_fast: int = 32,
         beta_slow: int = 1,
@@ -345,8 +346,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
 
         # In Qwen2.5-VL, the maximum index value is related to the duration of
         # the input video. We enlarge max_position_embeddings to 4 times to get
-        # a larger the cos and sin cache.
-        self.cache_max_position_num = max_position_embeddings * 4
+        # a larger the cos and sin cache. Models whose positions never exceed
+        # the token count pass `positions_factor=1`.
+        self.cache_max_position_num = max_position_embeddings * positions_factor
         super().__init__(
             head_size,
             rotary_dim,

vllm/model_executor/models/qwen3.py

@@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module):
         attn_type: str = AttentionType.DECODER,
         dual_chunk_attention_config: dict[str, Any] | None = None,
         per_layer_sliding_window: int | None = None,
+        mrope_positions_factor: int = 4,
     ) -> None:
         super().__init__()
         self.hidden_size = hidden_size
@@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module):
             max_position=max_position,
             rope_parameters=rope_parameters,
             dual_chunk_attention_config=dual_chunk_attention_config,
+            mrope_positions_factor=mrope_positions_factor,
         )
         attn_cls = (
             EncoderOnlyAttention
@@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module):
         quant_config: QuantizationConfig | None = None,
         prefix: str = "",
         per_layer_sliding_window: int | None = None,
+        mrope_positions_factor: int = 4,
     ) -> None:
         super().__init__()
         self.hidden_size = config.hidden_size
@@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module):
             attn_type=attn_type,
             dual_chunk_attention_config=dual_chunk_attention_config,
             per_layer_sliding_window=per_layer_sliding_window,
+            mrope_positions_factor=mrope_positions_factor,
         )
         self.mlp = Qwen3MLP(
             hidden_size=self.hidden_size,
@@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = {
     }
 )
 class Qwen3Model(Qwen2Model):
+    decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer
+
     def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
         super().__init__(
-            vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer
+            vllm_config=vllm_config,
+            prefix=prefix,
+            decoder_layer_type=self.decoder_layer_type,
         )
 
 

vllm/model_executor/models/qwen3_5.py

@@ -91,6 +91,7 @@ from .qwen3_vl import (
     Qwen3VLForConditionalGeneration,
     Qwen3VLMultiModalProcessor,
     Qwen3VLProcessingInfo,
+    mrope_positions_factor,
 )
 from .utils import (
     AutoWeightsLoader,
@@ -161,6 +162,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
                 quant_config=quant_config,
                 prefix=f"{prefix}.self_attn",
                 reduce_results=not self.use_attn_reduce_scatter_for_moe,
+                mrope_positions_factor=mrope_positions_factor(vllm_config),
             )
         else:
             raise ValueError(f"Invalid layer_type {self.layer_type}")

vllm/model_executor/models/qwen3_moe.py

@@ -256,6 +256,7 @@ class Qwen3MoeAttention(nn.Module):
         quant_config: QuantizationConfig | None = None,
         prefix: str = "",
         dual_chunk_attention_config: dict[str, Any] | None = None,
+        mrope_positions_factor: int = 4,
     ) -> None:
         super().__init__()
         self.hidden_size = hidden_size
@@ -303,6 +304,7 @@ class Qwen3MoeAttention(nn.Module):
             max_position=max_position_embeddings,
             rope_parameters=rope_parameters,
             dual_chunk_attention_config=dual_chunk_attention_config,
+            mrope_positions_factor=mrope_positions_factor,
         )
         attention_kwargs: dict[str, Any] = {}
         if dual_chunk_attention_config:
@@ -351,6 +353,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
         vllm_config: VllmConfig,
         prefix: str = "",
         is_fused_checkpoint_transposed: bool = False,
+        mrope_positions_factor: int = 4,
     ) -> None:
         super().__init__()
 
@@ -376,6 +379,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
             quant_config=quant_config,
             prefix=f"{prefix}.self_attn",
             dual_chunk_attention_config=dual_chunk_attention_config,
+            mrope_positions_factor=mrope_positions_factor,
         )
 
         # `mlp_only_layers` in the config.

vllm/model_executor/models/qwen3_next.py

@@ -280,6 +280,7 @@ class Qwen3NextAttention(nn.Module):
         quant_config: QuantizationConfig | None = None,
         reduce_results: bool = True,
         prefix: str = "",
+        mrope_positions_factor: int = 4,
     ) -> None:
         super().__init__()
         self.config = config
@@ -331,6 +332,7 @@ class Qwen3NextAttention(nn.Module):
             max_position=config.max_position_embeddings,
             rope_parameters=config.rope_parameters,
             dual_chunk_attention_config=self.dual_chunk_attention_config,
+            mrope_positions_factor=mrope_positions_factor,
         )
 
         # Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL

vllm/model_executor/models/qwen3_vl.py

@@ -52,7 +52,7 @@ from vllm.compilation.decorators import (
     should_torch_compile_mm_encoder,
     support_torch_compile,
 )
-from vllm.config import VllmConfig
+from vllm.config import VllmConfig, get_current_vllm_config
 from vllm.config.multimodal import (
     MultiModalConfig,
     MultiModalDummyOptions,
@@ -148,7 +148,7 @@ from .qwen2_vl import (
     Qwen2VLProcessingInfo,
     _create_qwen2vl_field_factory,
 )
-from .qwen3 import Qwen3ForCausalLM, Qwen3Model
+from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model
 from .utils import (
     AutoWeightsLoader,
     PPMissingLayer,
@@ -1748,6 +1748,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
         return PromptUpdateDetails.from_seq(all_token_ids)
 
 
+def mrope_positions_factor(vllm_config: VllmConfig) -> int:
+    # Video pruning keeps the positions of the tokens it drops. A drafter gets
+    # the target's positions, so it follows the target.
+    spec = vllm_config.speculative_config
+    model_config = (spec and spec.target_model_config) or vllm_config.model_config
+    mm_config = model_config.multimodal_config
+    pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled()
+    return 4 if pruned else 1
+
+
+class Qwen3VLDecoderLayer(Qwen3DecoderLayer):
+    def __init__(self, **kwargs) -> None:
+        factor = mrope_positions_factor(get_current_vllm_config())
+        super().__init__(**kwargs, mrope_positions_factor=factor)
+
+
 @support_torch_compile(
     dynamic_arg_dims={
         "input_ids": 0,
@@ -1761,6 +1777,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
     }
 )
 class Qwen3LLMModel(Qwen3Model):
+    decoder_layer_type = Qwen3VLDecoderLayer
+
     def forward(
         self,
         input_ids: torch.Tensor | None,

vllm/model_executor/models/qwen3_vl_moe.py

@@ -56,6 +56,7 @@ from .qwen3_vl import (
     Qwen3VLForConditionalGeneration,
     Qwen3VLMultiModalProcessor,
     Qwen3VLProcessingInfo,
+    mrope_positions_factor,
 )
 from .utils import StageMissingLayer, maybe_prefix
 
@@ -73,6 +74,7 @@ class Qwen3VLMoeDecoderLayer(Qwen3MoeDecoderLayer):
             vllm_config=vllm_config,
             prefix=prefix,
             is_fused_checkpoint_transposed=True,
+            mrope_positions_factor=mrope_positions_factor(vllm_config),
         )
 
 

vllm/models/qwen4_exp/amd/qsa.py

@@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLine
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding import get_rope
 from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
+from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
 from vllm.platforms import current_platform
 from vllm.utils.torch_utils import (
     LayerNameType,
@@ -248,6 +249,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
             head_size=self.head_dim,
             max_position=config.max_position_embeddings,
             rope_parameters=config.rope_parameters,
+            mrope_positions_factor=mrope_positions_factor(vllm_config),
         )
         self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
         self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)

vllm/models/qwen4_exp/nvidia/qsa.py

@@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import (
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
 from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
+from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
 from vllm.model_executor.models.utils import (
     AutoWeightsLoader,
     WeightsMapper,
@@ -427,6 +428,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
             head_size=self.head_dim,
             max_position=config.max_position_embeddings,
             rope_parameters=config.rope_parameters,
+            mrope_positions_factor=mrope_positions_factor(vllm_config),
         )
         self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
         self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)