[Model] Drop the 4x M-RoPE cache headroom for Qwen3-VL, Qwen3.5 and Qwen3.8 (#59853)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Signed-off-by: lesj0610 <lesj0610@gmail.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -508,6 +508,10 @@ class ModelConfig:
|
|||||||
# here early.
|
# here early.
|
||||||
if self.multimodal_config:
|
if self.multimodal_config:
|
||||||
factors["language_model_only"] = self.multimodal_config.language_model_only
|
factors["language_model_only"] = self.multimodal_config.language_model_only
|
||||||
|
# Sizes Qwen3.5's M-RoPE cache.
|
||||||
|
factors["video_pruning"] = (
|
||||||
|
self.multimodal_config.is_multimodal_pruning_enabled()
|
||||||
|
)
|
||||||
return hash_factors(factors)
|
return hash_factors(factors)
|
||||||
|
|
||||||
def _update_nested(
|
def _update_nested(
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ def get_rope(
|
|||||||
rope_parameters: dict[str, Any] | None = None,
|
rope_parameters: dict[str, Any] | None = None,
|
||||||
dtype: torch.dtype | None = None,
|
dtype: torch.dtype | None = None,
|
||||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||||
|
mrope_positions_factor: int = 4,
|
||||||
) -> RotaryEmbedding:
|
) -> RotaryEmbedding:
|
||||||
if dtype is None:
|
if dtype is None:
|
||||||
dtype = torch.get_default_dtype()
|
dtype = torch.get_default_dtype()
|
||||||
@@ -78,6 +79,7 @@ def get_rope(
|
|||||||
rope_parameters_args,
|
rope_parameters_args,
|
||||||
dual_chunk_attention_args,
|
dual_chunk_attention_args,
|
||||||
dtype,
|
dtype,
|
||||||
|
mrope_positions_factor,
|
||||||
)
|
)
|
||||||
if key in _ROPE_DICT:
|
if key in _ROPE_DICT:
|
||||||
return _ROPE_DICT[key]
|
return _ROPE_DICT[key]
|
||||||
@@ -118,6 +120,7 @@ def get_rope(
|
|||||||
dtype,
|
dtype,
|
||||||
mrope_section=rope_parameters["mrope_section"],
|
mrope_section=rope_parameters["mrope_section"],
|
||||||
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
||||||
|
positions_factor=mrope_positions_factor,
|
||||||
)
|
)
|
||||||
elif "use_fope" in rope_parameters and rope_parameters["use_fope"]:
|
elif "use_fope" in rope_parameters and rope_parameters["use_fope"]:
|
||||||
extra_kwargs = {
|
extra_kwargs = {
|
||||||
@@ -263,6 +266,7 @@ def get_rope(
|
|||||||
dtype,
|
dtype,
|
||||||
mrope_section=rope_parameters["mrope_section"],
|
mrope_section=rope_parameters["mrope_section"],
|
||||||
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
||||||
|
positions_factor=mrope_positions_factor,
|
||||||
scaling_factor=scaling_factor,
|
scaling_factor=scaling_factor,
|
||||||
**extra_kwargs,
|
**extra_kwargs,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -315,8 +315,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
|||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
mrope_section: list[int] | None = None,
|
mrope_section: list[int] | None = None,
|
||||||
mrope_interleaved: bool = False,
|
mrope_interleaved: bool = False,
|
||||||
# YaRN parameters.
|
|
||||||
*,
|
*,
|
||||||
|
positions_factor: int = 4,
|
||||||
|
# YaRN parameters.
|
||||||
scaling_factor: float | None = None,
|
scaling_factor: float | None = None,
|
||||||
beta_fast: int = 32,
|
beta_fast: int = 32,
|
||||||
beta_slow: int = 1,
|
beta_slow: int = 1,
|
||||||
@@ -345,8 +346,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
|||||||
|
|
||||||
# In Qwen2.5-VL, the maximum index value is related to the duration of
|
# In Qwen2.5-VL, the maximum index value is related to the duration of
|
||||||
# the input video. We enlarge max_position_embeddings to 4 times to get
|
# the input video. We enlarge max_position_embeddings to 4 times to get
|
||||||
# a larger the cos and sin cache.
|
# a larger the cos and sin cache. Models whose positions never exceed
|
||||||
self.cache_max_position_num = max_position_embeddings * 4
|
# the token count pass `positions_factor=1`.
|
||||||
|
self.cache_max_position_num = max_position_embeddings * positions_factor
|
||||||
super().__init__(
|
super().__init__(
|
||||||
head_size,
|
head_size,
|
||||||
rotary_dim,
|
rotary_dim,
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ class Qwen3Attention(nn.Module):
|
|||||||
attn_type: str = AttentionType.DECODER,
|
attn_type: str = AttentionType.DECODER,
|
||||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||||
per_layer_sliding_window: int | None = None,
|
per_layer_sliding_window: int | None = None,
|
||||||
|
mrope_positions_factor: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.hidden_size = hidden_size
|
self.hidden_size = hidden_size
|
||||||
@@ -124,6 +125,7 @@ class Qwen3Attention(nn.Module):
|
|||||||
max_position=max_position,
|
max_position=max_position,
|
||||||
rope_parameters=rope_parameters,
|
rope_parameters=rope_parameters,
|
||||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||||
|
mrope_positions_factor=mrope_positions_factor,
|
||||||
)
|
)
|
||||||
attn_cls = (
|
attn_cls = (
|
||||||
EncoderOnlyAttention
|
EncoderOnlyAttention
|
||||||
@@ -179,6 +181,7 @@ class Qwen3DecoderLayer(nn.Module):
|
|||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
per_layer_sliding_window: int | None = None,
|
per_layer_sliding_window: int | None = None,
|
||||||
|
mrope_positions_factor: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.hidden_size = config.hidden_size
|
self.hidden_size = config.hidden_size
|
||||||
@@ -211,6 +214,7 @@ class Qwen3DecoderLayer(nn.Module):
|
|||||||
attn_type=attn_type,
|
attn_type=attn_type,
|
||||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||||
per_layer_sliding_window=per_layer_sliding_window,
|
per_layer_sliding_window=per_layer_sliding_window,
|
||||||
|
mrope_positions_factor=mrope_positions_factor,
|
||||||
)
|
)
|
||||||
self.mlp = Qwen3MLP(
|
self.mlp = Qwen3MLP(
|
||||||
hidden_size=self.hidden_size,
|
hidden_size=self.hidden_size,
|
||||||
@@ -263,9 +267,13 @@ ALL_DECODER_LAYER_TYPES = {
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
class Qwen3Model(Qwen2Model):
|
class Qwen3Model(Qwen2Model):
|
||||||
|
decoder_layer_type: type[nn.Module] = Qwen3DecoderLayer
|
||||||
|
|
||||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
vllm_config=vllm_config, prefix=prefix, decoder_layer_type=Qwen3DecoderLayer
|
vllm_config=vllm_config,
|
||||||
|
prefix=prefix,
|
||||||
|
decoder_layer_type=self.decoder_layer_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ from .qwen3_vl import (
|
|||||||
Qwen3VLForConditionalGeneration,
|
Qwen3VLForConditionalGeneration,
|
||||||
Qwen3VLMultiModalProcessor,
|
Qwen3VLMultiModalProcessor,
|
||||||
Qwen3VLProcessingInfo,
|
Qwen3VLProcessingInfo,
|
||||||
|
mrope_positions_factor,
|
||||||
)
|
)
|
||||||
from .utils import (
|
from .utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
@@ -161,6 +162,7 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
|
|||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.self_attn",
|
prefix=f"{prefix}.self_attn",
|
||||||
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
reduce_results=not self.use_attn_reduce_scatter_for_moe,
|
||||||
|
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Invalid layer_type {self.layer_type}")
|
raise ValueError(f"Invalid layer_type {self.layer_type}")
|
||||||
|
|||||||
@@ -256,6 +256,7 @@ class Qwen3MoeAttention(nn.Module):
|
|||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||||
|
mrope_positions_factor: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.hidden_size = hidden_size
|
self.hidden_size = hidden_size
|
||||||
@@ -303,6 +304,7 @@ class Qwen3MoeAttention(nn.Module):
|
|||||||
max_position=max_position_embeddings,
|
max_position=max_position_embeddings,
|
||||||
rope_parameters=rope_parameters,
|
rope_parameters=rope_parameters,
|
||||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||||
|
mrope_positions_factor=mrope_positions_factor,
|
||||||
)
|
)
|
||||||
attention_kwargs: dict[str, Any] = {}
|
attention_kwargs: dict[str, Any] = {}
|
||||||
if dual_chunk_attention_config:
|
if dual_chunk_attention_config:
|
||||||
@@ -351,6 +353,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
|||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
is_fused_checkpoint_transposed: bool = False,
|
is_fused_checkpoint_transposed: bool = False,
|
||||||
|
mrope_positions_factor: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@@ -376,6 +379,7 @@ class Qwen3MoeDecoderLayer(nn.Module):
|
|||||||
quant_config=quant_config,
|
quant_config=quant_config,
|
||||||
prefix=f"{prefix}.self_attn",
|
prefix=f"{prefix}.self_attn",
|
||||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||||
|
mrope_positions_factor=mrope_positions_factor,
|
||||||
)
|
)
|
||||||
|
|
||||||
# `mlp_only_layers` in the config.
|
# `mlp_only_layers` in the config.
|
||||||
|
|||||||
@@ -280,6 +280,7 @@ class Qwen3NextAttention(nn.Module):
|
|||||||
quant_config: QuantizationConfig | None = None,
|
quant_config: QuantizationConfig | None = None,
|
||||||
reduce_results: bool = True,
|
reduce_results: bool = True,
|
||||||
prefix: str = "",
|
prefix: str = "",
|
||||||
|
mrope_positions_factor: int = 4,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.config = config
|
self.config = config
|
||||||
@@ -331,6 +332,7 @@ class Qwen3NextAttention(nn.Module):
|
|||||||
max_position=config.max_position_embeddings,
|
max_position=config.max_position_embeddings,
|
||||||
rope_parameters=config.rope_parameters,
|
rope_parameters=config.rope_parameters,
|
||||||
dual_chunk_attention_config=self.dual_chunk_attention_config,
|
dual_chunk_attention_config=self.dual_chunk_attention_config,
|
||||||
|
mrope_positions_factor=mrope_positions_factor,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL
|
# Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ from vllm.compilation.decorators import (
|
|||||||
should_torch_compile_mm_encoder,
|
should_torch_compile_mm_encoder,
|
||||||
support_torch_compile,
|
support_torch_compile,
|
||||||
)
|
)
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig, get_current_vllm_config
|
||||||
from vllm.config.multimodal import (
|
from vllm.config.multimodal import (
|
||||||
MultiModalConfig,
|
MultiModalConfig,
|
||||||
MultiModalDummyOptions,
|
MultiModalDummyOptions,
|
||||||
@@ -148,7 +148,7 @@ from .qwen2_vl import (
|
|||||||
Qwen2VLProcessingInfo,
|
Qwen2VLProcessingInfo,
|
||||||
_create_qwen2vl_field_factory,
|
_create_qwen2vl_field_factory,
|
||||||
)
|
)
|
||||||
from .qwen3 import Qwen3ForCausalLM, Qwen3Model
|
from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model
|
||||||
from .utils import (
|
from .utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
PPMissingLayer,
|
PPMissingLayer,
|
||||||
@@ -1748,6 +1748,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
return PromptUpdateDetails.from_seq(all_token_ids)
|
return PromptUpdateDetails.from_seq(all_token_ids)
|
||||||
|
|
||||||
|
|
||||||
|
def mrope_positions_factor(vllm_config: VllmConfig) -> int:
|
||||||
|
# Video pruning keeps the positions of the tokens it drops. A drafter gets
|
||||||
|
# the target's positions, so it follows the target.
|
||||||
|
spec = vllm_config.speculative_config
|
||||||
|
model_config = (spec and spec.target_model_config) or vllm_config.model_config
|
||||||
|
mm_config = model_config.multimodal_config
|
||||||
|
pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled()
|
||||||
|
return 4 if pruned else 1
|
||||||
|
|
||||||
|
|
||||||
|
class Qwen3VLDecoderLayer(Qwen3DecoderLayer):
|
||||||
|
def __init__(self, **kwargs) -> None:
|
||||||
|
factor = mrope_positions_factor(get_current_vllm_config())
|
||||||
|
super().__init__(**kwargs, mrope_positions_factor=factor)
|
||||||
|
|
||||||
|
|
||||||
@support_torch_compile(
|
@support_torch_compile(
|
||||||
dynamic_arg_dims={
|
dynamic_arg_dims={
|
||||||
"input_ids": 0,
|
"input_ids": 0,
|
||||||
@@ -1761,6 +1777,8 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
class Qwen3LLMModel(Qwen3Model):
|
class Qwen3LLMModel(Qwen3Model):
|
||||||
|
decoder_layer_type = Qwen3VLDecoderLayer
|
||||||
|
|
||||||
def forward(
|
def forward(
|
||||||
self,
|
self,
|
||||||
input_ids: torch.Tensor | None,
|
input_ids: torch.Tensor | None,
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ from .qwen3_vl import (
|
|||||||
Qwen3VLForConditionalGeneration,
|
Qwen3VLForConditionalGeneration,
|
||||||
Qwen3VLMultiModalProcessor,
|
Qwen3VLMultiModalProcessor,
|
||||||
Qwen3VLProcessingInfo,
|
Qwen3VLProcessingInfo,
|
||||||
|
mrope_positions_factor,
|
||||||
)
|
)
|
||||||
from .utils import StageMissingLayer, maybe_prefix
|
from .utils import StageMissingLayer, maybe_prefix
|
||||||
|
|
||||||
@@ -73,6 +74,7 @@ class Qwen3VLMoeDecoderLayer(Qwen3MoeDecoderLayer):
|
|||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
is_fused_checkpoint_transposed=True,
|
is_fused_checkpoint_transposed=True,
|
||||||
|
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLine
|
|||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||||
|
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.torch_utils import (
|
from vllm.utils.torch_utils import (
|
||||||
LayerNameType,
|
LayerNameType,
|
||||||
@@ -248,6 +249,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|||||||
head_size=self.head_dim,
|
head_size=self.head_dim,
|
||||||
max_position=config.max_position_embeddings,
|
max_position=config.max_position_embeddings,
|
||||||
rope_parameters=config.rope_parameters,
|
rope_parameters=config.rope_parameters,
|
||||||
|
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||||
)
|
)
|
||||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import (
|
|||||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||||
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_rope
|
||||||
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
from vllm.model_executor.models.qwen3_next import Qwen3NextAttention
|
||||||
|
from vllm.model_executor.models.qwen3_vl import mrope_positions_factor
|
||||||
from vllm.model_executor.models.utils import (
|
from vllm.model_executor.models.utils import (
|
||||||
AutoWeightsLoader,
|
AutoWeightsLoader,
|
||||||
WeightsMapper,
|
WeightsMapper,
|
||||||
@@ -427,6 +428,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
|||||||
head_size=self.head_dim,
|
head_size=self.head_dim,
|
||||||
max_position=config.max_position_embeddings,
|
max_position=config.max_position_embeddings,
|
||||||
rope_parameters=config.rope_parameters,
|
rope_parameters=config.rope_parameters,
|
||||||
|
mrope_positions_factor=mrope_positions_factor(vllm_config),
|
||||||
)
|
)
|
||||||
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||||
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||||
|
|||||||
Reference in New Issue
Block a user