[Model] Drop the 4x M-RoPE cache headroom for Qwen3-VL, Qwen3.5 and Qwen3.8 (#59853)

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
Signed-off-by: lesj0610 <lesj0610@gmail.com>
Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
SeongJun Lee
2026-10-10 06:51:52 +00:00
committed by GitHub
co-authored by lesj0610
parent c1692c85f1
commit a98247ab4d
11 changed files with 56 additions and 6 deletions
@@ -36,6 +36,7 @@ def get_rope(
rope_parameters: dict[str, Any] | None = None,
dtype: torch.dtype | None = None,
dual_chunk_attention_config: dict[str, Any] | None = None,
mrope_positions_factor: int = 4,
) -> RotaryEmbedding:
if dtype is None:
dtype = torch.get_default_dtype()
@@ -78,6 +79,7 @@ def get_rope(
rope_parameters_args,
dual_chunk_attention_args,
dtype,
mrope_positions_factor,
)
if key in _ROPE_DICT:
return _ROPE_DICT[key]
@@ -118,6 +120,7 @@ def get_rope(
dtype,
mrope_section=rope_parameters["mrope_section"],
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
positions_factor=mrope_positions_factor,
)
elif "use_fope" in rope_parameters and rope_parameters["use_fope"]:
extra_kwargs = {
@@ -263,6 +266,7 @@ def get_rope(
dtype,
mrope_section=rope_parameters["mrope_section"],
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
positions_factor=mrope_positions_factor,
scaling_factor=scaling_factor,
**extra_kwargs,
)
@@ -315,8 +315,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
dtype: torch.dtype,
mrope_section: list[int] | None = None,
mrope_interleaved: bool = False,
# YaRN parameters.
*,
positions_factor: int = 4,
# YaRN parameters.
scaling_factor: float | None = None,
beta_fast: int = 32,
beta_slow: int = 1,
@@ -345,8 +346,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
# In Qwen2.5-VL, the maximum index value is related to the duration of
# the input video. We enlarge max_position_embeddings to 4 times to get
# a larger the cos and sin cache.
self.cache_max_position_num = max_position_embeddings * 4
# a larger the cos and sin cache. Models whose positions never exceed
# the token count pass `positions_factor=1`.
self.cache_max_position_num = max_position_embeddings * positions_factor
super().__init__(
head_size,
rotary_dim,