[Model] Drop the 4x M-RoPE cache headroom for Qwen3-VL, Qwen3.5 and Qwen3.8 (#59853)
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com> Signed-off-by: lesj0610 <lesj0610@gmail.com> Co-authored-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -36,6 +36,7 @@ def get_rope(
|
||||
rope_parameters: dict[str, Any] | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||
mrope_positions_factor: int = 4,
|
||||
) -> RotaryEmbedding:
|
||||
if dtype is None:
|
||||
dtype = torch.get_default_dtype()
|
||||
@@ -78,6 +79,7 @@ def get_rope(
|
||||
rope_parameters_args,
|
||||
dual_chunk_attention_args,
|
||||
dtype,
|
||||
mrope_positions_factor,
|
||||
)
|
||||
if key in _ROPE_DICT:
|
||||
return _ROPE_DICT[key]
|
||||
@@ -118,6 +120,7 @@ def get_rope(
|
||||
dtype,
|
||||
mrope_section=rope_parameters["mrope_section"],
|
||||
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
||||
positions_factor=mrope_positions_factor,
|
||||
)
|
||||
elif "use_fope" in rope_parameters and rope_parameters["use_fope"]:
|
||||
extra_kwargs = {
|
||||
@@ -263,6 +266,7 @@ def get_rope(
|
||||
dtype,
|
||||
mrope_section=rope_parameters["mrope_section"],
|
||||
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
||||
positions_factor=mrope_positions_factor,
|
||||
scaling_factor=scaling_factor,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
@@ -315,8 +315,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
||||
dtype: torch.dtype,
|
||||
mrope_section: list[int] | None = None,
|
||||
mrope_interleaved: bool = False,
|
||||
# YaRN parameters.
|
||||
*,
|
||||
positions_factor: int = 4,
|
||||
# YaRN parameters.
|
||||
scaling_factor: float | None = None,
|
||||
beta_fast: int = 32,
|
||||
beta_slow: int = 1,
|
||||
@@ -345,8 +346,9 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
||||
|
||||
# In Qwen2.5-VL, the maximum index value is related to the duration of
|
||||
# the input video. We enlarge max_position_embeddings to 4 times to get
|
||||
# a larger the cos and sin cache.
|
||||
self.cache_max_position_num = max_position_embeddings * 4
|
||||
# a larger the cos and sin cache. Models whose positions never exceed
|
||||
# the token count pass `positions_factor=1`.
|
||||
self.cache_max_position_num = max_position_embeddings * positions_factor
|
||||
super().__init__(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
|
||||
Reference in New Issue
Block a user