[Bugfix] Align vLLM YaRN with Transformers and stop re-scaling max_model_len (#56446)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
a5b714eee4
commit
c191787a68
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from contextlib import contextmanager
|
||||
@@ -539,6 +540,14 @@ def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None:
|
||||
)
|
||||
rope_parameters["rope_type"] = "default"
|
||||
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
|
||||
elif rope_parameters["rope_type"] == "telechat3-yarn":
|
||||
# TeleChat3 is YaRN with 0.07 in place of YaRN's 0.1 attention
|
||||
# scaling coefficient. Precompute it so the config is plain YaRN.
|
||||
factor = rope_parameters["factor"]
|
||||
rope_parameters["rope_type"] = "yarn"
|
||||
rope_parameters["attention_factor"] = 0.07 * math.log(factor) + 1.0
|
||||
rope_parameters.pop("type", None)
|
||||
logger.warning("Replacing rope_type 'telechat3-yarn' with 'yarn'")
|
||||
|
||||
# Handle nested rope_parameters in interleaved sliding attention models
|
||||
if is_rope_parameters_nested(rope_parameters):
|
||||
|
||||
@@ -2,9 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from packaging.version import Version
|
||||
from transformers import PretrainedConfig, WhisperConfig
|
||||
from transformers import __version__ as TRANSFORMERS_VERSION
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -119,7 +117,6 @@ def _remap_mistral_yarn_args(config: dict) -> dict:
|
||||
"original_max_position_embeddings": ("original_max_position_embeddings", int),
|
||||
"beta": ("beta_fast", float),
|
||||
"alpha": ("beta_slow", float),
|
||||
"apply_scale": ("apply_yarn_scaling", bool),
|
||||
}
|
||||
|
||||
yarn_config = config.get("yarn") or {}
|
||||
@@ -136,9 +133,10 @@ def _remap_mistral_yarn_args(config: dict) -> dict:
|
||||
# Cast to remove Transformers > v5 type warnings
|
||||
config["rope_parameters"][new_name] = cast(yarn_config.pop(old_name))
|
||||
|
||||
# Ignore apply_yarn_scaling in Transformers > v5 RoPE validation to remove warnings
|
||||
if Version(TRANSFORMERS_VERSION) >= Version("5.3.0.dev0"):
|
||||
config["ignore_keys_at_rope_validation"] = {"apply_yarn_scaling"}
|
||||
# `apply_scale: false` means no magnitude correction, which Transformers
|
||||
# spells as an explicit attention_factor of 1.
|
||||
if not yarn_config.pop("apply_scale", True):
|
||||
config["rope_parameters"]["attention_factor"] = 1.0
|
||||
|
||||
assert len(yarn_config) == 0, f"Unparsed yarn config: {yarn_config}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user