[Bugfix] Align vLLM YaRN with Transformers and stop re-scaling max_model_len (#56446)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
a5b714eee4
commit
c191787a68
@@ -1853,6 +1853,54 @@ def test_get_and_verify_max_len_with_nope_layers(
|
||||
assert hf_config.rope_parameters["full_attention"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rope_type", "factor", "expected_max_len"),
|
||||
[
|
||||
# TeleChat3-36B-Thinking: 32768 already scaled from 8192 by 4
|
||||
("yarn", 4.0, 32768),
|
||||
# sarvam-105b: declares factor 40 but only serves 131072 of it
|
||||
("deepseek_yarn", 40.0, 32768),
|
||||
("deepseek_llama_scaling", 40.0, 32768),
|
||||
# Non-YaRN scaling still multiplies
|
||||
("linear", 4.0, 131072),
|
||||
],
|
||||
)
|
||||
def test_get_and_verify_max_len_yarn_is_already_scaled(
|
||||
rope_type, factor, expected_max_len
|
||||
):
|
||||
"""YaRN variants must not re-apply `factor` to max_position_embeddings.
|
||||
|
||||
Transformers treats max_position_embeddings as the final context length
|
||||
for every YaRN variant, so scaling it again overstates the limit and lets
|
||||
requests past the end of the cos/sin cache.
|
||||
"""
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.model import _get_and_verify_max_len
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
|
||||
hf_config = PretrainedConfig(max_position_embeddings=32768)
|
||||
hf_config.rope_parameters = {
|
||||
"rope_type": rope_type,
|
||||
"factor": factor,
|
||||
"original_max_position_embeddings": 8192,
|
||||
}
|
||||
model_arch_config = ModelArchConfigConvertorBase(hf_config, hf_config).convert()
|
||||
|
||||
actual_max_len = _get_and_verify_max_len(
|
||||
hf_config=hf_config,
|
||||
model_arch_config=model_arch_config,
|
||||
tokenizer_config=None,
|
||||
max_model_len=None,
|
||||
disable_sliding_window=False,
|
||||
sliding_window=None,
|
||||
)
|
||||
|
||||
assert actual_max_len == expected_max_len
|
||||
|
||||
|
||||
class MockConfig:
|
||||
"""Simple mock object for testing maybe_pull_model_tokenizer_for_runai"""
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ only get the `eos_token_id` from the tokenizer as defined by
|
||||
`BaseRenderer.get_eos_token_id`.
|
||||
"""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -52,6 +53,31 @@ def test_patch_legacy_rope_type_preserves_nope_layers():
|
||||
}
|
||||
|
||||
|
||||
def test_patch_legacy_rope_type_normalizes_telechat3_yarn():
|
||||
"""TeleChat3's RoPE is YaRN with 0.07 in place of the usual 0.1.
|
||||
|
||||
Encoding that as a precomputed attention_factor keeps the config
|
||||
plain YaRN, which Transformers and every "yarn" guard understand.
|
||||
`mscale` cannot express it: Transformers only applies mscale when
|
||||
mscale_all_dim is also truthy.
|
||||
"""
|
||||
rope_parameters = {
|
||||
"type": "telechat3-yarn",
|
||||
"rope_type": "telechat3-yarn",
|
||||
"factor": 4.0,
|
||||
"original_max_position_embeddings": 8192,
|
||||
}
|
||||
|
||||
patch_legacy_rope_type(rope_parameters)
|
||||
|
||||
assert rope_parameters == {
|
||||
"rope_type": "yarn",
|
||||
"factor": 4.0,
|
||||
"original_max_position_embeddings": 8192,
|
||||
"attention_factor": pytest.approx(0.07 * math.log(4.0) + 1.0),
|
||||
}
|
||||
|
||||
|
||||
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
|
||||
layer_types = ["linear_attention", "deepseek_sparse_attention"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user