[Bugfix] Handle null RoPE parameters for NoPE layers (#55949)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -1533,6 +1533,49 @@ def test_get_and_verify_max_len(
|
||||
assert actual_max_len == expected_max_len
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_model_len", [None, 1024])
|
||||
@pytest.mark.parametrize(
|
||||
("rope_parameters", "expected_max_len"),
|
||||
[
|
||||
({"rope_type": "default"}, 4096),
|
||||
({"rope_type": "linear", "factor": 2.0}, 8192),
|
||||
({"rope_type": "longrope"}, 2048),
|
||||
],
|
||||
)
|
||||
def test_get_and_verify_max_len_with_nope_layers(
|
||||
max_model_len, rope_parameters, expected_max_len
|
||||
):
|
||||
"""NoPE layers do not prevent deriving or scaling the context length."""
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.model import _get_and_verify_max_len
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
|
||||
hf_config = PretrainedConfig(
|
||||
max_position_embeddings=4096,
|
||||
original_max_position_embeddings=2048,
|
||||
)
|
||||
hf_config.rope_parameters = {
|
||||
"full_attention": None,
|
||||
"sliding_attention": rope_parameters,
|
||||
}
|
||||
model_arch_config = ModelArchConfigConvertorBase(hf_config, hf_config).convert()
|
||||
|
||||
actual_max_len = _get_and_verify_max_len(
|
||||
hf_config=hf_config,
|
||||
model_arch_config=model_arch_config,
|
||||
tokenizer_config=None,
|
||||
max_model_len=max_model_len,
|
||||
disable_sliding_window=False,
|
||||
sliding_window=None,
|
||||
)
|
||||
|
||||
assert actual_max_len == (max_model_len or expected_max_len)
|
||||
assert hf_config.rope_parameters["full_attention"] is None
|
||||
|
||||
|
||||
class MockConfig:
|
||||
"""Simple mock object for testing maybe_pull_model_tokenizer_for_runai"""
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.tokenizers import get_tokenizer
|
||||
from vllm.transformers_utils import config as config_module
|
||||
from vllm.transformers_utils.config import (
|
||||
get_safetensors_params_metadata,
|
||||
patch_legacy_rope_type,
|
||||
try_get_generation_config,
|
||||
)
|
||||
from vllm.transformers_utils.configs.glm5_next import (
|
||||
@@ -27,6 +28,28 @@ from vllm.transformers_utils.configs.glm5_next import (
|
||||
)
|
||||
|
||||
|
||||
def test_patch_legacy_rope_type_preserves_nope_layers():
|
||||
"""NoPE layers stay disabled while later RoPE layers are normalized."""
|
||||
rope_parameters = {
|
||||
"full_attention": None,
|
||||
"sliding_attention": {
|
||||
"type": "mrope",
|
||||
"mrope_section": [24, 20, 20],
|
||||
},
|
||||
}
|
||||
|
||||
patch_legacy_rope_type(rope_parameters)
|
||||
|
||||
assert rope_parameters == {
|
||||
"full_attention": None,
|
||||
"sliding_attention": {
|
||||
"type": "mrope",
|
||||
"rope_type": "default",
|
||||
"mrope_section": [24, 20, 20],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
|
||||
layer_types = ["linear_attention", "deepseek_sparse_attention"]
|
||||
|
||||
|
||||
@@ -2449,6 +2449,13 @@ def _get_and_verify_max_len(
|
||||
rope_parameters = getattr(hf_config, "rope_parameters", None)
|
||||
if rope_parameters and not is_rope_parameters_nested(rope_parameters):
|
||||
rope_parameters = {"": rope_parameters}
|
||||
if rope_parameters is not None:
|
||||
# Layers without RoPE do not contribute to context length scaling.
|
||||
rope_parameters = {
|
||||
layer_type: rp
|
||||
for layer_type, rp in rope_parameters.items()
|
||||
if rp is not None
|
||||
}
|
||||
|
||||
# NOTE(woosuk): Gemma3's max_model_len (128K) is already scaled by RoPE
|
||||
# scaling, so we skip applying the scaling factor again.
|
||||
|
||||
@@ -541,7 +541,8 @@ def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None:
|
||||
# Handle nested rope_parameters in interleaved sliding attention models
|
||||
if is_rope_parameters_nested(rope_parameters):
|
||||
for rope_parameters_layer_type in rope_parameters.values():
|
||||
_patch_legacy_rope_type(rope_parameters_layer_type)
|
||||
if rope_parameters_layer_type is not None:
|
||||
_patch_legacy_rope_type(rope_parameters_layer_type)
|
||||
else:
|
||||
_patch_legacy_rope_type(rope_parameters)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user