[Core][Model] Unify XD-RoPE into M-RoPE and derive the channel count (#56078)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-09 09:23:30 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent c55e15a44e
commit 719284fe15
18 changed files with 186 additions and 505 deletions
@@ -46,7 +46,6 @@ def get_test_cases() -> list[RotaryEmbeddingTestCase]:
Ernie4_5_VLRotaryEmbedding,
)
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
from vllm.model_executor.layers.rotary_embedding.xdrope import XDRotaryEmbedding
common_kwargs = {
"head_size": 128,
@@ -77,20 +76,6 @@ def get_test_cases() -> list[RotaryEmbeddingTestCase]:
expect_forward_native=False,
expect_forward=True,
),
# XDRotaryEmbedding tests
RotaryEmbeddingTestCase(
name="XDRotaryEmbedding.forward",
rope_class=XDRotaryEmbedding,
rope_kwargs={
**common_kwargs,
"scaling_alpha": 1.0,
"xdrope_section": [16, 16, 16, 16],
},
method_name="forward",
positions_shape=(4, 32), # 4D for P/W/H/T
expect_forward_native=False,
expect_forward=True,
),
# Ernie4_5_VLRotaryEmbedding tests
RotaryEmbeddingTestCase(
name="Ernie4_5_VLRotaryEmbedding.forward_native",
+19
View File
@@ -769,3 +769,22 @@ def test_attention_scale_rejects_unresolvable_expression():
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
fuser.scale(Attention())
class _ImageOnlyMRoPEModel:
"""`get_rope_index` without `video_grid_thw` or `**kwargs` (e.g. HunYuanVL)."""
def get_rope_index(self, input_ids, image_grid_thw):
seq_len = input_ids.shape[-1]
positions = torch.arange(seq_len).view(1, 1, -1).expand(4, 1, -1)
return positions, torch.tensor([0])
def test_get_mrope_input_positions_omits_unsupported_grid_kwargs():
"""Optional grids the model can't accept must not be passed when empty."""
mixin = SimpleNamespace(model=_ImageOnlyMRoPEModel())
positions, delta = MultiModalMixin.get_mrope_input_positions(mixin, [1, 2, 3], [])
assert positions.shape == (4, 3)
assert delta == 0
+48
View File
@@ -18,8 +18,10 @@ from vllm.tokenizers import get_tokenizer
from vllm.transformers_utils import config as config_module
from vllm.transformers_utils.config import (
get_safetensors_params_metadata,
mrope_num_dims,
patch_legacy_rope_type,
try_get_generation_config,
uses_mrope,
)
from vllm.transformers_utils.configs.glm5_next import (
Glm5NextConfig,
@@ -176,3 +178,49 @@ def test_safetensors_metadata_of_repo_without_safetensors():
assert get_safetensors_params_metadata("some/pytorch-only-model") == {}
get_safetensors_metadata.assert_called_once()
@pytest.mark.parametrize(
("section_key", "mrope_section", "expected_num_dims"),
[
("mrope_section", [16, 24, 24], 3),
("mrope_section", [16, 16, 16, 16], 4),
# Interleaved M-RoPE takes 2 sections but still consumes 3D positions
("mrope_section", [32, 32], 3),
# HunYuan-VL checkpoints ship the section under its legacy name
("xdrope_section", [16, 16, 16, 16], 4),
],
)
def test_mrope_num_dims(section_key, mrope_section, expected_num_dims):
config = PretrainedConfig()
config.rope_parameters = {"rope_type": "default", section_key: mrope_section}
assert uses_mrope(config)
assert mrope_num_dims(config) == expected_num_dims
@pytest.mark.parametrize("section_name", ["mrope_section", "xdrope_section"])
def test_mrope_num_dims_from_config_attribute(section_name):
"""Some configs expose the section as an attribute rather than under
`rope_parameters`."""
config = PretrainedConfig()
setattr(config, section_name, [16, 16, 16, 16])
assert uses_mrope(config)
assert mrope_num_dims(config) == 4
def test_mrope_num_dims_from_nested_rope_parameters():
"""Sections nested by layer type must be found, not silently defaulted."""
config = PretrainedConfig()
config.rope_parameters = {
"full_attention": {"mrope_section": [16, 16, 16, 16]},
"linear_attention": {"rope_type": "default"},
}
assert uses_mrope(config)
assert mrope_num_dims(config) == 4
def test_mrope_num_dims_without_mrope():
assert mrope_num_dims(PretrainedConfig()) == 0
-18
View File
@@ -105,24 +105,6 @@ def _mm_feature_mixed(offset: int, length: int) -> MultiModalFeatureSpec:
)
def test_strip_covered_mm_data_xdrope() -> None:
"""XD-RoPE models (e.g. HunyuanOCR) compute positions the same way, so a
covered item must keep its grid dims: stripping them made the worker index
an empty ``image_grid_thw`` and crash the engine on the second request with
an identical prompt."""
covered = _mm_feature_mixed(offset=0, length=100)
uncovered = _mm_feature_mixed(offset=300, length=100)
stripped = strip_covered_mm_data(
[covered, uncovered], num_computed_tokens=250, uses_xdrope=True
)
assert stripped[0].data is not None
assert list(stripped[0].data.keys()) == ["image_grid_thw"]
assert stripped[1].data is not None
assert set(stripped[1].data.keys()) == {"pixel_values", "image_grid_thw"}
def test_strip_covered_mm_data_mrope() -> None:
"""For M-RoPE models, covered items keep their keep_on_cpu metadata fields
(the worker needs them to compute positions); payload fields are dropped."""