[Core][Model] Unify XD-RoPE into M-RoPE and derive the channel count (#56078)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
c55e15a44e
commit
719284fe15
@@ -24,7 +24,6 @@ from .mrope_interleaved import MRotaryEmbeddingInterleaved
|
||||
from .ntk_scaling_rope import NTKScalingRotaryEmbedding
|
||||
from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
|
||||
from .telechat3_scaling_rope import TeleChat3RoPEScaledRotaryEmbedding
|
||||
from .xdrope import XDRotaryEmbedding
|
||||
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
|
||||
|
||||
_ROPE_DICT: dict[tuple[Any, ...], RotaryEmbedding] = {}
|
||||
@@ -228,18 +227,6 @@ def get_rope(
|
||||
raise ValueError(
|
||||
"Dynamic rope scaling must contain either 'alpha' or 'factor' field"
|
||||
)
|
||||
elif scaling_type == "xdrope":
|
||||
scaling_alpha = rope_parameters["alpha"]
|
||||
rotary_emb = XDRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_alpha,
|
||||
dtype,
|
||||
xdrope_section=rope_parameters["xdrope_section"],
|
||||
)
|
||||
elif scaling_type == "yarn":
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
original_max_position = rope_parameters["original_max_position_embeddings"]
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from .dynamic_ntk_alpha_rope import DynamicNTKAlphaRotaryEmbedding
|
||||
|
||||
|
||||
class XDRotaryEmbedding(DynamicNTKAlphaRotaryEmbedding):
|
||||
"""DynamicNTKAlphaRotaryEmbedding extended with MultiModal(XD) Sections.
|
||||
|
||||
Based on the original DynamicNTKAlphaRotaryEmbedding implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
scaling_alpha: float,
|
||||
dtype: torch.dtype,
|
||||
xdrope_section: list[int],
|
||||
) -> None:
|
||||
self.xdrope_section = xdrope_section
|
||||
super().__init__(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position_embeddings,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_alpha,
|
||||
dtype,
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""PyTorch-native implementation equivalent to forward().
|
||||
|
||||
Args:
|
||||
positions:
|
||||
[4, num_tokens] (P/W/H/T positions with multimodal inputs)
|
||||
query: [num_tokens, num_heads * head_size]
|
||||
key: [num_tokens, num_kv_heads * head_size]
|
||||
"""
|
||||
assert positions.ndim == 2
|
||||
assert key is not None
|
||||
|
||||
num_tokens = positions.shape[-1]
|
||||
cos_sin = self.cos_sin_cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
cos = torch.cat(
|
||||
[m[i] for i, m in enumerate(cos.split(self.xdrope_section, dim=-1))], dim=-1
|
||||
)
|
||||
sin = torch.cat(
|
||||
[m[i] for i, m in enumerate(sin.split(self.xdrope_section, dim=-1))], dim=-1
|
||||
)
|
||||
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
query_rot = self.apply_rotary_emb.forward_native(
|
||||
query_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
|
||||
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
key_rot = self.apply_rotary_emb.forward_native(
|
||||
key_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""PyTorch-native implementation equivalent to forward().
|
||||
|
||||
Args:
|
||||
positions:
|
||||
[4, num_tokens] (P/W/H/T positions with multimodal inputs)
|
||||
query: [num_tokens, num_heads * head_size]
|
||||
key: [num_tokens, num_kv_heads * head_size]
|
||||
"""
|
||||
assert positions.ndim == 2
|
||||
assert key is not None
|
||||
|
||||
num_tokens = positions.shape[-1]
|
||||
cos_sin = self.cos_sin_cache[positions]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
cos = torch.cat(
|
||||
[m[i] for i, m in enumerate(cos.split(self.xdrope_section, dim=-1))], dim=-1
|
||||
)
|
||||
sin = torch.cat(
|
||||
[m[i] for i, m in enumerate(sin.split(self.xdrope_section, dim=-1))], dim=-1
|
||||
)
|
||||
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
query_rot = self.apply_rotary_emb(
|
||||
query_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
|
||||
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
key_rot = self.apply_rotary_emb(
|
||||
key_rot,
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
|
||||
return query, key
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# No fused XPU kernel for XD-sectioned RoPE; the base
|
||||
# RotaryEmbedding.forward_xpu calls the generic C++ kernel, which
|
||||
# rejects the 2-D [4, num_tokens] xdrope positions. Use the native path.
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
|
||||
@staticmethod
|
||||
def get_next_input_positions(
|
||||
context_len: int,
|
||||
seq_len: int,
|
||||
xd_sections: int = 4,
|
||||
) -> list[list[int]]:
|
||||
return [list(range(context_len, seq_len)) for _ in range(xd_sections)]
|
||||
|
||||
@staticmethod
|
||||
def get_next_input_positions_tensor(
|
||||
out: np.ndarray,
|
||||
out_offset: int,
|
||||
context_len: int,
|
||||
num_new_tokens: int,
|
||||
):
|
||||
values = np.arange(
|
||||
context_len,
|
||||
context_len + num_new_tokens,
|
||||
dtype=out.dtype,
|
||||
)
|
||||
out[:, out_offset : out_offset + num_new_tokens] = values
|
||||
@@ -1783,7 +1783,8 @@ class SupportsMRoPE(Protocol):
|
||||
|
||||
Returns:
|
||||
Tuple of `(llm_positions, mrope_position_delta)`
|
||||
- llm_positions: Tensor of shape `[3, num_tokens]` with T/H/W positions
|
||||
- llm_positions: Tensor of shape `[num_dims, num_tokens]`, one row
|
||||
per M-RoPE position channel (e.g. T/H/W)
|
||||
- mrope_position_delta: Delta for position calculations
|
||||
"""
|
||||
...
|
||||
@@ -1803,55 +1804,6 @@ def supports_mrope(
|
||||
return isinstance(model, SupportsMRoPE)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsXDRoPE(Protocol):
|
||||
"""The interface required for all models that support XD-RoPE."""
|
||||
|
||||
supports_xdrope: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
A flag that indicates this model supports XD-RoPE.
|
||||
|
||||
Note:
|
||||
There is no need to redefine this flag if this class is in the
|
||||
XDRope of your model class.
|
||||
"""
|
||||
|
||||
def get_xdrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list["MultiModalFeatureSpec"],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Get XD-RoPE input positions and delta value for this specific model.
|
||||
|
||||
This method should be implemented by each model that supports XD-RoPE
|
||||
to provide model-specific logic for computing input positions.
|
||||
|
||||
Args:
|
||||
input_tokens: List of input token IDs
|
||||
mm_features: Information about each multi-modal data item
|
||||
|
||||
Returns:
|
||||
llm_positions: Tensor of shape `[xdrope_dim, num_tokens]` with
|
||||
4D(P/W/H/T) or 3D(W/H/T) positions.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_xdrope(model: type[object]) -> TypeIs[type[SupportsXDRoPE]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_xdrope(model: object) -> TypeIs[SupportsXDRoPE]: ...
|
||||
|
||||
|
||||
def supports_xdrope(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsXDRoPE]] | TypeIs[SupportsXDRoPE]:
|
||||
return isinstance(model, SupportsXDRoPE)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEncoderCudaGraph(Protocol):
|
||||
"""Interface for models whose vision encoder supports CUDA graph
|
||||
|
||||
@@ -1236,17 +1236,31 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
video_grid_thw = torch.stack(video_grid_thw) if video_grid_thw else None
|
||||
|
||||
# `get_rope_index` doesn't always accept arbitrary `kwargs`
|
||||
kwargs = {}
|
||||
if not hasattr(self, "_get_rope_index_accepts_mm_token_type_ids"):
|
||||
if not hasattr(self, "_get_rope_index_kwarg_names"):
|
||||
import inspect
|
||||
|
||||
sig = inspect.signature(self.model.get_rope_index)
|
||||
params = sig.parameters
|
||||
self._get_rope_index_accepts_mm_token_type_ids = (
|
||||
"mm_token_type_ids" in params
|
||||
or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
params = inspect.signature(self.model.get_rope_index).parameters
|
||||
self._get_rope_index_kwarg_names = (
|
||||
None
|
||||
if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
else frozenset(params)
|
||||
)
|
||||
if self._get_rope_index_accepts_mm_token_type_ids:
|
||||
|
||||
kwarg_names = self._get_rope_index_kwarg_names
|
||||
|
||||
def accepts_kwarg(name: str) -> bool:
|
||||
return kwarg_names is None or name in kwarg_names
|
||||
|
||||
# Drop a grid the model can't accept only when there is nothing to pass.
|
||||
kwargs = {
|
||||
name: value
|
||||
for name, value in (
|
||||
("image_grid_thw", image_grid_thw),
|
||||
("video_grid_thw", video_grid_thw),
|
||||
)
|
||||
if value is not None or accepts_kwarg(name)
|
||||
}
|
||||
if accepts_kwarg("mm_token_type_ids"):
|
||||
mm_token_type_ids = torch.zeros(len(input_tokens), dtype=torch.int)
|
||||
for feature in mm_features:
|
||||
position = feature.mm_position
|
||||
@@ -1257,8 +1271,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
mrope_positions, mrope_position_delta = self.model.get_rope_index(
|
||||
input_ids=torch.tensor(input_tokens).unsqueeze(0),
|
||||
image_grid_thw=image_grid_thw,
|
||||
video_grid_thw=video_grid_thw,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user