[Core][Model] Unify XD-RoPE into M-RoPE and derive the channel count (#56078)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
c55e15a44e
commit
719284fe15
@@ -1783,7 +1783,8 @@ class SupportsMRoPE(Protocol):
|
||||
|
||||
Returns:
|
||||
Tuple of `(llm_positions, mrope_position_delta)`
|
||||
- llm_positions: Tensor of shape `[3, num_tokens]` with T/H/W positions
|
||||
- llm_positions: Tensor of shape `[num_dims, num_tokens]`, one row
|
||||
per M-RoPE position channel (e.g. T/H/W)
|
||||
- mrope_position_delta: Delta for position calculations
|
||||
"""
|
||||
...
|
||||
@@ -1803,55 +1804,6 @@ def supports_mrope(
|
||||
return isinstance(model, SupportsMRoPE)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsXDRoPE(Protocol):
|
||||
"""The interface required for all models that support XD-RoPE."""
|
||||
|
||||
supports_xdrope: ClassVar[Literal[True]] = True
|
||||
"""
|
||||
A flag that indicates this model supports XD-RoPE.
|
||||
|
||||
Note:
|
||||
There is no need to redefine this flag if this class is in the
|
||||
XDRope of your model class.
|
||||
"""
|
||||
|
||||
def get_xdrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list["MultiModalFeatureSpec"],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Get XD-RoPE input positions and delta value for this specific model.
|
||||
|
||||
This method should be implemented by each model that supports XD-RoPE
|
||||
to provide model-specific logic for computing input positions.
|
||||
|
||||
Args:
|
||||
input_tokens: List of input token IDs
|
||||
mm_features: Information about each multi-modal data item
|
||||
|
||||
Returns:
|
||||
llm_positions: Tensor of shape `[xdrope_dim, num_tokens]` with
|
||||
4D(P/W/H/T) or 3D(W/H/T) positions.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_xdrope(model: type[object]) -> TypeIs[type[SupportsXDRoPE]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_xdrope(model: object) -> TypeIs[SupportsXDRoPE]: ...
|
||||
|
||||
|
||||
def supports_xdrope(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsXDRoPE]] | TypeIs[SupportsXDRoPE]:
|
||||
return isinstance(model, SupportsXDRoPE)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsEncoderCudaGraph(Protocol):
|
||||
"""Interface for models whose vision encoder supports CUDA graph
|
||||
|
||||
@@ -1236,17 +1236,31 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
video_grid_thw = torch.stack(video_grid_thw) if video_grid_thw else None
|
||||
|
||||
# `get_rope_index` doesn't always accept arbitrary `kwargs`
|
||||
kwargs = {}
|
||||
if not hasattr(self, "_get_rope_index_accepts_mm_token_type_ids"):
|
||||
if not hasattr(self, "_get_rope_index_kwarg_names"):
|
||||
import inspect
|
||||
|
||||
sig = inspect.signature(self.model.get_rope_index)
|
||||
params = sig.parameters
|
||||
self._get_rope_index_accepts_mm_token_type_ids = (
|
||||
"mm_token_type_ids" in params
|
||||
or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
params = inspect.signature(self.model.get_rope_index).parameters
|
||||
self._get_rope_index_kwarg_names = (
|
||||
None
|
||||
if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
else frozenset(params)
|
||||
)
|
||||
if self._get_rope_index_accepts_mm_token_type_ids:
|
||||
|
||||
kwarg_names = self._get_rope_index_kwarg_names
|
||||
|
||||
def accepts_kwarg(name: str) -> bool:
|
||||
return kwarg_names is None or name in kwarg_names
|
||||
|
||||
# Drop a grid the model can't accept only when there is nothing to pass.
|
||||
kwargs = {
|
||||
name: value
|
||||
for name, value in (
|
||||
("image_grid_thw", image_grid_thw),
|
||||
("video_grid_thw", video_grid_thw),
|
||||
)
|
||||
if value is not None or accepts_kwarg(name)
|
||||
}
|
||||
if accepts_kwarg("mm_token_type_ids"):
|
||||
mm_token_type_ids = torch.zeros(len(input_tokens), dtype=torch.int)
|
||||
for feature in mm_features:
|
||||
position = feature.mm_position
|
||||
@@ -1257,8 +1271,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
mrope_positions, mrope_position_delta = self.model.get_rope_index(
|
||||
input_ids=torch.tensor(input_tokens).unsqueeze(0),
|
||||
image_grid_thw=image_grid_thw,
|
||||
video_grid_thw=video_grid_thw,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user