[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -6,24 +6,14 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from .bailing_mrope import BailingMRotaryEmbedding
|
||||
from .base import RotaryEmbedding
|
||||
from .deepseek_scaling_rope import (
|
||||
DeepseekScalingRotaryEmbedding,
|
||||
DeepseekV4ScalingRotaryEmbedding,
|
||||
)
|
||||
from .dual_chunk_rope import DualChunkRotaryEmbedding
|
||||
from .dynamic_ntk_alpha_rope import DynamicNTKAlphaRotaryEmbedding
|
||||
from .dynamic_ntk_scaling_rope import DynamicNTKScalingRotaryEmbedding
|
||||
from .fope import FourierRotaryEmbedding
|
||||
from .gemma4_rope import Gemma4RotaryEmbedding
|
||||
from .linear_scaling_rope import LinearScalingRotaryEmbedding
|
||||
from .llama3_rope import Llama3RotaryEmbedding
|
||||
from .llama4_vision_rope import Llama4VisionRotaryEmbedding
|
||||
from .mrope import MRotaryEmbedding
|
||||
from .mrope_interleaved import MRotaryEmbeddingInterleaved
|
||||
from .ntk_scaling_rope import NTKScalingRotaryEmbedding
|
||||
from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
|
||||
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
|
||||
|
||||
_ROPE_DICT: dict[tuple[Any, ...], RotaryEmbedding] = {}
|
||||
@@ -99,16 +89,6 @@ def get_rope(
|
||||
dtype,
|
||||
**extra_kwargs,
|
||||
)
|
||||
elif scaling_type == "bailing_mrope":
|
||||
rotary_emb = BailingMRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
mrope_section=rope_parameters["mrope_section"],
|
||||
)
|
||||
elif scaling_type == "default":
|
||||
if "mrope_section" in rope_parameters:
|
||||
rotary_emb = MRotaryEmbedding(
|
||||
@@ -122,28 +102,6 @@ def get_rope(
|
||||
mrope_interleaved=rope_parameters.get("mrope_interleaved", False),
|
||||
positions_factor=mrope_positions_factor,
|
||||
)
|
||||
elif "use_fope" in rope_parameters and rope_parameters["use_fope"]:
|
||||
extra_kwargs = {
|
||||
k: v
|
||||
for k, v in rope_parameters.items()
|
||||
if k
|
||||
in (
|
||||
"num_key_value_heads",
|
||||
"num_inv_freq",
|
||||
"fope_sep_head",
|
||||
"fope_init_factor",
|
||||
)
|
||||
}
|
||||
extra_kwargs["init_cache"] = False
|
||||
rotary_emb = FourierRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
rotary_emb = RotaryEmbedding(
|
||||
head_size,
|
||||
@@ -153,93 +111,6 @@ def get_rope(
|
||||
is_neox_style,
|
||||
dtype,
|
||||
)
|
||||
elif scaling_type == "proportional":
|
||||
# Proportional RoPE is used by Gemma4 for global (full) attention.
|
||||
# Gemma4 uses a sparse/fractional RoPE with cross-mixing between halves.
|
||||
rotary_emb = Gemma4RotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
)
|
||||
elif scaling_type == "llama3":
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
low_freq_factor = rope_parameters["low_freq_factor"]
|
||||
high_freq_factor = rope_parameters["high_freq_factor"]
|
||||
original_max_position = rope_parameters["original_max_position_embeddings"]
|
||||
rotary_emb = Llama3RotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
scaling_factor,
|
||||
low_freq_factor,
|
||||
high_freq_factor,
|
||||
original_max_position,
|
||||
)
|
||||
elif scaling_type == "mllama4":
|
||||
rotary_emb = Llama4VisionRotaryEmbedding(
|
||||
head_size, rotary_dim, max_position, base, is_neox_style, dtype
|
||||
)
|
||||
elif scaling_type == "linear":
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
rotary_emb = LinearScalingRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_factor,
|
||||
dtype,
|
||||
)
|
||||
elif scaling_type == "ntk":
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
mixed_b = rope_parameters.get("mixed_b")
|
||||
rotary_emb = NTKScalingRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_factor,
|
||||
dtype,
|
||||
mixed_b,
|
||||
)
|
||||
elif scaling_type == "dynamic":
|
||||
if "alpha" in rope_parameters:
|
||||
scaling_alpha = rope_parameters["alpha"]
|
||||
rotary_emb = DynamicNTKAlphaRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_alpha,
|
||||
dtype,
|
||||
)
|
||||
elif "factor" in rope_parameters:
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
max_trained_positions = rope_parameters.get(
|
||||
"max_trained_positions", max_position
|
||||
)
|
||||
rotary_emb = DynamicNTKScalingRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
max_trained_positions,
|
||||
base,
|
||||
is_neox_style,
|
||||
scaling_factor,
|
||||
dtype,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Dynamic rope scaling must contain either 'alpha' or 'factor' field"
|
||||
)
|
||||
elif scaling_type == "yarn":
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
original_max_position = rope_parameters["original_max_position_embeddings"]
|
||||
@@ -310,27 +181,6 @@ def get_rope(
|
||||
dtype,
|
||||
**extra_kwargs,
|
||||
)
|
||||
elif scaling_type == "longrope":
|
||||
short_factor = rope_parameters["short_factor"]
|
||||
long_factor = rope_parameters["long_factor"]
|
||||
original_max_position = rope_parameters["original_max_position_embeddings"]
|
||||
extra_kwargs = {
|
||||
k: v
|
||||
for k, v in rope_parameters.items()
|
||||
if k in ("short_mscale", "long_mscale")
|
||||
}
|
||||
rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
original_max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
short_factor,
|
||||
long_factor,
|
||||
**extra_kwargs,
|
||||
)
|
||||
elif scaling_type == "openpangu":
|
||||
mrope_interleaved = rope_parameters.get("mrope_interleaved", False)
|
||||
if "mrope_section" in rope_parameters and mrope_interleaved:
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .mrope import MRotaryEmbedding
|
||||
|
||||
|
||||
def _validate_bailing_mrope_config(
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
mrope_section: list[int],
|
||||
) -> int:
|
||||
if rotary_dim <= 0 or rotary_dim % 2 != 0 or rotary_dim > head_size:
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE requires an even rotary_dim in (0, head_size], "
|
||||
f"got rotary_dim={rotary_dim} and head_size={head_size}"
|
||||
)
|
||||
if len(mrope_section) != 3:
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE expects [T, H, W] sections, "
|
||||
f"got mrope_section={mrope_section}"
|
||||
)
|
||||
|
||||
temporal_size, height_size, width_size = mrope_section
|
||||
if any(section_size < 0 for section_size in mrope_section):
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE sections must be non-negative, "
|
||||
f"got mrope_section={mrope_section}"
|
||||
)
|
||||
if height_size != width_size:
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE requires equal H and W sections, "
|
||||
f"got height_size={height_size} and width_size={width_size}"
|
||||
)
|
||||
spatial_size = height_size + width_size
|
||||
if spatial_size + temporal_size != rotary_dim // 2:
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE sections must sum to rotary_dim // 2, got "
|
||||
f"mrope_section={mrope_section} and rotary_dim={rotary_dim}"
|
||||
)
|
||||
return spatial_size
|
||||
|
||||
|
||||
def _validate_bailing_positions(
|
||||
positions: torch.Tensor,
|
||||
*,
|
||||
require_2d: bool = False,
|
||||
) -> None:
|
||||
if positions.dtype not in (torch.int32, torch.int64):
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE positions must have int32 or int64 dtype, got "
|
||||
f"dtype={positions.dtype}"
|
||||
)
|
||||
if positions.ndim == 1 and not require_2d:
|
||||
return
|
||||
if positions.ndim == 2:
|
||||
if positions.shape[0] == 3:
|
||||
return
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE expects T/H/W positions with shape [3, N], "
|
||||
f"got shape={tuple(positions.shape)}"
|
||||
)
|
||||
if require_2d:
|
||||
raise ValueError(
|
||||
"Bailing M-RoPE expects T/H/W positions with shape [3, N], "
|
||||
f"got shape={tuple(positions.shape)}"
|
||||
)
|
||||
raise ValueError(
|
||||
f"Bailing M-RoPE positions must be 1D or 2D, got shape={tuple(positions.shape)}"
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _triton_bailing_mrope_forward(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
positions_ptr,
|
||||
cos_sin_cache_ptr,
|
||||
q_stride_token,
|
||||
q_stride_head,
|
||||
q_stride_dim,
|
||||
k_stride_token,
|
||||
k_stride_head,
|
||||
k_stride_dim,
|
||||
positions_stride_axis,
|
||||
positions_stride_token,
|
||||
cache_stride_position,
|
||||
cache_stride_dim,
|
||||
n_qh: tl.constexpr,
|
||||
n_kh: tl.constexpr,
|
||||
rotary_dim: tl.constexpr,
|
||||
pad_n_qh: tl.constexpr,
|
||||
pad_n_kh: tl.constexpr,
|
||||
pad_half_rotary_dim: tl.constexpr,
|
||||
spatial_size: tl.constexpr,
|
||||
):
|
||||
"""Apply GPT-J-style H/W-interleaved, T-tail M-RoPE in place."""
|
||||
token_idx = tl.program_id(0)
|
||||
position_offset = token_idx * positions_stride_token
|
||||
temporal_position = tl.load(positions_ptr + position_offset).to(tl.int64)
|
||||
height_position = tl.load(
|
||||
positions_ptr + positions_stride_axis + position_offset
|
||||
).to(tl.int64)
|
||||
width_position = tl.load(
|
||||
positions_ptr + 2 * positions_stride_axis + position_offset
|
||||
).to(tl.int64)
|
||||
|
||||
frequency_offsets = tl.arange(0, pad_half_rotary_dim)
|
||||
frequency_mask = frequency_offsets < rotary_dim // 2
|
||||
spatial_position = tl.where(
|
||||
frequency_offsets % 2 == 0,
|
||||
height_position,
|
||||
width_position,
|
||||
)
|
||||
selected_position = tl.where(
|
||||
frequency_offsets < spatial_size,
|
||||
spatial_position,
|
||||
temporal_position,
|
||||
)
|
||||
cache_offsets = (
|
||||
selected_position * cache_stride_position + frequency_offsets * cache_stride_dim
|
||||
)
|
||||
cos_row = tl.load(
|
||||
cos_sin_cache_ptr + cache_offsets,
|
||||
mask=frequency_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
sin_row = tl.load(
|
||||
cos_sin_cache_ptr + cache_offsets + (rotary_dim // 2) * cache_stride_dim,
|
||||
mask=frequency_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
q_head_offsets = tl.arange(0, pad_n_qh)[:, None]
|
||||
k_head_offsets = tl.arange(0, pad_n_kh)[:, None]
|
||||
pair_offsets = frequency_offsets[None, :]
|
||||
q_mask = (q_head_offsets < n_qh) & frequency_mask[None, :]
|
||||
k_mask = (k_head_offsets < n_kh) & frequency_mask[None, :]
|
||||
|
||||
q_base = q_ptr + token_idx * q_stride_token
|
||||
q_even_offsets = q_head_offsets * q_stride_head + 2 * pair_offsets * q_stride_dim
|
||||
q_odd_offsets = q_even_offsets + q_stride_dim
|
||||
q_even = tl.load(q_base + q_even_offsets, mask=q_mask, other=0.0).to(tl.float32)
|
||||
q_odd = tl.load(q_base + q_odd_offsets, mask=q_mask, other=0.0).to(tl.float32)
|
||||
q_even_cos = (q_even * cos_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
|
||||
q_odd_sin = (q_odd * sin_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
|
||||
q_odd_cos = (q_odd * cos_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
|
||||
q_even_sin = (q_even * sin_row[None, :]).to(q_ptr.type.element_ty).to(tl.float32)
|
||||
tl.store(
|
||||
q_base + q_even_offsets,
|
||||
q_even_cos - q_odd_sin,
|
||||
mask=q_mask,
|
||||
)
|
||||
tl.store(
|
||||
q_base + q_odd_offsets,
|
||||
q_odd_cos + q_even_sin,
|
||||
mask=q_mask,
|
||||
)
|
||||
|
||||
k_base = k_ptr + token_idx * k_stride_token
|
||||
k_even_offsets = k_head_offsets * k_stride_head + 2 * pair_offsets * k_stride_dim
|
||||
k_odd_offsets = k_even_offsets + k_stride_dim
|
||||
k_even = tl.load(k_base + k_even_offsets, mask=k_mask, other=0.0).to(tl.float32)
|
||||
k_odd = tl.load(k_base + k_odd_offsets, mask=k_mask, other=0.0).to(tl.float32)
|
||||
k_even_cos = (k_even * cos_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
|
||||
k_odd_sin = (k_odd * sin_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
|
||||
k_odd_cos = (k_odd * cos_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
|
||||
k_even_sin = (k_even * sin_row[None, :]).to(k_ptr.type.element_ty).to(tl.float32)
|
||||
tl.store(
|
||||
k_base + k_even_offsets,
|
||||
k_even_cos - k_odd_sin,
|
||||
mask=k_mask,
|
||||
)
|
||||
tl.store(
|
||||
k_base + k_odd_offsets,
|
||||
k_odd_cos + k_even_sin,
|
||||
mask=k_mask,
|
||||
)
|
||||
|
||||
|
||||
def triton_bailing_mrope(
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
spatial_size: int,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused Bailing M-RoPE for packed or strided Q/K."""
|
||||
_validate_bailing_positions(positions, require_2d=True)
|
||||
num_tokens = positions.shape[-1]
|
||||
if num_tokens == 0:
|
||||
return query, key
|
||||
|
||||
query_3d = query.view(num_tokens, -1, head_size)
|
||||
key_3d = key.view(num_tokens, -1, head_size)
|
||||
n_q_heads = query_3d.shape[1]
|
||||
n_k_heads = key_3d.shape[1]
|
||||
pad_n_q_heads = triton.next_power_of_2(n_q_heads)
|
||||
pad_n_k_heads = triton.next_power_of_2(n_k_heads)
|
||||
pad_half_rotary_dim = triton.next_power_of_2(rotary_dim // 2)
|
||||
_triton_bailing_mrope_forward[(num_tokens,)](
|
||||
query_3d,
|
||||
key_3d,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
query_3d.stride(0),
|
||||
query_3d.stride(1),
|
||||
query_3d.stride(2),
|
||||
key_3d.stride(0),
|
||||
key_3d.stride(1),
|
||||
key_3d.stride(2),
|
||||
positions.stride(0),
|
||||
positions.stride(1),
|
||||
cos_sin_cache.stride(0),
|
||||
cos_sin_cache.stride(1),
|
||||
n_q_heads,
|
||||
n_k_heads,
|
||||
rotary_dim,
|
||||
pad_n_q_heads,
|
||||
pad_n_k_heads,
|
||||
pad_half_rotary_dim,
|
||||
spatial_size,
|
||||
num_warps=4,
|
||||
)
|
||||
return query, key
|
||||
|
||||
|
||||
class BailingMRotaryEmbedding(MRotaryEmbedding):
|
||||
"""Bailing M-RoPE with alternating H/W frequencies and a temporal tail."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
dtype: torch.dtype,
|
||||
mrope_section: list[int],
|
||||
) -> None:
|
||||
self.spatial_size = _validate_bailing_mrope_config(
|
||||
head_size, rotary_dim, mrope_section
|
||||
)
|
||||
super().__init__(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position_embeddings,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
mrope_section=mrope_section,
|
||||
)
|
||||
|
||||
def select_cos_sin(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
_validate_bailing_positions(positions)
|
||||
if cos_sin_cache is None:
|
||||
cos_sin_cache = self.cos_sin_cache
|
||||
cos_sin = cos_sin_cache[positions.to(torch.long)]
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
if positions.ndim == 1:
|
||||
return cos, sin
|
||||
|
||||
spatial_mask = torch.arange(self.spatial_size, device=cos.device) % 2 == 0
|
||||
spatial_mask = spatial_mask.unsqueeze(0)
|
||||
spatial_cos = torch.where(
|
||||
spatial_mask,
|
||||
cos[1, :, : self.spatial_size],
|
||||
cos[2, :, : self.spatial_size],
|
||||
)
|
||||
spatial_sin = torch.where(
|
||||
spatial_mask,
|
||||
sin[1, :, : self.spatial_size],
|
||||
sin[2, :, : self.spatial_size],
|
||||
)
|
||||
temporal_slice = slice(self.spatial_size, self.rotary_dim // 2)
|
||||
return (
|
||||
torch.cat((spatial_cos, cos[0, :, temporal_slice]), dim=-1),
|
||||
torch.cat((spatial_sin, sin[0, :, temporal_slice]), dim=-1),
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if key is None:
|
||||
raise ValueError("Bailing M-RoPE requires key states")
|
||||
|
||||
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
|
||||
cos, sin = self.select_cos_sin(positions, cos_sin_cache)
|
||||
num_tokens = positions.shape[-1]
|
||||
query_shape = query.shape
|
||||
query = query.view(num_tokens, -1, self.head_size)
|
||||
query_rot = self.apply_rotary_emb.forward_native(
|
||||
query[..., : self.rotary_dim],
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
query = torch.cat((query_rot, query[..., self.rotary_dim :]), dim=-1)
|
||||
|
||||
key_shape = key.shape
|
||||
key = key.view(num_tokens, -1, self.head_size)
|
||||
key_rot = self.apply_rotary_emb.forward_native(
|
||||
key[..., : self.rotary_dim],
|
||||
cos,
|
||||
sin,
|
||||
)
|
||||
key = torch.cat((key_rot, key[..., self.rotary_dim :]), dim=-1)
|
||||
return query.reshape(query_shape), key.reshape(key_shape)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if positions.ndim == 2 and not self.is_neox_style:
|
||||
if key is None:
|
||||
raise ValueError("Bailing M-RoPE requires key states")
|
||||
cos_sin_cache = self._match_cos_sin_cache_dtype(query)
|
||||
return triton_bailing_mrope(
|
||||
query,
|
||||
key,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.spatial_size,
|
||||
self.head_size,
|
||||
self.rotary_dim,
|
||||
)
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return self.forward_native(positions, query, key, offsets)
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
|
||||
|
||||
class DynamicNTKAlphaRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with Dynamic NTK alpha.
|
||||
|
||||
Based on the original RotaryEmbedding implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
scaling_alpha: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
self.scaling_alpha = scaling_alpha
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
|
||||
def _compute_cos_sin_cache(self) -> torch.Tensor:
|
||||
# For Hunyuan DynamicNTKAlphaRotaryEmbedding
|
||||
max_len = self.max_position_embeddings
|
||||
base = self.base * self.scaling_alpha ** (
|
||||
self.rotary_dim / (self.rotary_dim - 2)
|
||||
)
|
||||
inv_freq = self._compute_inv_freq(base)
|
||||
t = torch.arange(max_len, dtype=torch.float)
|
||||
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos()
|
||||
sin = freqs.sin()
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
|
||||
|
||||
class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with Dynamic NTK scaling.
|
||||
|
||||
Credits to the Reddit users /u/bloc97 and /u/emozilla
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
max_trained_positions: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
self.scaling_factor = scaling_factor
|
||||
self.max_trained_positions = max_trained_positions
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
|
||||
def _compute_cos_sin_cache(self) -> torch.Tensor:
|
||||
# NOTE(woosuk): self.max_position_embeddings is the original
|
||||
# maximum length before applying the rope scaling.
|
||||
# Thus, the maximum length after applying the rope scaling is
|
||||
# self.max_position_embeddings * self.scaling_factor.
|
||||
base = self.base * (
|
||||
(
|
||||
self.scaling_factor
|
||||
* self.max_position_embeddings
|
||||
/ self.max_trained_positions
|
||||
)
|
||||
- (self.scaling_factor - 1)
|
||||
) ** (self.rotary_dim / (self.rotary_dim - 2))
|
||||
inv_freq = self._compute_inv_freq(base)
|
||||
t = torch.arange(self.max_position_embeddings, dtype=torch.float)
|
||||
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos()
|
||||
sin = freqs.sin()
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
@@ -1,199 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
from .common import rotate_neox
|
||||
|
||||
|
||||
class FourierRotaryEmbedding(RotaryEmbedding):
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
dtype: torch.dtype,
|
||||
init_cache: bool,
|
||||
# extra parameters for FoPE
|
||||
num_key_value_heads: int,
|
||||
num_inv_freq: int,
|
||||
fope_sep_head: bool,
|
||||
fope_init_factor: float,
|
||||
):
|
||||
# fope related parameters
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.num_inv_freq = num_inv_freq
|
||||
self.fope_sep_head = fope_sep_head
|
||||
self.fope_init_factor = fope_init_factor
|
||||
|
||||
super().__init__(
|
||||
head_size=head_size,
|
||||
rotary_dim=rotary_dim,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
base=base,
|
||||
is_neox_style=is_neox_style,
|
||||
dtype=dtype,
|
||||
init_cache=init_cache,
|
||||
)
|
||||
|
||||
# setup buffers and parameters
|
||||
self.inv_freq: torch.Tensor
|
||||
self.register_buffer(
|
||||
"inv_freq", self._compute_inv_freq(self.base), persistent=False
|
||||
)
|
||||
|
||||
self.input_dim = self.inv_freq.shape[-1]
|
||||
self.output_dim = self.inv_freq.shape[-1]
|
||||
self.cos_coef = nn.Parameter(
|
||||
torch.empty(num_key_value_heads, self.input_dim, self.output_dim),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.sin_coef = nn.Parameter(
|
||||
torch.empty(num_key_value_heads, self.input_dim, self.output_dim),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.sin_coef.weight_loader = self.weight_loader
|
||||
self.cos_coef.weight_loader = self.weight_loader
|
||||
|
||||
self.cos_sin_cache: torch.Tensor
|
||||
cache = self._compute_cos_sin_cache().to(dtype)
|
||||
self.register_buffer("cos_sin_cache", cache, persistent=False)
|
||||
|
||||
# update cache in the first forward, where sin/cos_coef weights are ready
|
||||
self.update_cache = True
|
||||
|
||||
def _compute_inv_freq(self, base: float) -> torch.Tensor:
|
||||
"""Compute the inverse frequency."""
|
||||
inv_freq = 1.0 / (
|
||||
base
|
||||
** (
|
||||
torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim
|
||||
)
|
||||
)
|
||||
|
||||
inv_freq_idx_selected = torch.ones_like(inv_freq, dtype=torch.bool)
|
||||
if self.num_inv_freq is not None:
|
||||
inv_freq_idx_selected[self.num_inv_freq :] = False
|
||||
else:
|
||||
inv_freq_idx_selected = inv_freq > (
|
||||
2.0 * torch.pi / self.max_position_embeddings
|
||||
)
|
||||
|
||||
inv_freq = inv_freq[inv_freq_idx_selected]
|
||||
return inv_freq
|
||||
|
||||
def _compute_cos_sin_cache(self) -> torch.Tensor:
|
||||
"""Compute the cos and sin cache."""
|
||||
device = self.inv_freq.device
|
||||
t = torch.arange(self.max_position_embeddings, dtype=torch.float, device=device)
|
||||
|
||||
freqs = torch.einsum("j,i -> ji", t, self.inv_freq)
|
||||
if self.fope_sep_head:
|
||||
pos_cos = freqs.cos().unsqueeze(0).expand(self.num_key_value_heads, -1, -1)
|
||||
pos_sin = freqs.sin().unsqueeze(0).expand(self.num_key_value_heads, -1, -1)
|
||||
else:
|
||||
pos_cos = freqs.cos()
|
||||
pos_sin = freqs.sin()
|
||||
|
||||
if self.fope_sep_head:
|
||||
sin = torch.einsum("htD, hDd -> thd", pos_sin, self.sin_coef.float())
|
||||
cos = torch.einsum("htD, hDd -> thd", pos_cos, self.cos_coef.float())
|
||||
else:
|
||||
sin = torch.einsum("tD, Dd -> td", pos_sin, self.sin_coef.float())
|
||||
cos = torch.einsum("tD, Dd -> td", pos_cos, self.cos_coef.float())
|
||||
|
||||
sin = F.pad(
|
||||
input=sin,
|
||||
pad=(0, self.head_size // 2 - sin.size(-1)),
|
||||
mode="constant",
|
||||
value=1,
|
||||
)
|
||||
cos = F.pad(
|
||||
input=cos,
|
||||
pad=(0, self.head_size // 2 - cos.size(-1)),
|
||||
mode="constant",
|
||||
value=1,
|
||||
)
|
||||
|
||||
sin = torch.cat((sin, sin), dim=-1)
|
||||
cos = torch.cat((cos, cos), dim=-1)
|
||||
|
||||
# cache: (max_position_embeddings, num_kv_heads, kv_size * 2)
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
# update cos/sin cache in the first forward
|
||||
if self.update_cache:
|
||||
cache = self._compute_cos_sin_cache().to(self.dtype)
|
||||
self.cos_sin_cache.copy_(cache)
|
||||
self.update_cache = False
|
||||
|
||||
positions = positions.flatten()
|
||||
cos_sin = self.cos_sin_cache.index_select(0, positions)
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
|
||||
# apply rotary embedding
|
||||
# query: (seq_len, num_heads, head_size)
|
||||
# key: (seq_len, num_kv_heads, head_size)
|
||||
query = query.unflatten(-1, (-1, self.head_size))
|
||||
assert key is not None, "Key tensor is required for FoPE."
|
||||
key = key.unflatten(-1, (-1, self.head_size))
|
||||
|
||||
assert query.dim() == key.dim() == 3, (
|
||||
"Expected query key (seq_len, heads, head_dim)"
|
||||
)
|
||||
assert cos.dim() <= 3 and sin.dim() <= 3
|
||||
|
||||
need_reshape = False
|
||||
if cos.dim() == 3:
|
||||
# for fope
|
||||
need_reshape = True
|
||||
query_shape = query.shape
|
||||
key_shape = key.shape
|
||||
cos = cos.flatten(0, 1)
|
||||
sin = sin.flatten(0, 1)
|
||||
seq_len = cos.size(0)
|
||||
query = query.view(seq_len, -1, query.size(-1))
|
||||
key = key.view(seq_len, -1, key.size(-1))
|
||||
|
||||
# native implementation of apply rope for neox style
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
query = (query * cos) + (rotate_neox(query) * sin)
|
||||
key = (key * cos) + (rotate_neox(key) * sin)
|
||||
|
||||
if need_reshape:
|
||||
query = query.view(query_shape)
|
||||
key = key.view(key_shape)
|
||||
|
||||
return query, key
|
||||
|
||||
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
|
||||
"""Load fope weights."""
|
||||
world_size = get_tensor_model_parallel_world_size()
|
||||
rank = get_tensor_model_parallel_rank()
|
||||
num_key_value_heads = loaded_weight.size(0)
|
||||
|
||||
if num_key_value_heads < world_size:
|
||||
n_replicate = world_size // num_key_value_heads
|
||||
world_size = num_key_value_heads
|
||||
rank = rank // n_replicate
|
||||
|
||||
loaded_weight = loaded_weight.chunk(world_size, dim=0)[rank]
|
||||
param.data.copy_(loaded_weight)
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Gemma4-specific Rotary Positional Embeddings (proportional scaling).
|
||||
|
||||
Gemma4 uses "proportional" RoPE which computes inv_freq frequencies scaled
|
||||
by head_dim (not rotary_dim), and zero-pads for non-rotated dimensions when
|
||||
partial_rotary_factor < 1. The actual rotation uses standard neox-style
|
||||
rotate_half, matching HF transformers' apply_rotary_pos_emb.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
|
||||
|
||||
class Gemma4RotaryEmbedding(RotaryEmbedding):
|
||||
"""Gemma4 proportional RoPE.
|
||||
|
||||
Extends RotaryEmbedding (which provides standard neox-style rotation
|
||||
via ops.rotary_embedding CUDA kernel) but overrides the inv_freq
|
||||
computation to match HF's _compute_proportional_rope_parameters:
|
||||
- Frequency exponents use head_dim (not rotary_dim) as denominator
|
||||
- Non-rotated dims are zero-padded (cos=1, sin=0 = identity rotation)
|
||||
|
||||
When partial_rotary_factor=1.0 (the default for some variants), ALL dims are
|
||||
rotated and this is equivalent to standard RotaryEmbedding with
|
||||
head_dim-scaled frequencies.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
# Number of rotation angle pairs (from partial_rotary_factor)
|
||||
self.rope_angles = rotary_dim // 2
|
||||
# Non-rotated angle pairs per half
|
||||
self.nope_angles = (head_size // 2) - self.rope_angles
|
||||
|
||||
# Important: set rotary_dim = head_size so the base class's
|
||||
# forward_static applies rotation to ALL dims of the cos/sin cache.
|
||||
# The non-rotated dims will have cos=1, sin=0 (identity) thanks
|
||||
# to our _compute_inv_freq zero-padding.
|
||||
super().__init__(
|
||||
head_size,
|
||||
head_size, # rotary_dim = head_size (full application)
|
||||
max_position_embeddings,
|
||||
base,
|
||||
is_neox_style,
|
||||
dtype,
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, base: float) -> torch.Tensor:
|
||||
"""Compute frequencies matching HF proportional RoPE.
|
||||
|
||||
Key difference from base: exponent denominator is head_size (not
|
||||
rotary_dim), and non-rotated dims are zero-padded.
|
||||
"""
|
||||
# HF formula: base ** (arange(0, 2*rope_angles, 2) / head_dim)
|
||||
freq_exponents = (
|
||||
torch.arange(0, 2 * self.rope_angles, 2, dtype=torch.float) / self.head_size
|
||||
)
|
||||
inv_freq = 1.0 / (base**freq_exponents)
|
||||
|
||||
# Zero-pad for non-rotated dims (identity rotation: cos=1, sin=0)
|
||||
if self.nope_angles > 0:
|
||||
inv_freq = torch.cat(
|
||||
[
|
||||
inv_freq,
|
||||
torch.zeros(self.nope_angles, dtype=torch.float),
|
||||
]
|
||||
)
|
||||
return inv_freq
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}"
|
||||
s += f", rope_angles={self.rope_angles}, nope_angles={self.nope_angles}"
|
||||
s += f", max_position_embeddings={self.max_position_embeddings}"
|
||||
s += f", base={self.base}, is_neox_style={self.is_neox_style}"
|
||||
return s
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
|
||||
|
||||
class LinearScalingRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with linear scaling.
|
||||
|
||||
It supports multiple scaling factors. Since multiple LoRA adapters may have
|
||||
different scaling factors, we need multiple cos/sin caches. In this way,
|
||||
instead of running rotary embedding kernel per lora, we can run multiple
|
||||
lora in a batched way.
|
||||
|
||||
In addition to that, we also keep the cos/sin cache for the scaling factor
|
||||
of 1 (default) at all times.
|
||||
|
||||
Exemplary for two scaling factors x=1, y and z with embeddings
|
||||
[[x11, x12, ... x1m], ..., [xn1, xn2, ..., xnm]] and
|
||||
[[y11, y12, ... y1o], ..., [yn1, yn2, ..., yno]], and
|
||||
[[z11, z12, ... z1p], ..., [zn1, zn2, ..., znp]],
|
||||
|
||||
we construct the cos/sin cache as follows:
|
||||
[[x11, x12, ... x1m, y11, y12, ... y1o, z11, z12, ... z1p],
|
||||
...
|
||||
[xn1, xn2, ... xnm, yn1, yn2, ... yno, zn1, zn2, ... znp]]
|
||||
|
||||
We then use offsets to index into the cos/sin cache for
|
||||
the respective scaling factors.
|
||||
|
||||
The offset to cache can be accessed via `scaling_factor_to_offset` API.
|
||||
|
||||
Credits to the Reddit user /u/kaiokendev
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
scaling_factors: list[float] | float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
if isinstance(scaling_factors, float):
|
||||
scaling_factors = [scaling_factors]
|
||||
self.scaling_factors: list[float] = scaling_factors # noqa
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
# Lazy initialized.
|
||||
self._scaling_factor_to_offset: dict[float, int]
|
||||
|
||||
def _compute_cos_sin_cache(self) -> torch.Tensor:
|
||||
inv_freq = self._compute_inv_freq(self.base)
|
||||
cache_list: list[torch.Tensor] = []
|
||||
# offsets to the next cache in a tensor.
|
||||
# Each offset corresponds to the same index in scaling_factors.
|
||||
offsets: list[int] = []
|
||||
for scaling_factor in self.scaling_factors:
|
||||
# NOTE(woosuk): self.max_position_embeddings is the original
|
||||
# maximum length before applying the rope scaling.
|
||||
# Thus, the maximum length after applying the rope scaling is
|
||||
# self.max_position_embeddings * self.scaling_factor.
|
||||
max_len = self.max_position_embeddings * scaling_factor
|
||||
t = torch.arange(max_len, dtype=torch.float)
|
||||
t = t / scaling_factor
|
||||
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos()
|
||||
sin = freqs.sin()
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
if not cache_list:
|
||||
offset = 0
|
||||
else:
|
||||
last_offset = offsets[-1]
|
||||
next_max_len = cache_list[-1].shape[0]
|
||||
offset = last_offset + next_max_len
|
||||
offsets.append(offset)
|
||||
cache_list.append(cache)
|
||||
self._scaling_factor_to_offset = {
|
||||
float(scaling_factor): offsets[i]
|
||||
for i, scaling_factor in enumerate(self.scaling_factors)
|
||||
}
|
||||
assert len(self.scaling_factors) == len(offsets)
|
||||
return torch.cat(cache_list, dim=0)
|
||||
|
||||
@property
|
||||
def scaling_factor_to_offset(self) -> dict[float, int]:
|
||||
return self._scaling_factor_to_offset
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
|
||||
|
||||
class Llama3RotaryEmbedding(RotaryEmbedding):
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
dtype: torch.dtype,
|
||||
scaling_factor: float,
|
||||
low_freq_factor: float,
|
||||
high_freq_factor: float,
|
||||
orig_max_position: int,
|
||||
) -> None:
|
||||
self.scaling_factor = scaling_factor
|
||||
self.low_freq_factor = low_freq_factor
|
||||
self.high_freq_factor = high_freq_factor
|
||||
self.orig_max_position = orig_max_position
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, base: float) -> torch.Tensor:
|
||||
inv_freqs = super()._compute_inv_freq(base)
|
||||
low_freq_wavelen = self.orig_max_position / self.low_freq_factor
|
||||
high_freq_wavelen = self.orig_max_position / self.high_freq_factor
|
||||
|
||||
wave_len = 2 * math.pi / inv_freqs
|
||||
if self.low_freq_factor != self.high_freq_factor:
|
||||
smooth = (self.orig_max_position / wave_len - self.low_freq_factor) / (
|
||||
self.high_freq_factor - self.low_freq_factor
|
||||
)
|
||||
else:
|
||||
smooth = 0
|
||||
new_freqs = torch.where(
|
||||
wave_len < high_freq_wavelen,
|
||||
inv_freqs,
|
||||
torch.where(
|
||||
wave_len > low_freq_wavelen,
|
||||
inv_freqs / self.scaling_factor,
|
||||
(1 - smooth) * inv_freqs / self.scaling_factor + smooth * inv_freqs,
|
||||
),
|
||||
)
|
||||
return new_freqs
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbeddingBase
|
||||
|
||||
|
||||
class Llama4VisionRotaryEmbedding(RotaryEmbeddingBase):
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, base: float) -> torch.Tensor:
|
||||
inv_freqs = super()._compute_inv_freq(base)
|
||||
inv_freqs = inv_freqs[: (self.rotary_dim // 2)]
|
||||
return inv_freqs
|
||||
|
||||
def _compute_cos_sin_cache(self) -> torch.Tensor:
|
||||
inv_freq = self._compute_inv_freq(self.base)
|
||||
|
||||
# self.max_position_embeddings here is number of image patches
|
||||
# i.e. (image_size // patch_size) ** 2
|
||||
num_patches = self.max_position_embeddings
|
||||
img_idx = torch.arange(num_patches, dtype=torch.int32).reshape(num_patches, 1)
|
||||
img_idx = torch.cat([img_idx, img_idx[:1]], dim=0)
|
||||
img_idx[-1, -1] = -2 # set to ID_CLS_TOKEN
|
||||
num_patches_single_dim = int(math.sqrt(num_patches))
|
||||
frequencies_x = img_idx % num_patches_single_dim
|
||||
frequencies_y = img_idx // num_patches_single_dim
|
||||
freqs_x = (
|
||||
(frequencies_x + 1)[..., None] * inv_freq[None, None, :]
|
||||
).repeat_interleave(2, dim=-1)
|
||||
freqs_y = (
|
||||
(frequencies_y + 1)[..., None] * inv_freq[None, None, :]
|
||||
).repeat_interleave(2, dim=-1)
|
||||
freqs = torch.cat([freqs_x, freqs_y], dim=-1).float().contiguous()[..., ::2]
|
||||
freqs = freqs.masked_fill(img_idx.reshape(-1, 1, 1) < 0, 0)
|
||||
cache = torch.view_as_complex(
|
||||
torch.stack([torch.cos(freqs), torch.sin(freqs)], dim=-1)
|
||||
)
|
||||
return cache
|
||||
|
||||
def forward_native( # type: ignore[override]
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert key is not None
|
||||
# self.cos_sin_cache here is complex tensor so we cannot cast into
|
||||
# query's dtype directly with self._match_cos_sin_cache_dtype
|
||||
|
||||
# NOTE: by not storing cos_sin_cache in self, we can avoid
|
||||
# memory buffer update which is costly to runtime
|
||||
cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(query.device)
|
||||
query_ = torch.view_as_complex(query.float().reshape(*query.shape[:-1], -1, 2))
|
||||
key_ = torch.view_as_complex(key.float().reshape(*key.shape[:-1], -1, 2))
|
||||
broadcast_shape = [
|
||||
d if i == 1 or i == (query_.ndim - 1) else 1
|
||||
for i, d in enumerate(query_.shape)
|
||||
]
|
||||
freqs_ci = cos_sin_cache.view(*broadcast_shape)
|
||||
query_out = torch.view_as_real(query_ * freqs_ci).flatten(3)
|
||||
key_out = torch.view_as_real(key_ * freqs_ci).flatten(3)
|
||||
return query_out.type_as(query), key_out.type_as(key)
|
||||
|
||||
def forward_cuda( # type: ignore[override]
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
return self.forward_native(query, key)
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from .base import RotaryEmbedding
|
||||
|
||||
|
||||
class NTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with fixed and mixed NTK scaling.
|
||||
https://kexue.fm/archives/9706"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
mixed_b: float | None = None,
|
||||
) -> None:
|
||||
self.scaling_factor = scaling_factor
|
||||
self.mixed_b = mixed_b
|
||||
super().__init__(
|
||||
head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, base: float) -> torch.Tensor:
|
||||
base = self.base * (self.scaling_factor if self.mixed_b is None else 1)
|
||||
inv_freq = super()._compute_inv_freq(base)
|
||||
|
||||
if self.mixed_b is None:
|
||||
inv_freq = inv_freq / self.scaling_factor ** (2 / self.rotary_dim)
|
||||
else:
|
||||
a = (
|
||||
torch.tensor(self.scaling_factor).log()
|
||||
/ (self.rotary_dim / 2) ** self.mixed_b
|
||||
)
|
||||
lambda_1_m = (
|
||||
a * torch.arange(1, self.rotary_dim // 2 + 1).float() ** self.mixed_b
|
||||
).exp()
|
||||
inv_freq = inv_freq / lambda_1_m
|
||||
|
||||
return inv_freq
|
||||
@@ -1,159 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .common import rotate_neox
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
|
||||
"""Phi3 family of models scaled rotary embedding.
|
||||
|
||||
Based on the original RotaryEmbedding implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
max_position_embeddings: int,
|
||||
original_max_position_embeddings: int,
|
||||
base: float,
|
||||
is_neox_style: bool,
|
||||
dtype: torch.dtype,
|
||||
short_factor: list[float],
|
||||
long_factor: list[float],
|
||||
short_mscale: float | None = None,
|
||||
long_mscale: float | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
if is_neox_style is False:
|
||||
raise ValueError(
|
||||
"`Phi3LongRoPEScaledRotaryEmbedding` only supports neox_style."
|
||||
)
|
||||
|
||||
self.rotary_dim = rotary_dim
|
||||
self.head_size = head_size
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.original_max_position_embeddings = original_max_position_embeddings
|
||||
self.base = base
|
||||
self.short_factor = short_factor
|
||||
self.long_factor = long_factor
|
||||
|
||||
# Force long factors if max_model_len (runtime max length) exceeds
|
||||
# original_max_position_embeddings to prevent KV cache invalidation when
|
||||
# sequences cross this threshold during generation
|
||||
max_model_len = get_current_vllm_config().model_config.max_model_len
|
||||
self.use_long_rope = max_model_len > original_max_position_embeddings
|
||||
if self.use_long_rope:
|
||||
logger.warning_once(
|
||||
"Using LongRoPE scaling factors. This enables longer "
|
||||
"contexts (%d tokens vs original %d tokens) at the cost of "
|
||||
"some performance degradation for shorter sequences. If "
|
||||
"this is not desired, set `max_model_len` to be at most %d.",
|
||||
max_position_embeddings,
|
||||
original_max_position_embeddings,
|
||||
original_max_position_embeddings,
|
||||
)
|
||||
|
||||
scale = self.max_position_embeddings / self.original_max_position_embeddings
|
||||
if scale <= 1.0:
|
||||
scaling_factor = 1.0
|
||||
else:
|
||||
scaling_factor = math.sqrt(
|
||||
1 + math.log(scale) / math.log(self.original_max_position_embeddings)
|
||||
)
|
||||
if short_mscale is None:
|
||||
short_mscale = scaling_factor
|
||||
if long_mscale is None:
|
||||
long_mscale = scaling_factor
|
||||
|
||||
self.short_mscale = short_mscale
|
||||
self.long_mscale = long_mscale
|
||||
|
||||
short_cache = self._compute_cos_sin_cache(
|
||||
original_max_position_embeddings, short_factor, short_mscale
|
||||
)
|
||||
short_cache = short_cache.to(dtype)
|
||||
|
||||
long_cache = self._compute_cos_sin_cache(
|
||||
max_position_embeddings, long_factor, long_mscale
|
||||
)
|
||||
long_cache = long_cache.to(dtype)
|
||||
|
||||
long_short_cache = torch.cat([short_cache, long_cache], dim=0)
|
||||
self.register_buffer(
|
||||
"long_short_cos_sin_cache", long_short_cache, persistent=False
|
||||
)
|
||||
|
||||
def _compute_inv_freq(self, rescale_factors: list[float]) -> torch.Tensor:
|
||||
rescale_factors = torch.tensor(rescale_factors, dtype=torch.float32)
|
||||
inv_freq = 1.0 / (
|
||||
rescale_factors
|
||||
* (
|
||||
self.base
|
||||
** (
|
||||
torch.arange(0, self.rotary_dim, 2, dtype=torch.float)
|
||||
/ self.rotary_dim
|
||||
)
|
||||
)
|
||||
)
|
||||
return inv_freq
|
||||
|
||||
def _compute_cos_sin_cache(
|
||||
self,
|
||||
max_position_embeddings: int,
|
||||
rescale_factors: list[float],
|
||||
mscale: float,
|
||||
) -> torch.Tensor:
|
||||
inv_freq = self._compute_inv_freq(rescale_factors)
|
||||
t = torch.arange(max_position_embeddings, dtype=torch.float)
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq)
|
||||
cos = freqs.cos() * mscale
|
||||
sin = freqs.sin() * mscale
|
||||
cache = torch.cat((cos, sin), dim=-1)
|
||||
return cache
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert key is not None
|
||||
query = query.view(*query.shape[:-1], -1, self.head_size)
|
||||
key = key.view(*key.shape[:-1], -1, self.head_size)
|
||||
|
||||
if self.use_long_rope:
|
||||
k = self.original_max_position_embeddings
|
||||
long_prompt_offset = torch.full_like(positions, k).long()
|
||||
idx = torch.add(positions, long_prompt_offset)
|
||||
else:
|
||||
idx = positions
|
||||
idx = torch.add(idx, offsets) if offsets is not None else idx
|
||||
cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx)
|
||||
|
||||
cos, sin = cos_sin.chunk(2, dim=-1)
|
||||
cos = cos.repeat(1, 2).unsqueeze(-2)
|
||||
sin = sin.repeat(1, 2).unsqueeze(-2)
|
||||
|
||||
query_rot = query[..., : self.rotary_dim]
|
||||
query_pass = query[..., self.rotary_dim :]
|
||||
query_rot = query_rot * cos + rotate_neox(query_rot) * sin
|
||||
query = torch.cat((query_rot, query_pass), dim=-1)
|
||||
|
||||
key_rot = key[..., : self.rotary_dim]
|
||||
key_pass = key[..., self.rotary_dim :]
|
||||
key_rot = key_rot * cos + rotate_neox(key_rot) * sin
|
||||
key = torch.cat((key_rot, key_pass), dim=-1)
|
||||
|
||||
return query.flatten(-2), key.flatten(-2)
|
||||
Reference in New Issue
Block a user