[Attention] Mamba attention module refactor - LINEAR (#43556)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d0975a4b50
commit
90619351e3
@@ -0,0 +1,384 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fla.ops.layernorm_guard import (
|
||||
RMSNormGated,
|
||||
layernorm_fn,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
|
||||
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
|
||||
MiniMaxText01LinearAttention,
|
||||
MiniMaxText01LinearKernel,
|
||||
clear_linear_attention_cache_for_new_sequences,
|
||||
linear_attention_decode,
|
||||
linear_attention_prefill_and_mix,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
|
||||
|
||||
def _build_rope_parameters(config: PretrainedConfig) -> dict | None:
|
||||
rope_parameters = copy.deepcopy(getattr(config, "rope_parameters", None)) or {}
|
||||
if "rope_theta" not in rope_parameters and hasattr(config, "rope_theta"):
|
||||
rope_parameters["rope_theta"] = config.rope_theta
|
||||
if "partial_rotary_factor" not in rope_parameters and hasattr(
|
||||
config, "partial_rotary_factor"
|
||||
):
|
||||
rope_parameters["partial_rotary_factor"] = config.partial_rotary_factor
|
||||
|
||||
rope_scaling = getattr(config, "rope_scaling", None)
|
||||
if isinstance(rope_scaling, dict):
|
||||
rope_scaling = copy.deepcopy(rope_scaling)
|
||||
if "type" in rope_scaling and "rope_type" not in rope_scaling:
|
||||
rope_scaling["rope_type"] = rope_scaling.pop("type")
|
||||
rope_parameters.update(rope_scaling)
|
||||
|
||||
return rope_parameters or None
|
||||
|
||||
|
||||
class BailingGroupRMSNormGate(RMSNormGated):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size,
|
||||
eps=1e-5,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=None,
|
||||
dtype=None,
|
||||
):
|
||||
super().__init__(
|
||||
hidden_size,
|
||||
eps=eps,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
activation="sigmoid",
|
||||
)
|
||||
# Add custom weight loader for TP sharding
|
||||
self.weight.weight_loader = self._weight_loader
|
||||
|
||||
@staticmethod
|
||||
def _weight_loader(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load weight with TP sharding."""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
shard_size = loaded_weight.shape[0] // tp_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].contiguous())
|
||||
|
||||
|
||||
# --8<-- [start:bailing_moe_linear_attention]
|
||||
@PluggableLayer.register("bailing_moe_linear_attention")
|
||||
class BailingMoELinearAttention(LinearAttention):
|
||||
"""Pluggable Bailing MoE Linear Attention layer which allows OOT backends
|
||||
to add custom implementations.
|
||||
|
||||
This implements the linear attention mechanism from sglang, adapted for
|
||||
vLLM's v1 engine with MambaBase interface support.
|
||||
"""
|
||||
|
||||
# --8<-- [end:bailing_moe_linear_attention]
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "linear_attn",
|
||||
):
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.tp_heads = self.num_heads // self.tp_size
|
||||
|
||||
self.max_position_embeddings = config.max_position_embeddings
|
||||
self.rope_theta = getattr(config, "rope_theta", 600000)
|
||||
|
||||
self.tp_kv_heads = self.num_heads // self.tp_size
|
||||
self.q_size_per_rank = self.head_dim * self.tp_heads
|
||||
self.kv_size_per_rank = self.head_dim * self.tp_kv_heads
|
||||
|
||||
self.use_qk_norm = getattr(config, "use_qk_norm", False)
|
||||
self.linear_backend = "minimax"
|
||||
self.linear_scale = self.linear_backend == "minimax"
|
||||
self.linear_rope = getattr(config, "linear_rope", True)
|
||||
if hasattr(config, "use_linear_silu"):
|
||||
self.linear_silu = config.use_linear_silu
|
||||
elif hasattr(config, "linear_silu"):
|
||||
self.linear_silu = config.linear_silu
|
||||
else:
|
||||
self.linear_silu = False
|
||||
|
||||
self.query_key_value = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
self.num_heads, # MHA: kv_heads = num_heads
|
||||
bias=(config.use_bias or config.use_qkv_bias),
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.query_key_value",
|
||||
)
|
||||
|
||||
if self.use_qk_norm:
|
||||
self.query_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
self.key_layernorm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
self.g_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.g_proj",
|
||||
)
|
||||
self.dense = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
self.hidden_size,
|
||||
bias=config.use_bias,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.dense",
|
||||
reduce_results=True,
|
||||
)
|
||||
|
||||
self.group_norm_size = getattr(config, "group_norm_size", 1)
|
||||
self.rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-5))
|
||||
assert self.tp_size <= self.group_norm_size, (
|
||||
"tp_size must be <= group_norm_size for local rms norm"
|
||||
)
|
||||
assert self.group_norm_size % self.tp_size == 0, (
|
||||
"group_norm_size must be divisible by tp_size"
|
||||
)
|
||||
|
||||
# When group_norm_size == 1, group_size equals hidden_size // tp_size
|
||||
self.g_norm = BailingGroupRMSNormGate(
|
||||
hidden_size=self.hidden_inner_size // self.tp_size,
|
||||
eps=self.rms_norm_eps,
|
||||
group_size=(
|
||||
self.hidden_inner_size // self.group_norm_size
|
||||
if self.group_norm_size > 1
|
||||
else self.hidden_inner_size // self.tp_size
|
||||
),
|
||||
)
|
||||
|
||||
# use fp32 rotary embedding
|
||||
rope_parameters = _build_rope_parameters(config)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=self.max_position_embeddings,
|
||||
is_neox_style=True,
|
||||
rope_parameters=rope_parameters or None,
|
||||
)
|
||||
|
||||
# Build slope tensor for linear attention decay
|
||||
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
|
||||
if self.num_hidden_layers <= 1:
|
||||
self.slope_rate = slope_rate * (1 + 1e-5)
|
||||
else:
|
||||
self.slope_rate = slope_rate * (
|
||||
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
|
||||
)
|
||||
self.tp_slope = self.slope_rate[
|
||||
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
|
||||
].contiguous()
|
||||
|
||||
# Register for compilation
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@staticmethod
|
||||
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
"""Load weight for linear attention layers.
|
||||
|
||||
For FP8 quantized parameters, we need to use the weight_loader if available,
|
||||
as it handles special cases like tensor parallelism sharding.
|
||||
"""
|
||||
# Check if param has a weight_loader (for vLLM ModelWeightParameter)
|
||||
weight_loader = getattr(param, "weight_loader", None)
|
||||
if weight_loader is not None:
|
||||
# Use the weight_loader which handles TP sharding and quantization
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
# Fall back to direct copy for standard tensors
|
||||
assert param.size() == loaded_weight.size(), (
|
||||
f"Shape mismatch: {param.shape} vs {loaded_weight.shape}"
|
||||
)
|
||||
param.data.copy_(loaded_weight)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Forward method called by torch.ops.vllm.linear_attention"""
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
positions,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""Actual forward implementation."""
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata[self.prefix] # type: ignore
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
num_actual_tokens = hidden_states.shape[0]
|
||||
|
||||
# QKV projection
|
||||
qkv, _ = self.query_key_value(hidden_states[:num_actual_tokens])
|
||||
|
||||
# use rotary_emb support fp32
|
||||
qkv = qkv.to(torch.float32)
|
||||
if self.linear_silu:
|
||||
qkv = F.silu(qkv)
|
||||
|
||||
# Split q, k, v
|
||||
q, k, v = torch.split(
|
||||
qkv,
|
||||
[self.q_size_per_rank, self.kv_size_per_rank, self.kv_size_per_rank],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
# Apply QK norm if needed
|
||||
if self.use_qk_norm:
|
||||
q = q.reshape(-1, self.tp_heads, self.head_dim)
|
||||
k = k.reshape(-1, self.tp_kv_heads, self.head_dim)
|
||||
q = layernorm_fn(
|
||||
q,
|
||||
self.query_layernorm.weight.data,
|
||||
bias=None,
|
||||
eps=self.rms_norm_eps,
|
||||
is_rms_norm=True,
|
||||
)
|
||||
k = layernorm_fn(
|
||||
k,
|
||||
self.key_layernorm.weight.data,
|
||||
bias=None,
|
||||
eps=self.rms_norm_eps,
|
||||
is_rms_norm=True,
|
||||
)
|
||||
q = q.reshape(-1, self.q_size_per_rank)
|
||||
k = k.reshape(-1, self.kv_size_per_rank)
|
||||
|
||||
# Apply rotary embeddings
|
||||
if self.linear_rope:
|
||||
q, k = self.rotary_emb(positions[:num_actual_tokens], q, k)
|
||||
|
||||
# Reshape to [batch, heads, seq_len, head_dim]
|
||||
q = q.view((qkv.shape[0], self.tp_heads, self.head_dim))
|
||||
k = k.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
|
||||
v = v.view((qkv.shape[0], self.tp_kv_heads, self.head_dim))
|
||||
|
||||
# Apply scaling if using minimax backend
|
||||
if self.linear_scale:
|
||||
q = q * self.scaling
|
||||
|
||||
# Get KV cache and state indices
|
||||
if attn_metadata is not None:
|
||||
kv_cache = self.kv_cache[0]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
# Compute attention
|
||||
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
|
||||
if attn_metadata is None:
|
||||
hidden = torch.empty(
|
||||
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
|
||||
)
|
||||
else:
|
||||
if not decode_only:
|
||||
hidden = self._prefill_and_mix_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
else:
|
||||
hidden = self._decode_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
# Apply group norm and gate (matching SGLang behavior)
|
||||
gate, _ = self.g_proj(hidden_states[:num_actual_tokens])
|
||||
|
||||
if self.group_norm_size > 1:
|
||||
hidden = self.g_norm(hidden, gate)
|
||||
else:
|
||||
hidden = self.g_norm(hidden)
|
||||
hidden = F.sigmoid(gate) * hidden
|
||||
|
||||
hidden = hidden.to(hidden_states.dtype)
|
||||
|
||||
# Output projection
|
||||
dense_out, _ = self.dense(hidden)
|
||||
output[:num_actual_tokens] = dense_out
|
||||
|
||||
def _prefill_and_mix_infer(
|
||||
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
):
|
||||
"""Handle prefill (mixed with decode if any)."""
|
||||
return linear_attention_prefill_and_mix(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
kv_cache=kv_cache,
|
||||
state_indices_tensor=state_indices_tensor,
|
||||
attn_metadata=attn_metadata,
|
||||
slope_rate=self.tp_slope,
|
||||
block_size=self.BLOCK,
|
||||
decode_fn=self._decode_infer,
|
||||
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
|
||||
layer_idx=self.layer_idx,
|
||||
)
|
||||
|
||||
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
|
||||
"""Handle decode (single token per sequence)."""
|
||||
hidden = linear_attention_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
state_indices_tensor,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
return hidden
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
class LinearAttention(PluggableLayer, MambaBase):
|
||||
"""Base class for Linear attention layer."""
|
||||
|
||||
def __init__(
|
||||
self, config: PretrainedConfig, vllm_config: VllmConfig, prefix: str = ""
|
||||
):
|
||||
super().__init__()
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
self.prefix = prefix
|
||||
self.model_config = vllm_config.model_config
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.BLOCK = getattr(config, "block", 256)
|
||||
self.hidden_size = config.hidden_size
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.head_dim = (
|
||||
config.head_dim
|
||||
if hasattr(config, "head_dim")
|
||||
else config.hidden_size // self.num_heads
|
||||
)
|
||||
self.hidden_inner_size = self.head_dim * self.num_heads
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
assert self.num_heads % self.tp_size == 0
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.LINEAR
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.linear_attention_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, int, int], ...]:
|
||||
return MambaStateShapeCalculator.linear_attention_state_shape(
|
||||
num_heads=self.num_heads, tp_size=self.tp_size, head_dim=self.head_dim
|
||||
)
|
||||
@@ -0,0 +1,352 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.lightning_attn import (
|
||||
lightning_attention,
|
||||
linear_decode_forward_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.mamba.linear.base import LinearAttention
|
||||
from vllm.model_executor.layers.minimax_rms_norm import MiniMaxText01RMSNormTP
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
|
||||
|
||||
def clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
) -> None:
|
||||
num_prefills = getattr(attn_metadata, "num_prefills", 0)
|
||||
if num_prefills <= 0:
|
||||
return
|
||||
|
||||
num_decode_tokens = getattr(attn_metadata, "num_decode_tokens", 0)
|
||||
for prefill_idx in range(num_prefills):
|
||||
q_start = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx]
|
||||
q_end = attn_metadata.query_start_loc[num_decode_tokens + prefill_idx + 1]
|
||||
query_len = q_end - q_start
|
||||
context_len = (
|
||||
attn_metadata.seq_lens[num_decode_tokens + prefill_idx] - query_len
|
||||
)
|
||||
if context_len == 0:
|
||||
block_to_clear = state_indices_tensor[num_decode_tokens + prefill_idx]
|
||||
kv_cache[block_to_clear, ...] = 0
|
||||
|
||||
|
||||
def linear_attention_decode(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
q_start: int = 0,
|
||||
q_end: int | None = None,
|
||||
slot_start: int = 0,
|
||||
slot_end: int | None = None,
|
||||
block_size: int = 32,
|
||||
) -> torch.Tensor:
|
||||
q = q[q_start:q_end].unsqueeze(2).contiguous()
|
||||
k = k[q_start:q_end].unsqueeze(2).contiguous()
|
||||
v = v[q_start:q_end].unsqueeze(2).contiguous()
|
||||
slot_id = state_indices_tensor[slot_start:slot_end]
|
||||
return linear_decode_forward_triton(
|
||||
q, k, v, kv_cache, slope_rate, slot_id, block_size
|
||||
)
|
||||
|
||||
|
||||
def linear_attention_prefill_and_mix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
state_indices_tensor: torch.Tensor,
|
||||
attn_metadata: LinearAttentionMetadata,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
decode_fn: Callable[..., torch.Tensor],
|
||||
prefix_fn: Callable[..., torch.Tensor],
|
||||
layer_idx: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden = []
|
||||
for _prefill_idx in range(getattr(attn_metadata, "num_prefills", 0)):
|
||||
if _prefill_idx >= len(attn_metadata.query_start_loc):
|
||||
break
|
||||
if _prefill_idx >= len(state_indices_tensor):
|
||||
break
|
||||
offset = attn_metadata.num_decode_tokens
|
||||
_start = attn_metadata.query_start_loc[offset + _prefill_idx]
|
||||
_end = attn_metadata.query_start_loc[offset + _prefill_idx + 1]
|
||||
slot_id = state_indices_tensor[offset + _prefill_idx]
|
||||
qs = q[_start:_end].transpose(0, 1).contiguous()
|
||||
ks = k[_start:_end].transpose(0, 1).contiguous()
|
||||
vs = v[_start:_end].transpose(0, 1).contiguous()
|
||||
slice_layer_cache = kv_cache[slot_id, ...]
|
||||
out_slice = prefix_fn(
|
||||
qs,
|
||||
ks,
|
||||
vs,
|
||||
slice_layer_cache,
|
||||
slope_rate,
|
||||
block_size,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
hidden.append(out_slice.contiguous())
|
||||
|
||||
if attn_metadata.num_decode_tokens > 0:
|
||||
hidden_decode = decode_fn(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden.insert(0, hidden_decode)
|
||||
|
||||
if not hidden:
|
||||
return torch.empty((0, q.size(-1)), device=q.device, dtype=q.dtype)
|
||||
|
||||
hidden = torch.concat(hidden, dim=0).contiguous()
|
||||
return hidden
|
||||
|
||||
|
||||
class MiniMaxText01LinearKernel:
|
||||
@staticmethod
|
||||
def jit_linear_forward_prefix(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_caches: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
block_size: int,
|
||||
layer_idx: int | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
slope_rate = slope_rate.to(torch.float32)
|
||||
should_pad_dim = q.dim() == 3
|
||||
if should_pad_dim:
|
||||
q = q.unsqueeze(0)
|
||||
k = k.unsqueeze(0)
|
||||
v = v.unsqueeze(0)
|
||||
b, h, n, d = q.shape
|
||||
e = d
|
||||
kv_history = kv_caches.reshape(1, h, d, e).contiguous()
|
||||
output, kv_history = lightning_attention(
|
||||
q, k, v, slope_rate, block_size=block_size, kv_history=kv_history
|
||||
)
|
||||
kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e))
|
||||
assert output.shape[0] == 1, "batch size must be 1"
|
||||
return rearrange(output.squeeze(0), "h n d -> n (h d)")
|
||||
|
||||
|
||||
@PluggableLayer.register("minimax_text_01_attention")
|
||||
class MiniMaxText01LinearAttention(LinearAttention):
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
vllm_config,
|
||||
prefix: str = "linear_attn",
|
||||
) -> None:
|
||||
super().__init__(config, vllm_config, prefix)
|
||||
|
||||
self.tp_heads = self.num_heads // self.tp_size
|
||||
self.qkv_size = self.num_heads * self.head_dim
|
||||
self.tp_hidden = self.head_dim * self.tp_heads
|
||||
|
||||
self.qkv_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size * 3,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.output_gate = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_inner_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.output_gate",
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
self.norm = MiniMaxText01RMSNormTP(
|
||||
self.hidden_inner_size,
|
||||
eps=1e-5,
|
||||
)
|
||||
|
||||
slope_rate = MiniMaxText01LinearAttention._build_slope_tensor(self.num_heads)
|
||||
if self.num_hidden_layers <= 1:
|
||||
self.slope_rate = slope_rate * (1 + 1e-5)
|
||||
else:
|
||||
self.slope_rate = slope_rate * (
|
||||
1 - self.layer_idx / (self.num_hidden_layers - 1) + 1e-5
|
||||
)
|
||||
self.tp_slope = self.slope_rate[
|
||||
self.tp_rank * self.tp_heads : (self.tp_rank + 1) * self.tp_heads
|
||||
].contiguous()
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@staticmethod
|
||||
def weight_direct_load(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
assert param.size() == loaded_weight.size()
|
||||
param.data.copy_(loaded_weight)
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _build_slope_tensor(n_attention_heads: int):
|
||||
def get_slopes(n):
|
||||
def get_slopes_power_of_2(n):
|
||||
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
|
||||
ratio = start
|
||||
return [start * ratio**i for i in range(n)]
|
||||
|
||||
if math.log2(n).is_integer():
|
||||
return get_slopes_power_of_2(n)
|
||||
else:
|
||||
closest_power_of_2 = 2 ** math.floor(math.log2(n))
|
||||
return (
|
||||
get_slopes_power_of_2(closest_power_of_2)
|
||||
+ get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
|
||||
)
|
||||
|
||||
slopes = torch.tensor(
|
||||
get_slopes(n_attention_heads), dtype=torch.float32
|
||||
).reshape(n_attention_heads, 1, 1)
|
||||
return slopes
|
||||
|
||||
def _prefill_and_mix_infer(
|
||||
self, q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
):
|
||||
return linear_attention_prefill_and_mix(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
kv_cache=kv_cache,
|
||||
state_indices_tensor=state_indices_tensor,
|
||||
attn_metadata=attn_metadata,
|
||||
slope_rate=self.tp_slope,
|
||||
block_size=self.BLOCK,
|
||||
decode_fn=self._decode_infer,
|
||||
prefix_fn=MiniMaxText01LinearKernel.jit_linear_forward_prefix,
|
||||
layer_idx=self.layer_idx,
|
||||
)
|
||||
|
||||
def _decode_infer(self, q, k, v, kv_cache, state_indices_tensor, attn_metadata):
|
||||
hidden = linear_attention_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_cache,
|
||||
self.tp_slope,
|
||||
state_indices_tensor,
|
||||
q_start=0,
|
||||
q_end=attn_metadata.num_decode_tokens,
|
||||
slot_start=0,
|
||||
slot_end=attn_metadata.num_decodes,
|
||||
block_size=32,
|
||||
)
|
||||
return hidden
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
|
||||
) -> None:
|
||||
torch.ops.vllm.linear_attention(
|
||||
hidden_states,
|
||||
output,
|
||||
positions,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def _forward(
|
||||
self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor
|
||||
) -> None:
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw[self.prefix]
|
||||
assert isinstance(attn_metadata, LinearAttentionMetadata)
|
||||
num_actual_tokens = (
|
||||
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
|
||||
)
|
||||
else:
|
||||
num_actual_tokens = hidden_states.shape[0]
|
||||
|
||||
qkv, _ = self.qkv_proj(hidden_states[:num_actual_tokens])
|
||||
qkv32 = qkv.to(torch.float32)
|
||||
qkvact = torch.nn.functional.silu(qkv32)
|
||||
qkvact = qkvact.view((qkv.shape[0], self.tp_heads, -1))
|
||||
q, k, v = torch.split(qkvact, [self.head_dim] * 3, dim=-1)
|
||||
if attn_metadata is not None:
|
||||
kv_cache = self.kv_cache[0]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
clear_linear_attention_cache_for_new_sequences(
|
||||
kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
|
||||
decode_only = getattr(attn_metadata, "num_prefills", 0) == 0
|
||||
if attn_metadata is None:
|
||||
hidden = torch.empty(
|
||||
(q.shape[0], q.shape[1] * q.shape[2]), device=q.device, dtype=q.dtype
|
||||
)
|
||||
else:
|
||||
if not decode_only:
|
||||
hidden = self._prefill_and_mix_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
else:
|
||||
hidden = self._decode_infer(
|
||||
q, k, v, kv_cache, state_indices_tensor, attn_metadata
|
||||
)
|
||||
hidden = self.norm._forward(hidden)
|
||||
gate, _ = self.output_gate(hidden_states[:num_actual_tokens])
|
||||
hidden = F.sigmoid(gate) * hidden
|
||||
hidden = hidden.to(hidden_states.dtype)
|
||||
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden)
|
||||
|
||||
|
||||
def linear_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._forward(hidden_states=hidden_states, output=output, positions=positions)
|
||||
|
||||
|
||||
def linear_attention_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="linear_attention",
|
||||
op_func=linear_attention,
|
||||
mutates_args=["output"],
|
||||
fake_impl=linear_attention_fake,
|
||||
)
|
||||
Reference in New Issue
Block a user