Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Weight permutations for FlashMLA's mega-attention kernel.
|
|
|
|
The kernel reads Q with 16-element head-dim chunks interleaved across heads and
|
|
writes O with 32-element chunks interleaved across the 8 heads of a ``wo_a``
|
|
group. Every permutation here satisfies ``fused = standard[perm]``, and is
|
|
applied once to ``wq_b`` rows and ``wo_a`` columns at load time so the
|
|
surrounding GEMMs produce and consume the kernel's layouts directly -- no
|
|
per-step shuffle.
|
|
"""
|
|
|
|
import torch
|
|
|
|
from vllm.model_executor.model_loader.weight_utils import composed_weight_loader
|
|
|
|
HEAD_DIM = 512
|
|
Q_CHUNK = 16
|
|
O_CHUNK = 32
|
|
WV_GROUP_SIZE = 8
|
|
|
|
|
|
def q_fused_permutation(num_heads: int, head_dim: int = HEAD_DIM) -> torch.Tensor:
|
|
"""``fused[(d // 16) * (H * 16) + h * 16 + d % 16] = standard[h * D + d]``."""
|
|
h = torch.arange(num_heads).view(num_heads, 1)
|
|
d = torch.arange(head_dim).view(1, head_dim)
|
|
fused_index = (d // Q_CHUNK) * (num_heads * Q_CHUNK) + h * Q_CHUNK + d % Q_CHUNK
|
|
perm = torch.empty(num_heads * head_dim, dtype=torch.long)
|
|
perm[fused_index.reshape(-1)] = torch.arange(num_heads * head_dim)
|
|
return perm
|
|
|
|
|
|
def o_fused_permutation(
|
|
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
|
|
) -> torch.Tensor:
|
|
"""``fused[(c * G + h) * 32 + j] = standard[h * D + c * 32 + j]`` per group."""
|
|
h = torch.arange(heads_per_group).view(-1, 1, 1)
|
|
c = torch.arange(head_dim // O_CHUNK).view(1, -1, 1)
|
|
j = torch.arange(O_CHUNK).view(1, 1, -1)
|
|
fused_index = (c * heads_per_group + h) * O_CHUNK + j
|
|
perm = torch.empty(heads_per_group * head_dim, dtype=torch.long)
|
|
perm[fused_index.reshape(-1)] = torch.arange(heads_per_group * head_dim)
|
|
return perm
|
|
|
|
|
|
def o_fused_chunk_permutation(
|
|
heads_per_group: int = WV_GROUP_SIZE, head_dim: int = HEAD_DIM
|
|
) -> torch.Tensor:
|
|
"""Per-32-element-chunk form of :func:`o_fused_permutation`, for scales."""
|
|
return o_fused_permutation(heads_per_group, head_dim)[::O_CHUNK] // O_CHUNK
|
|
|
|
|
|
def permute_q_to_fused(q: torch.Tensor) -> torch.Tensor:
|
|
"""``[N, H, D]`` standard layout -> the same shape in the fused layout."""
|
|
n, h, d = q.shape
|
|
perm = q_fused_permutation(h, d).to(q.device)
|
|
return q.reshape(n, h * d)[:, perm].view(n, h, d)
|
|
|
|
|
|
def _bytes_view(t: torch.Tensor) -> torch.Tensor:
|
|
"""Byte view of a 1-byte-element tensor, so fp8/ue8m0 can be gathered."""
|
|
return t.view(torch.uint8) if t.element_size() == 1 else t
|
|
|
|
|
|
def permute_on_load(param: torch.Tensor, perm: torch.Tensor, dim: int) -> None:
|
|
"""Make ``param``'s weight loader gather the loaded shard by ``perm``."""
|
|
|
|
def gather(t: torch.Tensor) -> torch.Tensor:
|
|
return _bytes_view(t).index_select(dim, perm.to(t.device)).view(t.dtype)
|
|
|
|
param.weight_loader = composed_weight_loader( # type: ignore[attr-defined]
|
|
param.weight_loader, # type: ignore[attr-defined]
|
|
gather,
|
|
)
|