[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
@@ -1,231 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Replicated input embedding + its fused gather/norm kernels.
Groups the ``VLLM_REPLICATE_EMBED`` path in one place: the embedding factory,
the predicate that says whether the fusions apply, and the two Triton fusions
the full on-rank table unlocks --
* ``fused_embed_norm``: gather + a chained RMSNorm (e.g. the first decoder
layer's ``input_layernorm``), and
* ``fused_embed_eh_norm``: gather + pos-0 zeroing + enorm/hnorm + cat, the
embed/previous-hidden input norm for a speculative (MTP/eagle) depth layer
(the replicated-table analogue of the model-local ``fused_eh_norm``, which
takes precomputed embeds).
Self-contained (no model-local imports) so it can live under ``layers/``.
"""
import torch
import vllm.envs as envs
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _rms_norm(x, w, eps, HIDDEN_SIZE: tl.constexpr):
x = x.to(tl.float32)
mean_sq = tl.sum(x * x, axis=0) / HIDDEN_SIZE
rrms = tl.rsqrt(mean_sq + eps)
w = w.to(tl.float32)
return (x * rrms) * w
def make_input_embedding(
num_embeddings: int,
embedding_dim: int,
*,
params_dtype: torch.dtype | None = None,
quant_config=None,
prefix: str = "",
tie_word_embeddings: bool = False,
) -> VocabParallelEmbedding:
"""Input token embedding with an optional replicated escape hatch.
``VLLM_REPLICATE_EMBED=1`` builds the embedding with ``disable_tp``: the full
table lives on every rank and the lookup is a local gather with no mask and
no all-reduce, which unlocks the fused gather+norm path. The cost is a full
table per rank at TP>1 (no extra memory at TP=1, where vocab-parallel is
already unsharded). A replicated, unsharded table cannot be tied to a
vocab-parallel ``ParallelLMHead``, so tied word embeddings are rejected at
TP>1 (at TP=1 ``disable_tp`` is a no-op and tying still works).
"""
disable_tp = envs.VLLM_REPLICATE_EMBED
if disable_tp and tie_word_embeddings:
assert get_tensor_model_parallel_world_size() == 1, (
"VLLM_REPLICATE_EMBED is unsupported with tied word embeddings "
"(the replicated table cannot tie to a vocab-parallel lm_head)"
)
return VocabParallelEmbedding(
num_embeddings,
embedding_dim,
params_dtype=params_dtype,
quant_config=quant_config,
prefix=prefix,
disable_tp=disable_tp,
)
def has_full_vocab_on_rank(embedding: torch.nn.Module) -> bool:
"""Whether ``embedding.weight`` is the whole vocab as a plain [V, H] table.
The fused gather kernels index the table directly, so they need every row
on-rank (``disable_tp``, or any TP=1 run) and an unquantized weight.
"""
return getattr(embedding, "tp_size", 0) == 1 and isinstance(
getattr(embedding, "quant_method", None), UnquantizedEmbeddingMethod
)
@triton.jit
def _fused_embed_norm_kernel(
ids_ptr, # [T] token ids
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
table_stride_0,
out_ptr, # [T, H] gathered embedding (the residual stream)
normed_ptr, # [T, H] rmsnorm(out, chain_w) (HAS_NORM only)
chain_w_ptr, # [H] next norm weight (HAS_NORM only)
eps,
H: tl.constexpr,
BLOCK: tl.constexpr,
HAS_NORM: tl.constexpr,
):
tok = tl.program_id(0).to(tl.int64)
off = tl.arange(0, BLOCK)
mask = off < H
row = tl.load(ids_ptr + tok).to(tl.int64)
x = tl.load(table_ptr + row * table_stride_0 + off, mask=mask, other=0.0)
tl.store(out_ptr + tok * H + off, x, mask=mask)
if HAS_NORM:
w = tl.load(chain_w_ptr + off, mask=mask)
y = _rms_norm(x, w, eps, H).to(normed_ptr.dtype.element_ty)
tl.store(normed_ptr + tok * H + off, y, mask=mask)
# Base model fusion
def fused_embed_norm(
input_ids: torch.Tensor,
embed_table: torch.Tensor,
chain_weight: torch.Tensor | None = None,
eps: float = 0.0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Fused embedding row gather (``embed_table[input_ids]``).
Requires the full vocab on-rank (replicated embedding). When
``chain_weight`` is given, also emits ``rmsnorm(gathered, chain_weight)``
(the first decoder layer's ``input_layernorm``) as a second output in the
same launch, so the returned pair is ``(residual, normed_input)``. Bit-exact
vs a plain gather followed by an ``RMSNorm``.
"""
assert embed_table.ndim == 2, embed_table.shape
ids = input_ids.view(-1)
(t,) = ids.shape
h = embed_table.shape[1]
if chain_weight is not None:
assert chain_weight.shape == (h,), (chain_weight.shape, h)
out = torch.empty((t, h), dtype=embed_table.dtype, device=embed_table.device)
normed = torch.empty_like(out) if chain_weight is not None else None
if t > 0:
block = triton.next_power_of_2(h)
_fused_embed_norm_kernel[(t,)](
ids,
embed_table,
embed_table.stride(0),
out,
normed if normed is not None else out,
chain_weight if chain_weight is not None else embed_table,
eps,
h,
block,
HAS_NORM=chain_weight is not None,
num_warps=min(32, max(4, block // 512)),
)
if normed is not None:
return out, normed
return out
@triton.jit
def _fused_embed_eh_norm_kernel(
pos_ptr,
ids_ptr, # [T] token ids
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
table_stride,
prev_ptr, # [T, H] previous-step hidden
prev_stride,
enorm_w_ptr,
hnorm_w_ptr,
eps,
out_ptr, # [T, 2H]
out_stride,
H: tl.constexpr,
BLOCK: tl.constexpr,
):
"""MTP input fusion with a folded embedding gather: gather
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
cat. Requires the full table on-rank (replicated embedding)."""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
pos = tl.load(pos_ptr + tok)
row = tl.load(ids_ptr + tok).to(tl.int64)
e = tl.load(table_ptr + row * table_stride + off, mask=mask, other=0.0)
e = tl.where(pos == 0, 0.0, e.to(tl.float32))
ew = tl.load(enorm_w_ptr + off, mask=mask)
e_normed = _rms_norm(e, ew, eps, H)
tl.store(out_ptr + tok * out_stride + off, e_normed, mask=mask)
p = tl.load(prev_ptr + tok * prev_stride + off, mask=mask, other=0.0)
hw = tl.load(hnorm_w_ptr + off, mask=mask)
p_normed = _rms_norm(p, hw, eps, H)
tl.store(out_ptr + tok * out_stride + H + off, p_normed, mask=mask)
# MTP fusion
def fused_embed_eh_norm(
positions: torch.Tensor,
input_ids: torch.Tensor,
embed_table: torch.Tensor,
previous_hidden: torch.Tensor,
enorm_w: torch.Tensor,
hnorm_w: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""Fused ``cat([enorm(masked embed_table[ids]), hnorm(prev_hidden)])`` -> [N, 2H].
Folds the embedding row gather into the MTP eh-norm launch; requires the full
table on-rank (replicated embedding). Bit-exact vs gathering ``embed_table[
input_ids]`` and passing it to the model-local ``fused_eh_norm``.
"""
assert previous_hidden.ndim == 2 and embed_table.ndim == 2
n, h = previous_hidden.shape
assert positions.shape == (n,) and input_ids.view(-1).shape == (n,)
assert embed_table.shape[1] == h, (embed_table.shape, h)
assert enorm_w.shape == (h,) and hnorm_w.shape == (h,)
out = torch.empty(
n, 2 * h, dtype=previous_hidden.dtype, device=previous_hidden.device
)
_fused_embed_eh_norm_kernel[(n,)](
positions,
input_ids,
embed_table,
embed_table.stride(0),
previous_hidden,
previous_hidden.stride(0),
enorm_w,
hnorm_w,
eps,
out,
out.stride(0),
h,
triton.next_power_of_2(h),
)
return out
@@ -1,391 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused MoE Triton kernels."""
import functools
import json
import os
import torch
from vllm.logger import init_logger
from vllm.utils.platform_utils import get_device_name_as_file_name
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
# Valid tiles for the aiter a16w-mix int4 kernels (aiter#4646): stage1 requires
# tile_n in {64,128} and tile_k in {128,256} (k_wave=1); stage2 requires tile_n2==128
# and tile_k2 in {128,256}. The pre-#4646 defaults (tile_k=512/64, tile_n2=256) are no
# longer registered and make the port emit an empty-fragment kernel.
_FLYDSL_MOE_DEFAULT_CONFIG = {
1: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
2: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
4: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
8: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
16: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
24: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
32: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
48: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
64: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
128: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
256: {"tile_m": 16, "tile_n": 128, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
512: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
1024: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
2048: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
4096: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
8192: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
}
# aiter#4646's a16w-mix int4 kernels only register a fixed tile set. Tiles from before
# the bump (default config or tuned JSONs) carry tile_k=512/64 and tile_n2=256, which
# emit an empty-fragment kernel (IndexError at emit / wrong output). Snap any resolved
# tiles to the nearest registered value so a stale tuned config can't crash the kernel.
_FLYDSL_INT4_TILE_M = (16, 32, 64, 128)
_FLYDSL_INT4_TILE_N1 = (64, 128)
_FLYDSL_INT4_TILE_K = (128, 256)
def _snap_flydsl_int4_tiles(tile_m, tile_n, tile_k, tile_n2, tile_k2):
def _nearest(v, allowed):
return min(allowed, key=lambda a: (abs(a - v), a))
snapped = (
_nearest(tile_m, _FLYDSL_INT4_TILE_M),
_nearest(tile_n, _FLYDSL_INT4_TILE_N1),
_nearest(tile_k, _FLYDSL_INT4_TILE_K),
128, # stage2 N tile is only registered at 128
_nearest(tile_k2, _FLYDSL_INT4_TILE_K),
)
if snapped != (tile_m, tile_n, tile_k, tile_n2, tile_k2):
logger.warning_once(
"FlyDSL int4 MoE tiles %s are not registered post-aiter#4646; "
"using %s. Retune the config for the a16w-mix kernel.",
(tile_m, tile_n, tile_k, tile_n2, tile_k2),
snapped,
scope="local",
)
return snapped
def moe_sorting(
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
*,
num_experts: int,
model_dim: int,
block_m: int,
):
# Imported lazily so aiter's import-time sort-backend env vars are read
# when this is called, not whenever this module happens to be imported.
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
topk_ids_i32 = topk_ids.to(torch.int32)
topk_w_f32 = topk_weights.to(torch.float32)
sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = (
aiter_moe_sorting(
topk_ids_i32,
topk_w_f32,
num_experts,
model_dim,
torch.float16,
block_m,
)
)
if num_valid_ids.numel() > 1:
num_valid_ids = num_valid_ids[:1].contiguous()
return sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids
def build_routing_buffers(
*,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
num_experts: int,
model_dim: int,
tile_m: int,
):
res = moe_sorting(
topk_ids,
topk_weights,
num_experts=num_experts,
model_dim=model_dim,
block_m=tile_m,
)
if res is None:
raise RuntimeError(
"aiter moe_sorting failed/unavailable; cannot build routing buffers."
)
sorted_token_ids, sorted_weights, sorted_expert_ids, num_valid_ids = res
sorted_token_ids = sorted_token_ids.contiguous()
sorted_weights = sorted_weights.contiguous()
sorted_expert_ids = sorted_expert_ids.contiguous()
sorted_size = int(sorted_token_ids.numel())
blocks = int(sorted_expert_ids.numel())
return (
sorted_token_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
sorted_size,
blocks,
)
@functools.lru_cache
def try_get_optimal_config(num_experts, inter_dim):
device_name = get_device_name_as_file_name()
json_file_name = (
f"E={num_experts},N={inter_dim},device_name={device_name},"
"dtype=int4_w4a16,backend=flydsl.json"
)
config_file_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name
)
if os.path.exists(config_file_path):
with open(config_file_path) as f:
logger.info_once(
"Using tuned FlyDSL MoE config from %s",
config_file_path,
scope="global",
)
tuned_config = json.load(f)
return {int(key): val for key, val in tuned_config.items()}
logger.warning_once(
"Using default FlyDSL MoE config. Performance might be sub-optimal! "
"Config file not found at %s",
config_file_path,
scope="local",
)
return _FLYDSL_MOE_DEFAULT_CONFIG
def fused_flydsl_moe_impl(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
num_experts: int,
inter_dim: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
topk: int = 8,
group_size: int = 32,
doweight_stage1: bool = False,
in_dtype: str = "int4_bf16",
out_dtype: str = "bf16",
scale_is_bf16: bool = True,
tile_m: int | None = None,
tile_n: int | None = None,
tile_k: int | None = None,
tile_n2: int | None = None,
tile_k2: int | None = None,
) -> torch.Tensor:
# Imported lazily -- see the comment in moe_sorting() above.
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
flydsl_a16w4_gemm2,
)
device = hidden_states.device
tokens = hidden_states.shape[0]
model_dim = hidden_states.shape[1]
tuned_config = {}
if tile_m and tile_n and tile_k and tile_n2 and tile_k2:
tuned_config["tile_m"] = tile_m
tuned_config["tile_n"] = tile_n
tuned_config["tile_k"] = tile_k
tuned_config["tile_n2"] = tile_n2
tuned_config["tile_k2"] = tile_k2
else:
tuned_config = try_get_optimal_config(num_experts, inter_dim)
tuned_config = tuned_config[
min(tuned_config.keys(), key=lambda x: abs(x - tokens))
]
out_torch_dtype = torch.bfloat16 if out_dtype == "bf16" else torch.float16
tile_m = tuned_config["tile_m"]
tile_n = tuned_config["tile_n"]
tile_k = tuned_config["tile_k"]
tile_n2 = tuned_config["tile_n2"]
tile_k2 = tuned_config["tile_k2"]
tile_m, tile_n, tile_k, tile_n2, tile_k2 = _snap_flydsl_int4_tiles(
tile_m, tile_n, tile_k, tile_n2, tile_k2
)
routing = build_routing_buffers(
topk_ids=topk_ids,
topk_weights=topk_weights,
num_experts=num_experts,
model_dim=model_dim,
tile_m=tile_m,
)
(
sorted_token_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
sorted_size,
blocks,
) = routing
# aiter#4646 (v0.1.20) removed compile_moe_gemm{1,2} + the flyc.compile pointer ABI
# and replaced them with the a16w-mix wrappers, which compile+launch internally
# (functools-cached) and take torch tensors directly. w_dtype="int4" is the a16wi4
# (W4A16) path. The stage-1 intermediate now lives in sorted
# [sorted_size, inter_dim] layout, and topk weighting is applied in stage-2 via
# sorted_weights (the port has no stage-1 doweight).
sorted_weights_1d = sorted_weights.view(-1).contiguous()
empty_scale = torch.empty((0,), device=device, dtype=torch.uint8)
cumsum_tensor = num_valid_ids.to(torch.int32).contiguous()
m_indices = sorted_token_ids.to(torch.int32).contiguous()
# stage1: bf16 A x int4 W1 + SiLU -> bf16 sorted intermediate.
inter_sorted = torch.empty(
(sorted_size, inter_dim), device=device, dtype=torch.bfloat16
)
w1_scale_u8 = (
w1_scale.view(torch.uint8).contiguous().view(-1)
if w1_scale is not None
else empty_scale
)
flydsl_a16w4_gemm1(
a_bf16=hidden_states.to(torch.bfloat16).contiguous(),
w1_u8=w1.view(torch.uint8).contiguous(),
w1_scale_u8=w1_scale_u8,
sorted_expert_ids=sorted_expert_ids,
cumsum_tensor=cumsum_tensor,
m_indices=m_indices,
inter_sorted_bf16=inter_sorted,
n_tokens=tokens,
NE=num_experts,
D_HIDDEN=model_dim,
D_INTER=inter_dim,
topk=topk,
tile_m=tile_m,
tile_n=tile_n,
tile_k=tile_k,
waves_per_eu=None,
act="silu",
w_dtype="int4",
w_layout="standard",
)
# stage2: bf16 inter x int4 W2 -> weighted reduce into [tokens, model_dim].
out_stage2 = torch.zeros((tokens, model_dim), device=device, dtype=out_torch_dtype)
w2_scale_u8 = (
w2_scale.view(torch.uint8).contiguous().view(-1)
if w2_scale is not None
else empty_scale
)
flydsl_a16w4_gemm2(
inter_sorted_bf16=inter_sorted,
w2_u8=w2.view(torch.uint8).contiguous(),
w2_scale_u8=w2_scale_u8,
sorted_expert_ids=sorted_expert_ids,
cumsum_tensor=cumsum_tensor,
sorted_token_ids=sorted_token_ids,
sorted_weights=sorted_weights_1d,
flat_out=out_stage2.view(-1),
M_logical=tokens,
max_sorted=sorted_size,
NE=num_experts,
D_HIDDEN=model_dim,
D_INTER=inter_dim,
topk=topk,
tile_m=tile_m,
tile_n=tile_n2,
tile_k=tile_k2,
waves_per_eu=None,
w_dtype="int4",
)
return out_stage2
def fused_flydsl_moe_impl_fake(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
num_experts: int,
inter_dim: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
topk: int = 8,
group_size: int = 32,
doweight_stage1: bool = False,
in_dtype: str = "int4_bf16",
out_dtype: str = "bf16",
scale_is_bf16: bool = True,
tile_m: int | None = None,
tile_n: int | None = None,
tile_k: int | None = None,
tile_n2: int | None = None,
tile_k2: int | None = None,
) -> torch.Tensor:
return torch.empty_like(hidden_states)
direct_register_custom_op(
op_name="fused_flydsl_moe_impl",
op_func=fused_flydsl_moe_impl,
fake_impl=fused_flydsl_moe_impl_fake,
)
def fused_flydsl_moe(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
num_experts: int,
inter_dim: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
topk: int = 8,
group_size: int = 32,
doweight_stage1: bool = False,
in_dtype: str = "int4_bf16",
out_dtype: str = "bf16",
scale_is_bf16: bool = True,
config: dict | None = None,
) -> torch.Tensor:
tile_m = None
tile_n = None
tile_k = None
tile_n2 = None
tile_k2 = None
if config is not None:
tile_m = config.get("tile_m")
tile_n = config.get("tile_n")
tile_k = config.get("tile_k")
tile_n2 = config.get("tile_n2")
tile_k2 = config.get("tile_k2")
return torch.ops.vllm.fused_flydsl_moe_impl(
hidden_states=hidden_states,
w1=w1,
w2=w2,
num_experts=num_experts,
inter_dim=inter_dim,
topk_weights=topk_weights,
topk_ids=topk_ids,
w1_scale=w1_scale,
w2_scale=w2_scale,
topk=topk,
group_size=group_size,
doweight_stage1=doweight_stage1,
in_dtype=in_dtype,
out_dtype=out_dtype,
scale_is_bf16=scale_is_bf16,
tile_m=tile_m,
tile_n=tile_n,
tile_k=tile_k,
tile_n2=tile_n2,
tile_k2=tile_k2,
)
@@ -1,682 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton input-staging kernels for DeepGEMM MegaMoE.
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
global scales, or copy them as BF16, and repack the routing top-k tensors into
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
"""
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kMxfp8Dynamic,
)
from vllm.triton_utils import tl, triton
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
@triton.jit
def _quantize_fp8_groups(
hidden,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
USE_UE8M0: tl.constexpr,
):
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
num_groups: tl.constexpr = BLOCK_K // GROUP_K
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
amax = tl.max(hidden_groups, axis=2)
amax = tl.maximum(amax, 1.0e-4)
scale = amax / 448.0
if USE_UE8M0:
scale_bits = scale.to(tl.uint32, bitcast=True)
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
tl.uint32
)
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
else:
scale_exp = scale.to(tl.uint32, bitcast=True)
quant_scale = scale
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
@triton.jit
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
offsets = tl.arange(0, NUM_BYTES)
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
@triton.jit
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
m_block_id = token_id // SHARED_BLOCK_M
m_in_block = token_id % SHARED_BLOCK_M
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
transposed_m = (
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
)
return m_block_id * aligned_block_m + transposed_m
@triton.jit
def _store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
) -> None:
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
topk_offsets = tl.arange(0, BLOCK_TOPK)
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
if is_padding is not None:
token_is_padding = tl.load(
is_padding + token_id * is_padding_stride_m,
mask=token_mask,
other=True,
)
ids = tl.load(
topk_ids
+ token_id[:, None] * topk_ids_stride_m
+ topk_offsets[None, :] * topk_ids_stride_k,
mask=topk_mask,
other=0,
).to(tl.int64)
ids = tl.where(token_is_padding[:, None], -1, ids)
tl.store(
topk_idx_out
+ token_id[:, None] * topk_idx_stride_m
+ topk_offsets[None, :] * topk_idx_stride_k,
ids,
mask=topk_mask,
)
weights = tl.load(
topk_weights
+ token_id[:, None] * topk_weights_stride_m
+ topk_offsets[None, :] * topk_weights_stride_k,
mask=topk_mask,
other=0.0,
)
weights = tl.where(token_is_padding[:, None], 0.0, weights)
tl.store(
topk_weights_out
+ token_id[:, None] * topk_weights_out_stride_m
+ topk_offsets[None, :] * topk_weights_out_stride_k,
weights,
mask=topk_mask,
)
@triton.jit(do_not_specialize=["num_tokens"])
def _prepare_megamoe_inputs_kernel(
hidden_states,
x_out,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_k: tl.constexpr,
x_sf_stride_m: tl.constexpr,
x_sf_stride_k: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
QUANTIZE: tl.constexpr,
USE_UE8M0: tl.constexpr,
) -> None:
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_block_id = tl.program_id(1)
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
hidden = tl.load(
hidden_states
+ token_id[:, None] * hidden_stride_m
+ k_offsets[None, :] * hidden_stride_k,
mask=k_mask,
other=0.0,
).to(tl.float32)
if not QUANTIZE:
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
hidden.to(x_out.dtype.element_ty),
mask=k_mask,
)
else:
num_groups: tl.constexpr = BLOCK_K // GROUP_K
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
)
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
fp8,
mask=k_mask,
)
if USE_UE8M0:
packed_scale = _pack_bytes(scale_exp, num_groups)
tl.store(
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
packed_scale,
mask=token_mask,
)
else:
sf_offset = tl.arange(0, num_groups)
tl.store(
x_sf
+ token_id[:, None] * x_sf_stride_m
+ k_block_id * x_sf_stride_k
+ sf_offset[None, :],
quant_scale,
mask=token_mask[:, None],
)
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
# in an MN-major layout whose row permutation depends on the MegaMoE
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
# is already resident, avoiding another kernel and temporary tensor.
if USE_UE8M0 and shared_x_sf is not None:
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
tl.store(
shared_x_sf
+ shared_row * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
packed_scale,
mask=token_mask,
)
if k_block_id == 0:
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
def _prepare_megamoe_nvfp4_inputs_kernel(
hidden_states,
x_out,
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_sf_stride_m: tl.constexpr,
shared_x_stride_m: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
l2_activation_scale,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
num_local_experts: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
BLOCK_E: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
SHARED_MXFP8: tl.constexpr,
) -> None:
if l2_alpha is not None:
experts = tl.arange(0, BLOCK_E)
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_offsets = tl.arange(0, BLOCK_K)
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
amax = tl.zeros((BLOCK_M,), tl.float32)
for k_block_id in range(hidden_size // BLOCK_K):
hidden = tl.load(
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
tl.store(x_scales + token_id, x_scale, mask=token_mask)
for k_block_id in range(hidden_size // BLOCK_K):
k_ids = k_block_id * BLOCK_K + k_offsets
hidden = tl.load(
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
)
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
values = tl.reshape(
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
[BLOCK_M, BLOCK_K // 16, 16],
)
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
block_scale = block_scale.to(tl.float8e4nv)
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
magnitude = tl.abs(values)
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
code = (
(magnitude > 0.25).to(tl.int32)
+ (magnitude >= 0.75).to(tl.int32)
+ (magnitude > 1.25).to(tl.int32)
+ (magnitude >= 1.75).to(tl.int32)
+ (magnitude > 2.5).to(tl.int32)
+ (magnitude >= 3.5).to(tl.int32)
+ (magnitude > 5.0).to(tl.int32)
)
code = tl.where((values < 0) & (code != 0), code | 8, code)
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
tl.store(
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
(low | (high << 4)).to(tl.uint8),
mask=token_mask[:, None],
)
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
sf_bytes = tl.reshape(
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
)
tl.store(
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
_pack_bytes(sf_bytes, 4),
mask=token_mask[:, None],
)
if shared_x is not None:
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
if SHARED_MXFP8:
fp8, _, scale_exp = _quantize_fp8_groups(
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
)
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
tl.store(
shared_x_sf
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
_pack_bytes(scale_exp, BLOCK_K // 32),
mask=token_mask,
)
else:
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
scale = hidden_quant.scale
group_shape = scale.group_shape
if (
hidden_quant.dtype != FP8_DTYPE
or not hidden_quant.symmetric
or hidden_quant.scale2 is not None
or scale.static
or not group_shape.is_per_group()
or block_k % group_shape.col != 0
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
):
raise ValueError(
"MegaMoE input staging requires symmetric dynamic fp8 "
f"per-group quantization with a group size dividing {block_k} and "
f"E8M0 or fp32 scales, got {hidden_quant}."
)
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
def prepare_megamoe_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor | None,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
hidden_quant: QuantKey | None = kMxfp8Dynamic,
) -> None:
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
Args:
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Output buffer for the dispatched hidden states.
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
``hidden_quant`` is ``None``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x_sf: Optional output buffer for shared-expert scale factors.
Must be given together with ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
four per int32 into ``x_sf``; fp32 scales are stored directly.
``None`` copies the hidden states into ``x`` unquantized.
"""
num_tokens, hidden_size = hidden_states.shape
if num_tokens == 0:
return
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
block_k = 128
quantize = hidden_quant is not None
if quantize != (x_sf is not None):
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
group_k, use_ue8m0 = (
(block_k, False)
if hidden_quant is None
else _resolve_hidden_quant(hidden_quant, block_k)
)
top_k = topk_ids.shape[1]
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
if (shared_x_sf is None) != (shared_block_m is None):
raise ValueError(
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
)
if shared_x_sf is not None and not use_ue8m0:
raise ValueError(
"MegaMoE shared input staging currently requires "
"UE8M0-packed hidden scales."
)
if shared_x_sf is not None:
assert shared_block_m is not None
if shared_block_m <= 0:
raise ValueError("MegaMoE shared_block_m must be positive.")
expected_sf_k = hidden_size // 128
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
raise ValueError(
"MegaMoE shared_x_sf must have shape "
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
)
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
if shared_x_sf.shape[0] < required_rows:
raise ValueError(
"MegaMoE shared_x_sf has insufficient rows: requires "
f"{required_rows}, got {shared_x_sf.shape[0]}."
)
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
block_m = 8 if num_tokens >= 64 else 1
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
block_topk = triton.next_power_of_2(top_k)
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
_prepare_megamoe_inputs_kernel[grid](
hidden_states,
x,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x.stride(1),
x_sf.stride(0) if x_sf is not None else 0,
x_sf.stride(1) if x_sf is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
padding_stride_m,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
hidden_size,
top_k,
BLOCK_M=block_m,
BLOCK_K=block_k,
GROUP_K=group_k,
BLOCK_TOPK=block_topk,
SHARED_BLOCK_M=shared_block_m or 1,
QUANTIZE=quantize,
USE_UE8M0=use_ue8m0,
num_warps=4,
)
def prepare_megamoe_nvfp4_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor,
x_scales: torch.Tensor,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
w2_global_scale: torch.Tensor | None = None,
l2_alpha: torch.Tensor | None = None,
l2_activation_scale: float = 1.0,
) -> None:
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
quantized to E2M1 with one E4M3 scale per 16 values.
Args:
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
or a BF16 copy of the hidden states.
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
TMA layout for ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
w2_global_scale: Per-local-expert L2 weight global scales.
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
dequantization scale of each local expert. Written even without
local tokens, as other ranks' tokens still reach these experts.
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
"""
num_tokens, hidden_size = hidden_states.shape
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
shared_block_m is None
):
raise ValueError(
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
"for an fp8 shared_x."
)
if (w2_global_scale is None) != (l2_alpha is None):
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
if tensor is not None and tensor.stride(-1) != 1:
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
block_m = 8 if num_tokens >= 64 else 1
top_k = topk_ids.shape[1]
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
# One program even without tokens, to write l2_alpha.
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
_prepare_megamoe_nvfp4_inputs_kernel[grid](
hidden_states,
x.view(torch.uint8),
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x_sf.stride(0),
shared_x.stride(0) if shared_x is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
is_padding.stride(0) if is_padding is not None else 0,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
l2_activation_scale,
hidden_size,
top_k,
num_local_experts,
BLOCK_M=block_m,
BLOCK_K=128,
BLOCK_TOPK=triton.next_power_of_2(top_k),
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
SHARED_BLOCK_M=shared_block_m or 1,
SHARED_MXFP8=shared_mxfp8,
num_warps=4,
)
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
@triton.jit
def _relu_squared_static_fp8_quant_kernel(
x_ptr,
scale_ptr,
output_ptr,
num_elements,
FP8_MIN: tl.constexpr,
FP8_MAX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < num_elements
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
input_is_nan = x != x
relu = tl.maximum(x, 0.0)
# Preserve the standalone ReLU2 kernel's BF16 output boundary.
activated = (relu * relu).to(tl.bfloat16).to(tl.float32)
# Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply.
quantized = activated * (1.0 / tl.load(scale_ptr))
quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN)
# Match the O2 native chain by preserving NaN through the FP8 conversion.
quantized = tl.where(input_is_nan, x, quantized)
tl.store(output_ptr + offsets, quantized, mask=mask)
def relu_squared_static_fp8_quant(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""BF16 ReLU2 followed by static per-tensor FP8 quantization."""
assert x.dtype == torch.bfloat16
assert x.is_contiguous()
scale = linear.input_scale
assert scale.dtype == torch.float32
assert x.device == scale.device
assert scale.numel() == 1
output = torch.empty_like(x, dtype=current_platform.fp8_dtype())
if x.numel() != 0:
block_size = min(triton.next_power_of_2(x.shape[-1]), 2048)
num_warps = min(max(block_size // 256, 1), 4)
grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
_relu_squared_static_fp8_quant_kernel[grid](
x,
scale,
output,
x.numel(),
FP8_MIN=_FP8_MIN,
FP8_MAX=_FP8_MAX,
BLOCK_SIZE=block_size,
num_warps=num_warps,
)
return QuantizedActivation(
data=output,
scale=scale,
orig_dtype=x.dtype,
orig_shape=x.shape,
quant_key=kFp8StaticTensorSym,
)
-405
View File
@@ -1,405 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Top-k kernels for the DSA sparse attention indexer."""
import functools
import torch
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.worker.workspace import current_workspace_manager
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
# cooperative_topk's hard row limit: one cluster wave covers at most this many
# rows. "auto" uses cooperative_topk for every batch within the limit and falls
# back to persistent_topk past it; explicit ``cooperative`` requests are
# validated against the same bound.
AUTO_COOPERATIVE_MAX_ROWS = 64
# ---------------------------------------------------------------------------
# DeepSelect (vllm._deepselect_C)
# ---------------------------------------------------------------------------
# Matches the -1 fill convention used for topk_indices_buffer elsewhere.
IDX_OOB_FILL_VALUE = -1
try:
import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select)
_HAS_DEEP_SELECT_C = True
except ImportError:
_HAS_DEEP_SELECT_C = False
@functools.lru_cache(maxsize=1)
def get_deep_select_stride_requirement() -> tuple[int, int]:
"""Stride alignment requirement (input, output) in bytes."""
return torch.ops.deep_select.get_alignment_requirement()
def is_deep_select_supported(input: torch.Tensor, topk: int) -> bool:
"""Whether the kernel accepts this input (dtype/stride/topk constraints)."""
return (
topk <= 4096
and input.shape[1] < 2**23
and input.dtype in (torch.float32, torch.bfloat16)
and input.stride(1) == 1
and input.stride(0)
* input.element_size()
% get_deep_select_stride_requirement()[0]
== 0
)
def _get_empty_and_aligned_tensor(
dim0: int, dim1: int, device: torch.device, dtype: torch.dtype
) -> torch.Tensor:
"""Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned."""
output_stride_requirement = get_deep_select_stride_requirement()[1] // (
dtype.itemsize
)
assert output_stride_requirement > 0
dim1_rounded = (
(dim1 + output_stride_requirement - 1) // output_stride_requirement
) * output_stride_requirement
return torch.empty((dim0, dim1_rounded), device=device, dtype=dtype)[:, :dim1]
def deep_select_topk(
input: torch.Tensor,
topk: int,
end: torch.Tensor | None = None,
output_idx: torch.Tensor | None = None,
indices_dtype: torch.dtype = torch.int32,
) -> torch.Tensor:
"""Select the top-k indices per row of `input` with DeepSelect.
Args:
input: (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and
stride(0) must be 1024B-aligned.
topk: Number of elements to select per row; must be <= 4096.
end: Optional (num_rows,) int32 tensor with the exclusive right
boundary of each row. Rows with `end[i] < topk` get their
remaining indices filled with -1.
output_idx: Optional preallocated (num_rows, topk) output tensor whose
stride(0) is 32B-aligned (e.g. a slice of a wider buffer).
indices_dtype: Output dtype when `output_idx` is not provided.
Returns:
The (num_rows, topk) indices tensor.
"""
assert input.dim() == 2 and input.stride(1) == 1
assert (
input.stride(0) * input.element_size() % get_deep_select_stride_requirement()[0]
== 0
)
num_rows = input.shape[0]
if output_idx is None:
output_idx = _get_empty_and_aligned_tensor(
num_rows, topk, input.device, indices_dtype
)
else:
assert output_idx.dtype == indices_dtype
assert output_idx.shape[0] >= num_rows and output_idx.shape[1] >= topk
torch.ops.deep_select.topk(
input,
topk,
None, # begin is not supported
end,
False, # sorted_value
False, # sorted_index
None, # output_value
output_idx,
None, # output_idx_offset
IDX_OOB_FILL_VALUE,
float("-inf"), # value_oob_fill_value
False, # return_value
False, # abort_when_nan_found: dummy/capture passes can feed NaN
# logits from uninitialized KV cache; the reference topk kernels do
# not trap on NaN, and trapping would abort CUDA graph capture.
)
return output_idx
# ---------------------------------------------------------------------------
# Backend selection
# ---------------------------------------------------------------------------
@functools.cache
def get_indexer_topk(backend: str) -> "SparseIndexerTopk":
return SparseIndexerTopk(backend)
class SparseIndexerTopk(torch.nn.Module):
"""The sparse indexer's decode top-k stage.
Selects among the available top-k kernels (see
kernel_config.sparse_indexer_topk_backend) and runs the chosen one.
"""
def __init__(self, backend: str | None = None) -> None:
super().__init__()
if backend is None:
backend = (
get_current_vllm_config().kernel_config.sparse_indexer_topk_backend
)
self._backend = backend
self._is_cuda = current_platform.is_cuda()
self._has_deep_select = self._is_cuda and (
current_platform.is_device_capability_family(100)
)
self._has_flashinfer_topk = has_flashinfer()
self._cooperative_capable = self._is_cuda and (
current_platform.has_device_capability(90)
and not current_platform.is_device_capability_family(120)
)
self._is_rocm = current_platform.is_rocm()
self._aiter_enabled = False
self._aiter_capable = False
if self._is_rocm:
from vllm._aiter_ops import rocm_aiter_ops
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
def resolve_backend(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""Resolve the decode top-k implementation from the configured
backend ("auto" = the pre-existing chain, or a validated explicit
value)."""
if self._backend == "auto":
return self._resolve_auto(logits, topk_tokens, num_rows)
failures: list[str] = []
if self._backend == "cooperative":
failures = self._cooperative_constraints(logits, topk_tokens, num_rows)
elif self._backend == "persistent":
if not self._is_cuda:
failures.append("requires a CUDA platform")
if topk_tokens not in (512, 1024, 2048):
failures.append(
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
)
elif self._backend == "deep_select":
if not self._is_cuda:
failures.append("requires a CUDA platform")
elif not self._has_deep_select:
failures.append("requires SM100a/SM103a (10.x device family)")
elif not _HAS_DEEP_SELECT_C:
failures.append("vllm._deepselect_C could not be imported")
elif not is_deep_select_supported(logits, topk_tokens):
failures.append(
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
f" stride={logits.stride()}, topk={topk_tokens}"
)
elif self._backend == "aiter":
if not self._is_rocm:
failures.append("requires a ROCm platform")
elif not self._aiter_enabled:
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
elif not self._aiter_capable:
failures.append(
"AITER has no tuned indexer top-k kernels for this GPU arch"
)
if topk_tokens not in (512, 1024, 2048, 4096):
failures.append(
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
)
elif self._backend == "flashinfer":
if not self._is_cuda:
failures.append("requires a CUDA platform")
if not self._has_flashinfer_topk:
failures.append(
"flashinfer.topk.top_k_ragged_transform is not importable"
)
if logits.dtype != torch.float32 or logits.stride(1) != 1:
failures.append(
f"requires fp32 logits with stride(1) == 1, got "
f"dtype={logits.dtype}, stride={logits.stride()}"
)
if failures:
raise RuntimeError(
f"sparse_indexer_topk_backend='{self._backend}' was requested, but: "
+ "; ".join(failures)
)
return self._backend
def _resolve_auto(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The priority chain: cooperative -> persistent -> per_row.
aiter/deep_select/flashinfer/torch are opt-in only.
cooperative_topk is preferred whenever it is applicable, i.e. within
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
persistent_topk.
aiter not auto-resolved as it does not outperform the per-row on every shape,
hence we delegate to the caller to decide when to use it.
"""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
if self._is_cuda and topk_tokens in (512, 1024, 2048):
return "persistent"
return "per_row"
def _cooperative_constraints(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> list[str]:
"""Unmet constraints of cooperative_topk (empty when applicable)."""
failures = []
if not self._is_cuda:
failures.append("requires a CUDA platform")
if topk_tokens not in (512, 1024, 2048):
failures.append(
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
)
if num_rows > AUTO_COOPERATIVE_MAX_ROWS:
failures.append(
f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}"
)
if logits.stride(0) % 4 != 0:
failures.append(
f"logits.stride(0) must be divisible by 4, got {logits.stride(0)}"
)
if self._is_cuda and not self._cooperative_capable:
failures.append("requires SM90+ and is not supported on the SM12x family")
return failures
@staticmethod
def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor:
"""Per-row exclusive end offsets (int32, (num_rows,)) for top-k
kernels that take ragged lengths (DeepSelect, FlashInfer, torch
reference).
seq_lens is (B, next_n) per-row effective lens for native spec decode
and (B, 1) otherwise, in which case per-row lens are derived the same
way as the other decode top-k kernels.
"""
if seq_lens.numel() == num_rows:
row_ends = seq_lens.reshape(-1)
else:
next_n_offsets = torch.arange(
next_n, dtype=torch.int32, device=seq_lens.device
)
row_ends = (
(seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets)
.clamp_(min=0)
.reshape(-1)
)
assert row_ends.dtype == torch.int32
return row_ends
def forward(
self,
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices)
elif backend == "cooperative":
(topk_workspace,) = current_workspace_manager().get_simultaneous(
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
torch.ops._C.cooperative_topk(
logits,
seq_lens,
topk_indices,
topk_workspace,
topk_tokens,
max_seq_len,
)
elif backend == "persistent":
(topk_workspace,) = current_workspace_manager().get_simultaneous(
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
# persistent_topk's last argument is the host-side "max seq_len
# across rows" bound (it gates the sampled_topk path and clamps
# per-row lengths), not the padded row width: logits is sized to
# max_model_len, so passing its width would incorrectly enable the
# long-row kernels on short-context batches.
torch.ops._C.persistent_topk(
logits,
seq_lens,
topk_indices,
topk_workspace,
topk_tokens,
max_seq_len,
)
elif backend == "flashinfer":
# Deferred: importing flashinfer initializes CUDA at import time.
from flashinfer.topk import top_k_ragged_transform
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
offsets = torch.zeros(
logits.shape[0], dtype=torch.int32, device=logits.device
)
# top_k_ragged_transform selects within [0, row_ends[i]) per
# row and -1-fills past the row length.
indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens)
topk_indices.copy_(indices)
elif backend == "torch":
# Debug reference: mask everything past each row's end, then topk.
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
cols = torch.arange(logits.shape[1], device=logits.device)
masked = logits.masked_fill(
cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf")
)
indices = masked.topk(topk_tokens, dim=-1).indices
in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze(
0
) < row_ends.unsqueeze(1)
indices = torch.where(in_range, indices, -1)
topk_indices.copy_(indices)
elif backend == "aiter":
from vllm._aiter_ops import rocm_aiter_ops
# AITER's decode kernel only implements the 1D/next_n == 1 form,
# which is equivalent once the ragged ends are expanded.
rocm_aiter_ops.indexer_top_k_decode(
logits,
1,
self._row_ends(seq_lens, next_n, logits.shape[0]),
topk_indices,
topk_tokens,
)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
@staticmethod
def _per_row(
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
) -> None:
ops.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
@@ -1,741 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from einops import rearrange
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
@triton.jit
def _fwd_diag_kernel(
Q,
K,
V,
Out,
S,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
CBLOCK: tl.constexpr,
):
# This kernel computes the diagonal blocks of the attention matrix
# Each diagonal block represents attention
# where queries attend to keys in the same block
off = tl.program_id(0)
off_bh = off // NUM_BLOCK # batch-head index
off_block = off % NUM_BLOCK # block index within the sequence
off_cblock = tl.program_id(1) # sub-block index within a block
off_h = off_bh % h # head index
# Calculate base offsets for the current batch and head
qk_offset = off_bh * n * d
v_offset = off_bh * n * e
o_offset = off_bh * n * e
# Calculate offsets for the current block
block_offset = off_block * BLOCK
qk_block_offset = block_offset * d
v_block_offset = block_offset * e
o_block_offset = block_offset * e
# Calculate offsets for the current sub-block
cblock_offset = off_cblock * CBLOCK
q_cblock_offset = cblock_offset * d
o_cblock_offset = cblock_offset * e
# Calculate pointers to the query, key, value, and output tensors
Q_block_ptr = (
Q
+ qk_offset
+ qk_block_offset
+ q_cblock_offset
+ tl.arange(0, CBLOCK)[:, None] * d
+ tl.arange(0, d)[None, :]
)
K_trans_block_ptr = (
K
+ qk_offset
+ qk_block_offset
+ tl.arange(0, CBLOCK)[None, :] * d
+ tl.arange(0, d)[:, None]
)
V_block_ptr = (
V
+ v_offset
+ v_block_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, e)[None, :]
)
O_block_ptr = (
Out
+ o_offset
+ o_block_offset
+ o_cblock_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, e)[None, :]
)
# Load the decay rate for the current head
S_block_ptr = S + off_h
s = tl.load(S_block_ptr)
i = off_cblock
q_index = tl.arange(0, CBLOCK) + i * CBLOCK
# Load query values
q = tl.load(Q_block_ptr, mask=block_offset + q_index[:, None] < n, other=0.0).to(
tl.float32
)
# Initialize output accumulator
qkv = tl.zeros([CBLOCK, e], dtype=tl.float32)
# Process all sub-blocks up to and
# including the current one (causal attention)
for j in range(i + 1):
kv_index = tl.arange(0, CBLOCK) + j * CBLOCK
diff = q_index[:, None] - kv_index[None, :]
s_index = s * diff
# Apply causal mask: only attend to positions before the current one
s_index = tl.where(diff >= 0, -s_index, float("-inf"))
decay = tl.exp(s_index)
# Load key and value
k_trans = tl.load(
K_trans_block_ptr,
mask=block_offset + kv_index[None, :] < n,
other=0.0,
).to(tl.float32)
v = tl.load(
V_block_ptr,
mask=block_offset + kv_index[:, None] < n,
other=0.0,
).to(tl.float32)
# Compute attention scores and apply decay
qk = tl.dot(q, k_trans) * decay
# Compute weighted values and accumulate
qkv += tl.dot(qk, v)
# Move to the next sub-block
K_trans_block_ptr += CBLOCK * d
V_block_ptr += CBLOCK * e
# Store the result
tl.store(
O_block_ptr,
qkv.to(O_block_ptr.dtype.element_ty),
mask=block_offset + q_index[:, None] < n,
)
@triton.jit
def _fwd_kv_parallel(
K,
V,
K_decay,
KV,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
D_FBLOCK: tl.constexpr,
E_FBLOCK: tl.constexpr,
NUM_FBLOCK: tl.constexpr,
CBLOCK: tl.constexpr,
NUM_CBLOCK: tl.constexpr,
):
# This kernel computes the key-value outer
# products for each block in parallel
off_bh = tl.program_id(0) # batch-head index
off_block = tl.program_id(1) # block index
off_h = off_bh % h # head index
block_offset = off_block * BLOCK
# Calculate offsets for the current block
k_block_offset = block_offset * d
v_block_offset = block_offset * e
kv_block_offset = off_block * d * e
# Calculate base offsets for the current batch and head
k_offset = off_bh * n * d
v_offset = off_bh * n * e
kv_offset = off_bh * NUM_BLOCK * d * e
# Calculate pointers to the key, value, and key-value tensors
K_trans_block_ptr = (
K
+ k_offset
+ k_block_offset
+ tl.arange(0, CBLOCK)[None, :] * d
+ tl.arange(0, D_FBLOCK)[:, None]
)
V_block_ptr = (
V
+ v_offset
+ v_block_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
KV_block_ptr = (
KV
+ kv_offset
+ kv_block_offset
+ tl.arange(0, D_FBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
# Load the decay factors for the current head and block
k_decay_ptr = K_decay + off_h * BLOCK + tl.arange(0, CBLOCK)
kv_index = tl.arange(0, CBLOCK)
# Initialize the key-value outer product accumulator
kv = tl.zeros([D_FBLOCK, E_FBLOCK], dtype=tl.float32)
# Handle the last block which might be smaller than BLOCK
split_n = n - (NUM_BLOCK - 1) * BLOCK if off_block == NUM_BLOCK - 1 else BLOCK
left_shift = tl.cdiv(split_n, CBLOCK) * CBLOCK - split_n
num_blocks = min(tl.cdiv(split_n, CBLOCK), NUM_CBLOCK)
k_decay_ptr += (NUM_CBLOCK - num_blocks) * CBLOCK
# Process all sub-blocks in the current block
for j in range(num_blocks):
left_bound = (1 - j) * left_shift
# Load key and value, handling boundary conditions
k_trans = tl.load(
K_trans_block_ptr - left_shift * d,
mask=kv_index[None, :] >= left_bound,
other=0.0,
)
v = tl.load(
V_block_ptr - left_shift * e,
mask=kv_index[:, None] >= left_bound,
other=0.0,
)
# Load decay factor and compute weighted key-value outer product
k_decay = tl.load(k_decay_ptr)
# NOTE: Need to add the extra dim here due to AMD MLIR lowering error.
# Please don't move it back until issue is resolved.
# Issue: https://github.com/ROCm/triton/issues/907
k_decay = k_decay[None, :]
kv += tl.dot(k_trans * k_decay, v)
# Move to the next sub-block
K_trans_block_ptr += CBLOCK * d
V_block_ptr += CBLOCK * e
k_decay_ptr += CBLOCK
# Store the result
tl.store(KV_block_ptr, kv.to(KV_block_ptr.dtype.element_ty))
@triton.jit
def _fwd_kv_reduce(
S,
KV,
KV_HISTORY,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
D_FBLOCK: tl.constexpr,
E_FBLOCK: tl.constexpr,
):
# This kernel reduces the key-value outer products
# across blocks and updates the KV history
off_bh = tl.program_id(0) # batch-head index
off_h = off_bh % h # head index
kv_offset = off_bh * NUM_BLOCK * d * e
# Calculate pointer to the key-value tensor
KV_block_ptr = (
KV
+ kv_offset
+ tl.arange(0, D_FBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
# Load the decay rate for the current head
s_ptrs = S + off_h
s = tl.load(s_ptrs)
# Calculate pointer to the key-value history tensor
kv_history_offset = off_bh * d * e
KV_HISTORY_block_ptr = (
KV_HISTORY
+ kv_history_offset
+ tl.arange(0, D_FBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
# Load the previous key-value history
kv_pre = tl.load(KV_HISTORY_block_ptr).to(tl.float32)
# Process all blocks in reverse order to compute the prefix sum
for i in range(NUM_BLOCK):
block_size = min(n - i * BLOCK, BLOCK)
# Compute decay factor for the current block
block_decay = tl.exp(-s.to(tl.float32) * block_size)
# Load the current key-value outer product
kv_cur = tl.load(KV_block_ptr).to(tl.float32)
# Store the previous key-value history to the current block
tl.store(KV_block_ptr, kv_pre.to(KV_block_ptr.dtype.element_ty))
# Update the key-value history with the current block
kv_pre = block_decay * kv_pre + kv_cur
KV_block_ptr += d * e
# Store the updated key-value history
tl.store(KV_HISTORY_block_ptr, kv_pre)
@triton.jit
def _fwd_none_diag_kernel(
Q,
Out,
S,
KV,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
E_FBLOCK: tl.constexpr,
CBLOCK: tl.constexpr,
NUM_CBLOCK: tl.constexpr,
):
# This kernel computes the non-diagonal blocks of the attention matrix
# Each non-diagonal block represents attention
# where queries attend to keys in different blocks
off_bh = tl.program_id(0) # batch-head index
off_h = off_bh % h # head index
off_nc = tl.program_id(1)
off_n = off_nc // NUM_CBLOCK # block index
off_c = off_nc % NUM_CBLOCK # sub-block index
off_e = tl.program_id(2) # output feature block index
n_offset = off_n * BLOCK
c_offset = off_c * CBLOCK
e_offset = off_e * E_FBLOCK
block_offset = n_offset + c_offset
# Calculate offsets for the current batch, head, and block
q_offset = off_bh * n * d + (n_offset + c_offset) * d
o_offset = off_bh * n * e + (n_offset + c_offset) * e + e_offset
kv_offset = off_bh * NUM_BLOCK * d * e + off_n * d * e + e_offset
# Calculate pointers to the query, output, and key-value tensors
Q_block_ptr = (
Q + q_offset + tl.arange(0, CBLOCK)[:, None] * d + tl.arange(0, d)[None, :]
)
O_block_ptr = (
Out
+ o_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
KV_block_ptr = (
KV + kv_offset + tl.arange(0, d)[:, None] * e + tl.arange(0, E_FBLOCK)[None, :]
)
# Load the decay rate for the current head
S_block_ptr = S + off_h
s = tl.load(S_block_ptr)
c_array = tl.arange(0, CBLOCK)
# Load the key-value outer product for the current block
kv = tl.load(KV_block_ptr).to(tl.float32)
q_index = block_offset + tl.arange(0, CBLOCK)
# Load query values
q = tl.load(Q_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
# Compute decay factors for the current sub-block
q_decay = tl.exp(-s.to(tl.float32) * (off_c * CBLOCK + c_array[:, None]))
# Compute non-diagonal attention output
qkv_none_diag = tl.dot(q, kv) * q_decay
# Load diagonal attention output (computed by _fwd_diag_kernel)
qkv_diag = tl.load(O_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
# Combine diagonal and non-diagonal attention outputs
qkv = qkv_diag + qkv_none_diag
# Store the result
tl.store(
O_block_ptr, qkv.to(O_block_ptr.dtype.element_ty), mask=q_index[:, None] < n
)
class _attention(torch.autograd.Function):
@staticmethod
def forward(ctx, q, k, v, s, kv_history):
# Forward pass of the lightning attention algorithm
q = q.contiguous()
k = k.contiguous()
v = v.contiguous()
s = s.contiguous()
# Check CUDA compute capability (Ampere+ required for flash attention
# path). Other accelerators (ROCm, XPU) rely on their own Triton
# backend support and skip this check.
if current_platform.is_cuda():
capability = torch.cuda.get_device_capability()
if capability[0] < 8:
raise RuntimeError(
"Flash attention currently only supported",
"for compute capability >= 80",
)
# Get input dimensions
b, h, n, d = q.shape
e = v.shape[-1]
# Initialize output tensor
o = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device)
# Set block sizes
BLOCK = 256
NUM_BLOCK = triton.cdiv(n, BLOCK)
CBLOCK = 32
NUM_CBLOCK = BLOCK // CBLOCK
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
# Compute decay factors for keys
array = torch.arange(0, BLOCK, device=q.device) + 1
k_decay = torch.exp(-s * (BLOCK - array.reshape(1, -1)))
# Step 1: Compute diagonal blocks of attention
grid = (b * h * NUM_BLOCK, NUM_CBLOCK)
_fwd_diag_kernel[grid](
q,
k,
v,
o,
s,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
CBLOCK=CBLOCK,
)
# Set feature block sizes
NUM_FBLOCK = 1
D_FBLOCK = d // NUM_FBLOCK
assert d % NUM_FBLOCK == 0
E_FBLOCK = e // NUM_FBLOCK
assert e % NUM_FBLOCK == 0
CBLOCK = 64
NUM_CBLOCK = BLOCK // CBLOCK
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
# Step 2: Compute key-value outer products for each block in parallel
kv = torch.empty((b, h, NUM_BLOCK, d, e), dtype=torch.float32, device=q.device)
grid = (b * h, NUM_BLOCK)
_fwd_kv_parallel[grid](
k,
v,
k_decay,
kv,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
D_FBLOCK=D_FBLOCK,
E_FBLOCK=E_FBLOCK,
NUM_FBLOCK=NUM_FBLOCK,
CBLOCK=CBLOCK,
NUM_CBLOCK=NUM_CBLOCK,
)
# Step 3: Reduce key-value outer products
# across blocks and update KV history
grid = (b * h, NUM_FBLOCK)
_fwd_kv_reduce[grid](
s,
kv,
kv_history,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
D_FBLOCK=D_FBLOCK,
E_FBLOCK=E_FBLOCK,
)
# Step 4: Compute non-diagonal blocks of attention
grid = (b * h, NUM_BLOCK * NUM_CBLOCK)
_fwd_none_diag_kernel[grid](
q,
o,
s,
kv,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
E_FBLOCK=E_FBLOCK,
CBLOCK=CBLOCK,
NUM_CBLOCK=NUM_CBLOCK,
)
# Save tensors for backward pass
ctx.save_for_backward(q, k, v, s, kv)
ctx.BLOCK = BLOCK
return o, torch.cat([kv, kv_history.unsqueeze(2)], dim=2)
# Apply the lightning attention function
lightning_attention_ = _attention.apply
def lightning_attention(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
ed: torch.Tensor,
block_size: int = 256,
kv_history: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Apply lightning attention algorithm
to compute attention efficiently.
Args:
q: Query tensor of shape [batch, heads, seq_len, dim]
k: Key tensor of shape [batch, heads, seq_len, dim]
v: Value tensor of shape [batch, heads, seq_len, dim_v]
ed: Decay rate tensor of shape [heads]
block_size: Size of blocks for block-sparse attention
kv_history: Optional key-value history from previous computations
Returns:
output: Attention output
kv: Updated key-value history
"""
d = q.shape[-1]
e = v.shape[-1]
if ed.dim() == 1:
ed = ed.view(1, -1, 1, 1)
# Split the computation into chunks for better parallelism
m = 128 if d >= 128 else 64
assert d % m == 0, f"Dimension d ({d}) must be divisible by m ({m})"
arr = [m * i for i in range(d // m + 1)]
if arr[-1] != d:
arr.append(d)
n = len(arr)
output = 0
# Initialize or clone key-value history
if kv_history is None:
kv_history = torch.zeros(
(q.shape[0], q.shape[1], d, e), dtype=torch.float32, device=q.device
)
else:
kv_history = kv_history.clone().contiguous()
# Process each chunk and accumulate results
for i in range(n - 1):
s = arr[i]
e = arr[i + 1]
q1 = q[..., s:e]
k1 = k[..., s:e]
o, kv = lightning_attention_(q1, k1, v, ed, kv_history)
output = output + o
return output, kv
@triton.jit
def _linear_attn_decode_kernel(
q_ptr,
k_ptr,
v_ptr,
kv_cache_ptr,
slope_rate,
slot_idx,
output_ptr,
D: tl.constexpr,
qkv_b_stride,
qkv_h_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
pad_slot_id: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Kernel for linear attention decoding with KV cache.
This kernel computes attention for a single token using the KV cache.
"""
pid_b = tl.program_id(0) # batch index
pid_h = tl.program_id(1) # head index
pid_d = tl.program_id(2) # dimension block index
# Load slot index for the current batch
slot_id = tl.load(slot_idx + pid_b).to(tl.int64)
# Skip if slot_id is PAD_SLOT_ID (padding)
if slot_id == pad_slot_id:
return
batch_id = pid_b
head_id = pid_h
# Load decay rate for the current head
ratio = tl.load(slope_rate + pid_h)
# Calculate offsets for dimensions
qk_d_offsets = tl.arange(0, D)
v_d_offsets = tl.arange(0, BLOCK_SIZE) + pid_d * BLOCK_SIZE
cache_d_offsets = (
qk_d_offsets[:, None] * cache_d0_stride + v_d_offsets[None, :] * cache_d1_stride
)
# Calculate offsets for the current batch and head
q_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
k_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
v_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
cache_offset = slot_id * cache_b_stride + head_id * cache_h_stride
# Create masks for loading tensors
qk_mask = qk_d_offsets < D
v_mask = v_d_offsets < D
# Load query, key, and value tensors
q = tl.load(q_ptr + q_offset + qk_d_offsets, mask=qk_mask, other=0.0)
k = tl.load(k_ptr + k_offset + qk_d_offsets, mask=qk_mask, other=0.0)
v = tl.load(v_ptr + v_offset + v_d_offsets, mask=v_mask, other=0.0)
# Compute key-value outer product
kv_outer = k[:, None] * v[None, :]
kv_mask = qk_mask[:, None] & v_mask[None, :]
# Apply decay to previous KV cache
ratio = tl.exp(-ratio)
kv_ptr = kv_cache_ptr + cache_offset + cache_d_offsets
kv_cache_old = tl.load(kv_ptr, mask=kv_mask, other=0.0)
kv_outer = kv_outer + ratio * kv_cache_old
# Compute attention output
output = q[:, None].to(tl.float32) * kv_outer
output = tl.sum(output, axis=0)
# Update KV cache and store output
tl.store(kv_ptr, kv_outer, mask=kv_mask)
tl.store(output_ptr + q_offset + v_d_offsets, output, mask=v_mask)
def linear_decode_forward_triton(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_caches: torch.Tensor,
slope_rate: torch.Tensor,
slot_idx: torch.Tensor,
BLOCK_SIZE: int = 32,
) -> torch.Tensor:
"""Perform linear attention decoding using Triton kernels.
Args:
q: Query tensor of shape [B, H, 1, D]
k: Key tensor of shape [B, H, 1, D]
v: Value tensor of shape [B, H, 1, D]
kv_caches: Key-value cache tensor
slope_rate: Decay rate tensor
slot_idx: Slot indices for batches
BLOCK_SIZE: Size of blocks for processing
Returns:
output: Attention output tensor
"""
B, H, _, D = q.shape
assert k.shape == (B, H, 1, D)
assert v.shape == (B, H, 1, D)
# Initialize output tensor
output = torch.empty_like(q)
# Set grid dimensions for the kernel
grid = (B, H, D // BLOCK_SIZE)
# Calculate strides for tensors
qkv_b_stride = q.stride(0)
qkv_h_stride = q.stride(1)
cache_b_stride = kv_caches.stride(0)
cache_h_stride = kv_caches.stride(1)
cache_d0_stride = kv_caches.stride(2)
cache_d1_stride = kv_caches.stride(3)
# Launch the kernel
_linear_attn_decode_kernel[grid](
q,
k,
v,
kv_caches,
slope_rate,
slot_idx,
output,
D,
qkv_b_stride,
qkv_h_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
pad_slot_id=PAD_SLOT_ID,
BLOCK_SIZE=BLOCK_SIZE,
)
# Reshape output and return
output = rearrange(output, "b h n d -> b n (h d)")
return output.squeeze(1).contiguous()
@@ -1,382 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
# --8<-- [start:short_conv]
@PluggableLayer.register("short_conv")
class ShortConv(MambaBase, PluggableLayer):
# --8<-- [end:short_conv]
def __init__(
self,
config,
dim: int,
layer_idx: int,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.conv_dim = dim
self.L_cache = config.conv_L_cache
self.bias = config.conv_bias
self.conv = ColumnParallelLinear(
input_size=self.L_cache,
output_size=dim,
bias=self.bias,
prefix=f"{prefix}.conv1d",
)
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
# `ColumnParallelLinear` and `set_weight_attrs`
# doesn't allow to override it
self.conv.weight.data = self.conv.weight.data.unsqueeze(1)
self.in_proj = MergedColumnParallelLinear(
input_size=dim,
output_sizes=[dim] * 3,
bias=self.bias,
quant_config=quant_config,
prefix=f"{prefix}.in_proj",
)
self.out_proj = RowParallelLinear(
input_size=dim,
output_size=dim,
bias=self.bias,
quant_config=quant_config,
prefix=f"{prefix}.out_proj",
)
vllm_config = get_current_vllm_config()
self.num_spec = vllm_config.num_speculative_tokens
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
self.kv_cache = (torch.tensor([]),)
self.model_config = model_config
self.cache_config = cache_config
self.prefix = prefix
def forward_native(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
# for causal conv can be plugged in here later.
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_fn_cpu as causal_conv1d_torch,
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
assert attn_metadata is None or isinstance(
attn_metadata, ShortConvAttentionMetadata
)
BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1)
# (dim, kernel_size) — same reshape as forward_cuda
conv_weights = self.conv.weight.view(
self.conv.weight.size(0), self.conv.weight.size(2)
)
if attn_metadata is None:
# Profile run — output value doesn't matter
Bx = (B * x).contiguous()
output_tensor, _ = self.out_proj(C * Bx)
output[: hidden_states.shape[0]] = output_tensor
return
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
) # (num_blocks, dim, state_len)
num_prefills = attn_metadata.num_prefills
num_decodes = attn_metadata.num_decode_tokens
num_prefill_tokens = attn_metadata.num_prefill_tokens
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
num_actual_tokens = num_decodes + num_prefill_tokens
B_d, B_p = torch.split(
B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
C_d, C_p = torch.split(
C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
x_d, x_p = torch.split(
x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
conv_output_list = []
if has_prefill:
assert attn_metadata.state_indices_tensor_p is not None
Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens)
out_p = causal_conv1d_torch(
Bx_p,
conv_weights,
self.conv.bias,
conv_state,
attn_metadata.query_start_loc_p,
attn_metadata.state_indices_tensor_p.flatten(),
attn_metadata.has_initial_states_p,
activation=None,
).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim)
conv_output_list.append(C_p * out_p)
if has_decode:
assert attn_metadata.state_indices_tensor_d is not None
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
Bx_d = B_d * x_d # (num_decodes, dim)
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
conv_state_view = conv_state[state_indices_d].contiguous()
out_d = causal_conv1d_update_torch(
Bx_d.unsqueeze(-1),
conv_state_view,
conv_weights,
self.conv.bias,
activation=None,
).squeeze(-1)
conv_state[state_indices_d] = conv_state_view
else:
out_d = causal_conv1d_update_cpu(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_d,
)
conv_output_list.insert(0, C_d * out_d)
hidden_states_out = torch.vstack(conv_output_list)
output[:num_actual_tokens], _ = self.out_proj(hidden_states_out)
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
torch.ops.vllm.short_conv(
hidden_states,
output,
self.prefix,
)
def forward_cuda(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
forward_context = get_forward_context()
# ShortConvAttentionMetadata contains metadata necessary for the
# short_conv triton kernels to operate in continuous batching and in
# chunked prefill modes; they are computed at top-level model forward
# since they stay the same and reused for all mamba layers in the same
# iteration.
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
has_initial_states_p = attn_metadata.has_initial_states_p
query_start_loc_p = attn_metadata.query_start_loc_p
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1)
conv_weights = self.conv.weight.view(
self.conv.weight.size(0), self.conv.weight.size(2)
)
if attn_metadata is None:
# V1 profile run
Bx = (B * x).contiguous()
hidden_states = C * Bx
contextualized_states, _ = self.out_proj(hidden_states)
return contextualized_states
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
num_actual_tokens = num_decodes + num_prefill_tokens
# NOTE: V1 puts decode before prefill
# Separate prefill and decode by splitting varlen input
# Split along token dimension
B_d, B_p = torch.split(
B[:num_actual_tokens],
[num_decodes, num_prefill_tokens],
dim=0,
)
C_d, C_p = torch.split(
C[:num_actual_tokens],
[num_decodes, num_prefill_tokens],
dim=0,
)
x_d, x_p = torch.split(
x[:num_actual_tokens],
[num_decodes, num_prefill_tokens],
dim=0,
)
conv_output_list = []
if has_prefill:
Bx_p = (B_p * x_p).transpose(0, 1)
Bx = causal_conv1d_fn(
Bx_p,
conv_weights,
self.conv.bias,
activation=None,
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
y = C_p * Bx
conv_output_list.append(y)
if has_decode:
Bx_d = (B_d * x_d).contiguous()
if num_accepted_tokens is not None:
# Speculative decode: the verify step feeds >1 query token per
# decode request, so use the spec-aware conv update path
# (mirrors mamba_mixer2). state_indices_tensor_d is
# (num_decodes, 1 + num_spec_tokens) here.
assert state_indices_tensor_d is not None
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
)
else:
# Non-spec decode
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
)
y = C_d * Bx
conv_output_list.insert(0, y)
# Merge prefill and decode outputs before passing to gated MLP
hidden_states = torch.vstack(conv_output_list)
# Final linear projection
output[:num_actual_tokens], _ = self.out_proj(hidden_states)
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.short_conv_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, ...]]:
return MambaStateShapeCalculator.short_conv_state_shape(
tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.conv_dim,
conv_kernel=self.L_cache,
num_spec=self.num_spec,
)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.SHORT_CONV
@eager_break_during_capture
def short_conv(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
if not current_platform.is_cpu():
self.forward_cuda(hidden_states=hidden_states, output=output)
else:
self.forward_native(hidden_states=hidden_states, output=output)
direct_register_custom_op(
op_name="short_conv",
op_func=short_conv,
mutates_args=["output"],
)
@@ -1,10 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.minimax_rms_norm.rms_norm_tp import (
MiniMaxText01RMSNormTP,
)
__all__ = [
"MiniMaxText01RMSNormTP",
]
@@ -1,316 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import array
import contextlib
import ctypes
import struct
import sys
import threading
import torch
try:
from cuda.bindings import runtime as cudart
except ImportError:
from cuda import cudart
_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment
_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t)
# ---------------------------------------------------------------------------
# CUDA helpers
# ---------------------------------------------------------------------------
def _check(error):
"""Raise on CUDA runtime error."""
success = getattr(cudart.cudaError_t, "cudaSuccess", None) or cudart.cudaError_t(0)
if error != success:
raise RuntimeError(f"CUDA runtime error: {error}")
# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so
# copy the raw handle bytes through getPtr() instead. This is safe because the
# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]).
def _ipc_handle_to_bytes(handle) -> bytes:
return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE)
def _ipc_handle_from_bytes(raw: bytes):
assert len(raw) == _IPC_HANDLE_SIZE, len(raw)
handle = cudart.cudaIpcMemHandle_t()
ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE)
return handle
def _cuda_malloc(size: int):
aligned = ((size + _ALIGN - 1) >> 21) << 21
err, ptr = cudart.cudaMalloc(aligned)
_check(err)
return ptr, aligned
def _cuda_free(ptr: int):
if ptr:
_check(cudart.cudaFree(ptr)[0])
def _cuda_memset_zero(ptr: int, size: int):
_check(cudart.cudaMemset(ptr, 0, size)[0])
def _cuda_memcpy_d2d(dst: int, src: int, size: int):
_check(
cudart.cudaMemcpy(
dst, src, size, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice
)[0]
)
# ---------------------------------------------------------------------------
# IPC buffer
# ---------------------------------------------------------------------------
class IpcBuffer:
"""Allocates CUDA device memory and exchanges IPC handles with all ranks
so that every rank holds a valid device pointer to every other rank's buffer.
"""
def __init__(self, rank: int, world_size: int, size: int, process_group=None):
self.rank = rank
self.world_size = world_size
self.peer_ptrs: list[int] = [0] * world_size
self.local_ptr: int = 0
self._alive = False
if size <= 0:
return
self.local_ptr, _ = _cuda_malloc(size)
_cuda_memset_zero(self.local_ptr, size)
self._alive = True
# --- exchange IPC handles via torch.distributed ---
err, local_handle = cudart.cudaIpcGetMemHandle(self.local_ptr)
_check(err)
all_handles: list[bytes | None] = [None] * world_size
torch.distributed.all_gather_object(
all_handles, _ipc_handle_to_bytes(local_handle), group=process_group
)
for r in range(world_size):
if r == rank:
self.peer_ptrs[r] = self.local_ptr
else:
raw = all_handles[r]
assert raw is not None
handle = _ipc_handle_from_bytes(raw)
err, ptr = cudart.cudaIpcOpenMemHandle(
handle, cudart.cudaIpcMemLazyEnablePeerAccess
)
_check(err)
self.peer_ptrs[r] = ptr
def serialize(self) -> list[int]:
"""Return peer pointers as a list of int64 values (one per rank)."""
raw = b""
for ptr in self.peer_ptrs:
raw += struct.pack("P", ptr)
return array.array("Q", raw).tolist()
def cleanup(self):
if not self._alive:
return
self._alive = False
for r in range(self.world_size):
if self.peer_ptrs[r] == 0:
continue
if r == self.rank:
_cuda_free(self.peer_ptrs[r])
else:
with contextlib.suppress(RuntimeError):
_check(cudart.cudaIpcCloseMemHandle(self.peer_ptrs[r])[0])
self.peer_ptrs[r] = 0
self.local_ptr = 0
def __del__(self):
if not sys.is_finalizing():
self.cleanup()
# ---------------------------------------------------------------------------
# Lamport negative-zero initialization
# ---------------------------------------------------------------------------
def _lamport_fill_neg_zero(device_ptr: int, size_bytes: int):
"""Fill device memory with IEEE-754 negative zero (-0.0f = 0x80000000).
This is the "slot empty" sentinel for the Lamport protocol: the kernel
spin-waits until a value is *not* negative zero.
"""
if size_bytes == 0 or device_ptr == 0:
return
n_floats = size_bytes // 4
# torch preserves -0.0 in IEEE-754
fill = torch.full((n_floats,), -0.0, dtype=torch.float32, device="cuda")
_cuda_memcpy_d2d(device_ptr, fill.data_ptr(), size_bytes)
del fill
# ---------------------------------------------------------------------------
# LamportWorkspace — the main class
# ---------------------------------------------------------------------------
class LamportWorkspace:
"""Self-contained workspace for Lamport-based cross-GPU AllReduce.
Parameters
----------
rank : int
Local rank (0-based).
world_size : int
Total number of ranks in the TP group.
comm_size : int
Size in bytes of *one* Lamport buffer slot. The total IPC allocation
per rank is ``3 * comm_size`` (triple-buffering). Must be large enough
to hold the per-slot data written by the kernel. Use
``compute_comm_size_for_minimax()`` for a safe default.
process_group : optional
``torch.distributed`` process group for IPC handle exchange.
``None`` uses the default group.
"""
def __init__(self, rank: int, world_size: int, comm_size: int, process_group=None):
assert world_size >= 2, "Lamport workspace requires at least 2 ranks"
assert comm_size > 0, "comm_size must be positive"
self.rank = rank
self.world_size = world_size
self.comm_size = comm_size
# 1) Lamport triple-buffer (the only IPC memory the kernel reads/writes)
lamport_total = 3 * comm_size
self._lamport = IpcBuffer(rank, world_size, lamport_total, process_group)
_lamport_fill_neg_zero(self._lamport.local_ptr, lamport_total)
# 2) flag_buffer on device: int32[3] = {counter, unused, lamport_flag}
# counter — used for block-level sync inside the kernel
# unused — reserved (index 1)
# lamport_flag — triple-buffer rotation index (0 → 1 → 2 → 0 …)
self._flag_buf = torch.zeros(3, dtype=torch.int32, device="cuda")
# 3) layout_buffer on device: int64[2] = {clear_size, comm_size}
# clear_size — bytes to clear from *previous* slot (set by kernel)
# comm_size — size of one triple-buffer slot
self._layout_buf = torch.tensor(
[0, comm_size], dtype=torch.int64, device="cuda"
)
# 4) Assemble device-side void* pointer array
N = world_size
ptrs: list[int] = []
ptrs += [0] * N # [0 .. N-1] ipc_buffers (placeholder)
ptrs += [0] * N # [N .. 2N-1] ipc_barriers (placeholder)
ptrs += self._lamport.serialize() # [2N .. 3N-1] lamport peer ptrs
ptrs.append(self._flag_buf.data_ptr()) # [3N] flag_buffer
ptrs.append(self._layout_buf.data_ptr()) # [3N+1] layout_buffer
self._workspace = torch.tensor(ptrs, dtype=torch.int64, device="cuda")
@property
def workspace(self) -> torch.Tensor:
"""Device tensor (int64) that can be passed to the kernel
as ``void** workspace``."""
return self._workspace
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
@staticmethod
def compute_comm_size_for_minimax(
max_tokens: int,
world_size: int,
fused_qk: bool = True,
) -> int:
"""Return a safe ``comm_size`` (in bytes) for MiniMaxReduceRMSKernel.
The kernel stores per-token variance scalars in the Lamport buffer:
- single-matrix path: ``world_size × max_tokens × 4`` bytes per slot
- fused Q+K path: ``world_size × 2 × ceil(max_tokens/4) × 16`` bytes per slot
The returned value is rounded up to 2 MiB alignment.
"""
if fused_qk:
groups = (max_tokens + 3) // 4
slot_bytes = world_size * 2 * groups * 16 # 16 = sizeof(float4)
else:
slot_bytes = world_size * max_tokens * 4 # 4 = sizeof(float)
return ((slot_bytes + _ALIGN - 1) >> 21) << 21
def cleanup(self):
if hasattr(self, "_lamport"):
self._lamport.cleanup()
def __del__(self):
if not sys.is_finalizing():
self.cleanup()
def __repr__(self):
return (
f"LamportWorkspace(rank={self.rank}, world_size={self.world_size}, "
f"comm_size={self.comm_size})"
)
# ---------------------------------------------------------------------------
# Cached convenience function (mirrors TRT-LLM's get_allreduce_workspace)
# ---------------------------------------------------------------------------
_cache_lock = threading.Lock()
_workspace_cache: dict = {}
def get_allreduce_workspace(
rank: int,
world_size: int,
comm_size: int | None = None,
max_tokens: int = 16384,
process_group=None,
) -> torch.Tensor:
"""Return a cached workspace tensor for the given (rank, world_size) pair.
On first call the workspace is allocated and IPC handles are exchanged;
subsequent calls with the same arguments return the cached tensor.
Parameters
----------
rank, world_size : int
TP rank and TP size.
comm_size : int, optional
Explicit slot size in bytes. If ``None``, computed automatically
from ``max_tokens`` and ``world_size`` (fused Q+K path).
max_tokens : int
Maximum number of tokens per batch (used when ``comm_size is None``).
process_group : optional
``torch.distributed`` process group.
"""
if comm_size is None:
comm_size = LamportWorkspace.compute_comm_size_for_minimax(
max_tokens, world_size, fused_qk=True
)
pg_id = id(process_group) if process_group is not None else 0
key = (rank, world_size, comm_size, pg_id)
with _cache_lock:
if key not in _workspace_cache:
ws = LamportWorkspace(rank, world_size, comm_size, process_group)
_workspace_cache[key] = ws
return _workspace_cache[key].workspace
@@ -1,421 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
# Max number of tokens supported by the Lamport fused allreduce+RMSNorm kernel.
# Larger batches fall back to the eager allreduce + RMSNorm path.
MINIMAX_QK_NORM_MAX_TOKEN_NUM = 2048
_MINIMAX_FUSED_AR_RMS_QK = getattr(torch.ops._C, "minimax_allreduce_rms_qk", None)
def _all_reduce_variance(var: torch.Tensor) -> torch.Tensor:
"""All-reduce a per-token variance tensor across the TP group.
Variance is accumulated in fp32 for numerical stability. The FlashInfer
fused all-reduce caches a single global workspace keyed to the model's
16-bit activation dtype (``use_fp32_lamport=False``); routing an fp32
reduction through it would read against a mismatched workspace and corrupt
the result. FlashInfer's fast-path only triggers for 2D inputs, so reducing
a flattened (1D) view keeps these fp32 reductions on custom all-reduce /
pynccl, both of which handle fp32 correctly.
"""
return tensor_model_parallel_all_reduce(var.flatten()).view_as(var)
@triton.jit
def _minimax_qk_var_kernel(
qkv_ptr, # [num_tokens, hidden], 16-bit activations
var_ptr, # [num_tokens, 2], fp32
row_stride, # element stride between tokens in qkv
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
kv_size: tl.constexpr,
BLOCK: tl.constexpr,
):
"""TP-pre stage: per-token mean-of-squares for the q and k segments.
Accumulates in fp32 while reading the 16-bit qkv in place, so no fp32
copy of q/k is materialized. ``var[:, 0]`` is the q variance and
``var[:, 1]`` the k variance; both are the local-shard means, ready for
the all-reduce that follows.
"""
token = tl.program_id(0)
base = qkv_ptr + token * row_stride
q_acc = 0.0
for off in range(0, q_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < q_size
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
q_acc += tl.sum(x * x, axis=0)
k_acc = 0.0
for off in range(0, kv_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < kv_size
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
k_acc += tl.sum(x * x, axis=0)
tl.store(var_ptr + token * 2 + 0, q_acc / q_size)
tl.store(var_ptr + token * 2 + 1, k_acc / kv_size)
@triton.jit
def _minimax_rms_apply_kernel(
qkv_ptr, # [num_tokens, hidden]
var_ptr, # [num_tokens, 2], fp32, all-reduced sum of per-shard means
q_w_ptr, # [q_size], q per-channel weight
k_w_ptr, # [kv_size], k per-channel weight
q_out_ptr, # [num_tokens, q_size], contiguous
k_out_ptr, # [num_tokens, kv_size], contiguous
row_stride, # element stride between tokens in qkv
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
kv_size: tl.constexpr,
tp_world: tl.constexpr, # folds the post-all-reduce /tp_world into rsqrt
eps: tl.constexpr,
BLOCK: tl.constexpr,
):
"""TP-post stage: ``x * rsqrt(var / tp_world + eps) * weight``.
A single program normalizes both the q and k segments of one token, so q
and k share one launch instead of two. The all-reduce yields the sum of
per-shard means, so the ``/ tp_world`` that recovers the global
mean-of-squares is folded into the ``rsqrt`` here rather than run as a
separate elementwise pass over the ``[num_tokens, 2]`` variance tensor.
"""
token = tl.program_id(0)
base = qkv_ptr + token * row_stride
q_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 0) / tp_world + eps)
q_out_row = q_out_ptr + token * q_size
for off in range(0, q_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < q_size
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
w = tl.load(q_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
y = x * q_inv * w
tl.store(q_out_row + idx, y.to(q_out_ptr.dtype.element_ty), mask=mask)
k_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 1) / tp_world + eps)
k_out_row = k_out_ptr + token * kv_size
for off in range(0, kv_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < kv_size
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
w = tl.load(k_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
y = x * k_inv * w
tl.store(k_out_row + idx, y.to(k_out_ptr.dtype.element_ty), mask=mask)
def _minimax_qk_norm_tp_eager(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_world: int,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pure-torch reference path used when Triton is unavailable."""
q, k, _ = qkv.split([q_size, kv_size, kv_size], dim=-1)
orig_dtype = q.dtype
q = q.to(torch.float32)
k = k.to(torch.float32)
q_var = q.pow(2).mean(dim=-1, keepdim=True)
k_var = k.pow(2).mean(dim=-1, keepdim=True)
qk_var = torch.cat([q_var, k_var], dim=-1)
qk_var = _all_reduce_variance(qk_var) / tp_world
q_var, k_var = qk_var.chunk(2, dim=-1)
q = q * torch.rsqrt(q_var + eps) * q_weight
k = k * torch.rsqrt(k_var + eps) * k_weight
return q.to(orig_dtype), k.to(orig_dtype)
def _minimax_qk_norm_tp_fallback(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_rank: int,
tp_world: int,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""All-reduce + QK RMSNorm without the Lamport fused kernel.
The all-reduce is a TP communication barrier and cannot live inside a
single kernel, so the eager-torch path is split into two Triton kernels
around it: a variance reduction before the all-reduce and a normalize
after. Compared to the ``torch.compile`` path this avoids materializing
fp32 copies of q/k and the ``cat``/``chunk`` temporaries.
"""
if not HAS_TRITON:
return _minimax_qk_norm_tp_eager(
qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps
)
num_tokens = qkv.shape[0]
row_stride = qkv.stride(0)
BLOCK = 1024
grid = (num_tokens,)
qk_var = torch.empty(num_tokens, 2, dtype=torch.float32, device=qkv.device)
_minimax_qk_var_kernel[grid](
qkv, qk_var, row_stride, q_size=q_size, kv_size=kv_size, BLOCK=BLOCK
)
# All-reduce sums the per-shard means; the /tp_world that turns this back
# into the global mean is folded into the apply kernel's rsqrt below.
qk_var = _all_reduce_variance(qk_var)
q_out = torch.empty(num_tokens, q_size, dtype=qkv.dtype, device=qkv.device)
k_out = torch.empty(num_tokens, kv_size, dtype=qkv.dtype, device=qkv.device)
_minimax_rms_apply_kernel[grid](
qkv,
qk_var,
q_weight,
k_weight,
q_out,
k_out,
row_stride,
q_size=q_size,
kv_size=kv_size,
tp_world=tp_world,
eps=eps,
BLOCK=BLOCK,
)
return q_out, k_out
def _minimax_qk_norm_fusion(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_rank: int,
tp_world: int,
eps: float,
workspace: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
if (
workspace is not None
and tp_world > 1
and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM
and _MINIMAX_FUSED_AR_RMS_QK is not None
):
return _MINIMAX_FUSED_AR_RMS_QK(
qkv,
q_weight,
k_weight,
workspace,
q_size,
kv_size,
tp_rank,
tp_world,
eps,
)
# ROCm AITER fused QK-norm + AllReduce; skips unaligned shapes.
if tp_world > 1 and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM:
pack_size = 16 // qkv.element_size()
warp_work_size = 32 * pack_size
if q_size % warp_work_size == 0 and kv_size % warp_work_size == 0:
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
if aiter_ar is not None:
q_out, k_out, _ = aiter_ar.aiter_ca.custom_fused_qknorm_ar(
qkv, q_weight, k_weight, eps
)
return q_out, k_out
return _minimax_qk_norm_tp_fallback(
qkv, q_weight, k_weight, q_size, kv_size, tp_rank, tp_world, eps
)
def _minimax_qk_norm_fusion_fake(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_rank: int,
tp_world: int,
eps: float,
workspace: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
return (
torch.empty([num_tokens, q_size], dtype=qkv.dtype, device=qkv.device),
torch.empty([num_tokens, kv_size], dtype=qkv.dtype, device=qkv.device),
)
direct_register_custom_op(
op_name="minimax_qk_norm_fusion",
op_func=_minimax_qk_norm_fusion,
fake_impl=_minimax_qk_norm_fusion_fake,
mutates_args=[],
)
@CustomOp.register("minimax_text01_rmsnorm_tp")
class MiniMaxText01RMSNormTP(CustomOp):
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
*,
weight_shard_world_size: int | None = None,
weight_shard_rank: int | None = None,
) -> None:
super().__init__()
self.tp_world = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
self.weight_shard_world = weight_shard_world_size or self.tp_world
self.weight_shard_rank = (
self.tp_rank if weight_shard_rank is None else weight_shard_rank
)
self.weight = nn.Parameter(torch.ones(hidden_size // self.weight_shard_world))
self.weight.weight_loader = partial(
self.weight_loader,
shard_world_size=self.weight_shard_world,
shard_rank=self.weight_shard_rank,
)
self.variance_epsilon = eps
self.workspace = None
if _MINIMAX_FUSED_AR_RMS_QK is not None and self.tp_world > 1:
from .lamport_workspace import (
get_allreduce_workspace,
)
# The Lamport workspace exchanges CUDA IPC handles and enables peer
# access between GPUs. This requires P2P (IPC peer access) to be
# available; on topologies where it is not (e.g. consumer PCIe cards
# with P2P disabled in the driver), allocation raises. Fall back to
# the eager allreduce + RMSNorm path instead of failing model load.
try:
self.workspace = get_allreduce_workspace(
rank=self.tp_rank,
world_size=self.tp_world,
max_tokens=MINIMAX_QK_NORM_MAX_TOKEN_NUM,
process_group=get_tp_group().cpu_group,
)
except Exception as e:
logger.warning_once(
"Failed to initialize MiniMax fused allreduce+RMSNorm "
"Lamport workspace: %s. This is expected on GPUs without "
"P2P (IPC peer access) support. Falling back to the eager "
"allreduce + RMSNorm path.",
e,
)
self.workspace = None
@staticmethod
def weight_loader(
param: nn.Parameter,
loaded_weight: torch.Tensor,
shard_world_size: int | None = None,
shard_rank: int | None = None,
) -> None:
if shard_world_size is None:
shard_world_size = get_tensor_model_parallel_world_size()
if shard_rank is None:
shard_rank = get_tensor_model_parallel_rank()
shard_size = loaded_weight.shape[0] // shard_world_size
shard = slice(shard_rank * shard_size, (shard_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard])
def _forward(
self,
x: torch.Tensor,
) -> torch.Tensor:
orig_dtype = x.dtype
x = x.to(torch.float32)
variance = x.pow(2).mean(dim=-1, keepdim=True, dtype=torch.float32)
if self.tp_world > 1:
variance = _all_reduce_variance(variance) / self.tp_world
x = x * torch.rsqrt(variance + self.variance_epsilon)
x = (x * self.weight).to(orig_dtype)
return x
def forward(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
assert residual is None, "RMSNorm does not support residual connection."
return self._forward(x)
@staticmethod
def forward_qk(
q_norm: "MiniMaxText01RMSNormTP",
k_norm: "MiniMaxText01RMSNormTP",
q: torch.Tensor,
k: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
orig_dtype = q.dtype
q = q.to(torch.float32)
k = k.to(torch.float32)
q_var = q.pow(2).mean(dim=-1, keepdim=True)
k_var = k.pow(2).mean(dim=-1, keepdim=True)
q = q * torch.rsqrt(q_var + q_norm.variance_epsilon) * q_norm.weight
k = k * torch.rsqrt(k_var + k_norm.variance_epsilon) * k_norm.weight
q = q.to(orig_dtype)
k = k.to(orig_dtype)
return q, k
@staticmethod
def forward_qkv(
q_norm: "MiniMaxText01RMSNormTP",
k_norm: "MiniMaxText01RMSNormTP",
qkv: torch.Tensor,
q_size: int,
kv_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
assert qkv.ndim == 2
assert q_norm.variance_epsilon == k_norm.variance_epsilon
# Case 0: tp_size=1
if get_tensor_model_parallel_world_size() == 1:
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
q, k = MiniMaxText01RMSNormTP.forward_qk(q_norm, k_norm, q, k)
return q, k, v
# Case : tp_size>1
q, k = torch.ops.vllm.minimax_qk_norm_fusion(
qkv,
q_norm.weight,
k_norm.weight,
q_size,
kv_size,
q_norm.tp_rank,
q_norm.tp_world,
q_norm.variance_epsilon,
q_norm.workspace,
)
_, _, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
return q, k, v