[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,271 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select
from vllm.model_executor.layers.indexer_topk import (
deep_select_topk,
get_deep_select_stride_requirement,
is_deep_select_supported,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
@triton.jit
def _max_with_nan(a, b):
return tl.maximum(a, b, propagate_nan=tl.PropagateNan.ALL)
@triton.jit(do_not_specialize=["width", "nblocks"])
def _block_scores_kernel(
logits,
starts,
ends,
scores,
valid_blocks,
stride_row,
stride_col,
stride_start,
stride_end,
stride_scores,
width,
nblocks,
BLOCK_SIZE: tl.constexpr,
HAS_STARTS: tl.constexpr,
ROW_REPEAT: tl.constexpr,
TILE: tl.constexpr,
FULL_WIDTH: tl.constexpr,
):
# Writes each row's valid block count; FULL_WIDTH also scores the -inf tail.
row = tl.program_id(0).to(tl.int64)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
end = tl.load(ends + row // ROW_REPEAT * stride_end)
col_end = tl.minimum(end, width)
num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE)
if tl.program_id(1) == 0:
tl.store(valid_blocks + row, num_valid)
num_blocks = nblocks if FULL_WIDTH else num_valid
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
num_tiles = tl.cdiv(num_blocks, TILE)
for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)):
blocks = tile_id * TILE + tl.arange(0, TILE)
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(offsets[None, :] < BLOCK_SIZE) & (cols < col_end),
other=-float("inf"),
).to(tl.float32)
reduced = tl.reduce(values, 1, _max_with_nan)
# Pin the newest block. DeepSelect leaves NaN rows partially written,
# so also pin NaN blocks (torch.topk ranks NaN first too).
pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE)
if not FULL_WIDTH:
pinned |= reduced != reduced
reduced = tl.where(pinned, float("inf"), reduced)
tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks)
@triton.jit(do_not_specialize=["k"])
def _store_candidates_kernel(
values,
indices,
output,
out_stride_row,
out_stride_col,
k,
OUT_K: tl.constexpr,
TILE: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
value = tl.load(values + row * k + cols, cols < k, other=-float("inf"))
index = tl.load(indices + row * k + cols, cols < k, other=-1)
# NaN scores can occur during warmup; only -inf denotes padding.
tl.store(
output + row * out_stride_row + cols * out_stride_col,
tl.where(value != -float("inf"), index, -1),
cols < OUT_K,
)
@triton.jit(do_not_specialize=["width", "nblocks"])
def _candidate_flags_kernel(
candidates,
starts,
flags,
stride_row,
stride_col,
stride_start,
width,
nblocks,
BLOCK_SIZE: tl.constexpr,
K: tl.constexpr,
HAS_STARTS: tl.constexpr,
ROW_REPEAT: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
offsets = tl.arange(0, 1024)
for tile in range(tl.cdiv(nblocks + 1, 1024)):
slots = tile * 1024 + offsets
tl.store(flags + row * (nblocks + 1) + slots, 0, slots <= nblocks)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
cols = tl.arange(0, triton.next_power_of_2(K))
block = tl.load(
candidates + row * stride_row + cols * stride_col, cols < K, other=-1
).to(tl.int64)
# Preserve the packed-column clamp for candidates beyond the logits width.
block = tl.where(start + block * BLOCK_SIZE >= width, nblocks, block)
tl.debug_barrier()
tl.store(flags + row * (nblocks + 1) + block, 1, (cols < K) & (block >= 0))
@triton.jit(do_not_specialize=["width", "nblocks"])
def _mask_candidates_kernel(
logits,
starts,
ends,
flags,
stride_row,
stride_col,
stride_start,
stride_end,
width,
nblocks,
BLOCK_SIZE: tl.constexpr,
HAS_STARTS: tl.constexpr,
ROW_REPEAT: tl.constexpr,
TILE: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
end = tl.load(ends + row // ROW_REPEAT * stride_end)
valid = (cols >= start) & (cols < end) & (cols < width)
block = (cols - start) // BLOCK_SIZE
keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0)
edge = tl.load(flags + row * (nblocks + 1) + nblocks)
keep = (keep != 0) | ((cols == width - 1) & (edge != 0))
tl.store(
logits + row * stride_row + cols * stride_col,
-float("inf"),
(cols < width) & ~(valid & keep),
)
def select_candidate_blocks(
logits: torch.Tensor,
row_ks: torch.Tensor | None,
row_ke: torch.Tensor,
topk_blocks: int,
block_size: int,
out: torch.Tensor,
row_repeat: int = 1,
) -> None:
"""Select local block IDs by maximum score, pinning each row's newest block.
Row bounds are in packed column space; absent starts mean zero.
Decode rows share bounds in groups of ``row_repeat``. Output is -1 padded.
"""
assert logits.is_cuda
rows, width = logits.shape
if not rows:
return
if not width:
out.fill_(-1)
return
nblocks = triton.cdiv(width, block_size)
# Logits are max_model_len wide (CUDA-graph static) but rows are shorter:
# DeepSelect stops at each row's valid block count. Order does not matter.
use_deep_select = has_deep_select() and nblocks >= topk_blocks
if use_deep_select:
in_align, out_align = get_deep_select_stride_requirement()
padded = round_up(nblocks, in_align // logits.element_size())
scores = logits.new_empty((rows, padded))[:, :nblocks]
use_deep_select = (
out.stride(1) == 1
and out.stride(0) * out.element_size() % out_align == 0
and is_deep_select_supported(scores, topk_blocks)
)
else:
scores = logits.new_empty((rows, nblocks))
valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device)
_block_scores_kernel[(rows, 32)](
logits,
row_ks,
row_ke,
scores,
valid_blocks,
*logits.stride(),
row_ks.stride(0) if row_ks is not None else 0,
row_ke.stride(0),
scores.stride(0),
width,
nblocks,
block_size,
row_ks is not None,
row_repeat,
TILE=128,
FULL_WIDTH=not use_deep_select,
)
if use_deep_select:
deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out)
return
# Keep the existing top-k tie behavior.
top = scores.topk(min(topk_blocks, nblocks), dim=-1)
_store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))](
top.values,
top.indices,
out,
*out.stride(),
top.values.shape[1],
topk_blocks,
256,
)
def apply_candidate_mask(
logits: torch.Tensor,
row_ks: torch.Tensor | None,
row_ke: torch.Tensor,
candidate_blocks: torch.Tensor,
block_size: int,
row_repeat: int = 1,
) -> None:
"""Mask packed logits outside causal bounds and request-local candidates."""
assert logits.is_cuda
rows, width = logits.shape
if not rows or not width:
return
nblocks = triton.cdiv(width, block_size)
flags = torch.empty((rows, nblocks + 1), device=logits.device, dtype=torch.uint8)
start_stride = row_ks.stride(0) if row_ks is not None else 0
_candidate_flags_kernel[(rows,)](
candidate_blocks,
row_ks,
flags,
*candidate_blocks.stride(),
start_stride,
width,
nblocks,
block_size,
candidate_blocks.shape[1],
row_ks is not None,
row_repeat,
)
_mask_candidates_kernel[(rows, triton.cdiv(width, 1024))](
logits,
row_ks,
row_ke,
flags,
*logits.stride(),
start_stride,
row_ke.stride(0),
width,
nblocks,
block_size,
row_ks is not None,
row_repeat,
1024,
)
@@ -1,498 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepGEMM sparse MQA-logits path for the DeepSeek V4.1 two-level indexer.
The candidate-source indexer publishes the top candidate blocks (in units of
``candidate_block_size`` compressed positions). Instead of scoring every KV
position and masking the dense logits down to those blocks, this path scores
only the candidate tokens with DeepGEMM's sparse MQA logits kernels
(``fp8_fp4_(paged_)sparse_mqa_logits``, DeepGEMM >= 2.8, SM100 only), runs
DeepSelect's top-k directly on the bf16 logits they produce, and remaps the
selected columns back to request-local positions.
Only the MXFP4 indexer cache is supported: the kernels require UE8M0-packed
(granularity-32) Q/KV scales, which is exactly the MXFP4 cache layout. The
FP8 indexer cache stores one fp32 scale per quant block instead.
The functions here take caller-owned output buffers (``sparse_indices``,
``end``, ``col_indices``) so the hot path allocates nothing; the indexer
metadata builder owns them.
"""
import importlib.util
import torch
from vllm.model_executor.layers.indexer_topk import (
deep_select_topk,
get_deep_select_stride_requirement,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import (
fp8_fp4_paged_sparse_mqa_logits,
fp8_fp4_sparse_mqa_logits,
get_paged_sparse_mqa_logits_metadata,
get_sparse_mqa_logits_metadata,
)
# Sparse-block sizes (in tokens) supported by the DeepGEMM kernels.
SPARSE_BLOCK_KV_CHOICES = (16, 8)
def has_deep_select() -> bool:
"""Whether the DeepSelect top-k extension is built and this GPU runs it."""
return (
importlib.util.find_spec("vllm._deepselect_C") is not None
and current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
)
def check_deep_select_layout(num_sparse_cols: int, topk_tokens: int) -> None:
"""Raise unless DeepSelect accepts the sparse logits layout.
The sparse kernels return contiguous bf16 ``[rows, num_sparse_cols]``
logits and the top-k writes int32 ``[rows, topk_tokens]`` indices;
DeepSelect needs both row strides aligned (1024B / 32B on SM100).
"""
in_align, out_align = get_deep_select_stride_requirement()
logits_row_bytes = num_sparse_cols * torch.bfloat16.itemsize
if num_sparse_cols >= 2**23 or logits_row_bytes % in_align != 0:
raise ValueError(
f"DeepSelect needs the sparse logits row ({num_sparse_cols} bf16 "
f"columns, {logits_row_bytes} B) to be {in_align}B-aligned."
)
if topk_tokens > 4096 or (topk_tokens * torch.int32.itemsize) % out_align != 0:
raise ValueError(
f"DeepSelect needs topk <= 4096 with a {out_align}B-aligned int32 "
f"output row, got topk={topk_tokens}."
)
_INT32_BIG = tl.constexpr(2**31 - 1)
@triton.jit(do_not_specialize_on_alignment=["end_ptr"])
def _expand_candidates_kernel(
cand_ptr,
cand_stride,
ks_ptr,
ks_stride,
ke_ptr,
ke_stride,
si_ptr,
si_stride,
end_ptr,
end_stride,
K: tl.constexpr,
S: tl.constexpr,
RATIO: tl.constexpr,
SBK: tl.constexpr,
):
"""One program per row: expand + filter + sort candidate blocks.
Each row's valid sparse blocks come out sorted ascending (no dedup pass:
production candidates are unique, and the kernel tolerates repeats),
padded by repeating the last valid block — DeepGEMM's padding convention.
``end`` counts valid sparse-token columns; valid columns form a prefix
because block ids are sorted.
"""
row = tl.program_id(0)
ks = tl.load(ks_ptr + row * ks_stride)
ke = tl.load(ke_ptr + row * ke_stride)
cols = tl.arange(0, K)
cand = tl.load(cand_ptr + row * cand_stride + cols) # K is pow2, no mask
if RATIO == 2:
cand = tl.interleave(cand * RATIO, cand * RATIO + 1)
# Keep only blocks inside the row's KV range [0, ke - ks); the lower
# bound is free since candidates are non-negative.
kv_len = tl.maximum(ke - ks, 0)
max_block = (kv_len + SBK - 1) // SBK
valid = (cand >= 0) & (cand < max_block)
vals = tl.where(valid, cand, _INT32_BIG)
vals = tl.sort(vals)
# Anchor at the row's ks rounded down to a sparse-block boundary
# (identity for the paged path, where ks == 0).
base = ks // SBK
si = tl.where(vals != _INT32_BIG, vals, 0) + base
n_valid = tl.sum(valid.to(tl.int32))
last_rel = tl.max(tl.where(valid, cand, -1))
pad_val = tl.where(n_valid > 0, last_rel + base, base)
tl.store(
si_ptr + row * si_stride + tl.arange(0, S),
tl.where(tl.arange(0, S) < n_valid, si, pad_val),
)
# Valid sparse-token column count; only the last block can be partial.
# Absolute start of the last block: (base + last_rel) * SBK + ks % SBK,
# which is exactly last_rel * SBK + ks.
last_start = last_rel * SBK + ks
fill = tl.minimum(tl.maximum(ke - last_start, 1), SBK)
end = tl.where(n_valid > 0, (n_valid - 1) * SBK + fill, 0)
tl.store(end_ptr + row * end_stride, end)
@triton.jit
def _sparse_topk_remap_kernel(
col_ptr,
col_stride,
si_ptr,
si_stride,
ks_ptr,
ks_stride,
out_ptr,
out_stride,
k,
width,
SBK: tl.constexpr,
K_POW2: tl.constexpr,
):
"""Remap sparse top-k columns to request-local positions.
Column ``j * SBK + o`` scores the token at
``(si[row, j] - ks // SBK) * SBK + o``; columns outside ``[0, width)``
(including DeepSelect's sentinel for NaN rows) become -1.
"""
row = tl.program_id(0)
ks_base = tl.load(ks_ptr + row * ks_stride) // SBK
cols = tl.arange(0, K_POW2)
c = tl.load(col_ptr + row * col_stride + cols, mask=cols < k, other=-1)
valid = (c >= 0) & (c < width)
cc = tl.where(valid, c, 0)
blocks = tl.load(si_ptr + row * si_stride + cc // SBK)
pos = (blocks - ks_base) * SBK + cc % SBK
tl.store(out_ptr + row * out_stride + cols, tl.where(valid, pos, -1), mask=cols < k)
def pick_sparse_block_kv(candidate_block_size: int) -> int:
"""Largest sparse-block size (tokens) dividing the candidate block size."""
for sbk in SPARSE_BLOCK_KV_CHOICES:
if candidate_block_size % sbk == 0:
return sbk
raise ValueError(
f"candidate_block_size={candidate_block_size} is not a multiple of "
f"any supported sparse_block_kv {SPARSE_BLOCK_KV_CHOICES}."
)
def candidate_blocks_to_sparse_indices(
candidate_blocks: torch.Tensor,
row_ks: torch.Tensor,
row_ke: torch.Tensor,
candidate_block_size: int,
sparse_block_kv: int,
out: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Expand request-local candidate blocks into DeepGEMM sparse block ids.
Args:
candidate_blocks: [rows, K] int32 request-local candidate block ids
(-1 padded), in units of ``candidate_block_size`` positions. K
must be a power of two (the in-kernel sort); rows may be strided.
row_ks: [rows] int32 per-row K range start; bounds are in the same
(packed-workspace) coordinates the sparse kernel iterates over.
Pass zeros for the paged path, whose blocks are context-relative.
row_ke: [rows] int32 per-row K range end, in the same coordinates as
``row_ks``.
candidate_block_size: Positions per candidate block.
sparse_block_kv: Positions per sparse block (8 or 16).
out: Optional ``(sparse_indices, end)`` buffers to write into,
shaped [rows, K * ratio] and [rows], both int32.
Returns:
sparse_indices: [rows, K * ratio] int32 block ids for DeepGEMM,
sorted ascending per row with the valid prefix first (unique in
production, where candidates are unique) and the remaining slots
repeating the last valid block (the kernel's padding convention).
With the unaligned-ks variant, block ``i`` starts at
``i * sparse_block_kv + ks % sparse_block_kv``.
end: [rows] int32 count of valid sparse-token columns; valid columns
always form a prefix because the block ids are sorted.
"""
rows, num_candidates = candidate_blocks.shape
ratio = candidate_block_size // sparse_block_kv
assert ratio * sparse_block_kv == candidate_block_size and ratio in (1, 2), (
candidate_block_size,
sparse_block_kv,
)
assert num_candidates & (num_candidates - 1) == 0, num_candidates
num_sparse = num_candidates * ratio
if out is None:
device = candidate_blocks.device
out = (
torch.empty(rows, num_sparse, dtype=torch.int32, device=device),
torch.empty(rows, dtype=torch.int32, device=device),
)
sparse_indices, end = out
assert sparse_indices.shape == (rows, num_sparse) and end.shape == (rows,)
assert candidate_blocks.stride(1) == 1 and sparse_indices.stride(1) == 1
for t in (candidate_blocks, row_ks, row_ke, sparse_indices, end):
assert t.dtype == torch.int32, t.dtype
_expand_candidates_kernel[(rows,)](
candidate_blocks,
candidate_blocks.stride(0),
row_ks,
row_ks.stride(0),
row_ke,
row_ke.stride(0),
sparse_indices,
sparse_indices.stride(0),
end,
end.stride(0),
K=num_candidates,
S=num_sparse,
RATIO=ratio,
SBK=sparse_block_kv,
num_warps=16,
)
return sparse_indices, end
def sparse_topk_remap(
logits: torch.Tensor,
sparse_indices: torch.Tensor,
end: torch.Tensor,
row_ks: torch.Tensor,
sparse_block_kv: int,
topk_tokens: int,
topk_indices: torch.Tensor,
*,
col_indices: torch.Tensor,
) -> None:
"""Row top-k over sparse logits, remapped to request-local positions.
Column ``j * sparse_block_kv + o`` of ``logits`` scores the token at
request-local position
``(sparse_indices[row, j] - ks // sparse_block_kv) * sparse_block_kv + o``.
DeepSelect consumes the bf16 logits as they come out of the sparse
kernels; the valid columns form a prefix (block ids are sorted), so the
per-row ``end`` bounds drive it directly. Slots beyond a row's valid
count come back as -1.
Args:
logits: [rows, width] bf16 sparse logits, as produced by the sparse
kernels.
sparse_indices: [rows, S] int32 sparse block ids backing ``logits``.
end: [rows] int32 per-row count of valid logit columns.
row_ks: [rows] int32 per-row K range start.
sparse_block_kv: Positions per sparse block (8 or 16).
topk_tokens: Number of tokens to keep per row.
topk_indices: [rows, topk_tokens] int32 output buffer.
col_indices: [rows, topk_tokens] int32 scratch for the sparse-column
top-k result (row stride 32B-aligned for DeepSelect).
"""
rows, width = logits.shape
assert logits.dtype == torch.bfloat16 and width >= topk_tokens
assert end.dtype == torch.int32 and end.shape == (rows,)
cols = col_indices[:rows]
deep_select_topk(logits, topk_tokens, end=end, output_idx=cols)
_sparse_topk_remap_kernel[(rows,)](
cols,
cols.stride(0),
sparse_indices,
sparse_indices.stride(0),
row_ks,
row_ks.stride(0),
topk_indices,
topk_indices.stride(0),
topk_tokens,
width,
SBK=sparse_block_kv,
K_POW2=triton.next_power_of_2(topk_tokens),
)
def sparse_mqa_logits_prefill_chunk(
q: torch.Tensor,
q_scale: torch.Tensor,
k_quant: torch.Tensor,
k_scale: torch.Tensor,
weights: torch.Tensor,
cu_seqlen_ks: torch.Tensor,
cu_seqlen_ke: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
sparse_block_kv: int,
topk_tokens: int,
topk_indices: torch.Tensor,
*,
sparse_indices: torch.Tensor,
end: torch.Tensor,
col_indices: torch.Tensor,
kernel_metadata: torch.Tensor | None = None,
) -> torch.Tensor:
"""Sparse-MQA logits + top-k for one prefill chunk (packed KV workspace).
Args:
q: [rows, H, D] packed Q values (MXFP4 viewed as int8).
q_scale: [rows, H] int32 packed UE8M0 Q scales.
k_quant: [total_kv, D] packed K workspace (MXFP4 viewed as int8).
k_scale: [total_kv] int32 packed UE8M0 K scales.
weights: [rows, H] bf16 per-head weights; the sparse kernels take
bf16 and do not fold the Q scale in.
cu_seqlen_ks: [rows] int32 per-token K start bounds in the packed
workspace.
cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed
workspace.
candidate_blocks: [rows, K] int32 request-local candidate block ids.
candidate_block_size: Positions per candidate block.
sparse_block_kv: Positions per sparse block (8 or 16).
topk_tokens: Number of tokens to keep per row.
topk_indices: [rows, topk_tokens] output buffer.
sparse_indices: Caller-owned scratch, see
`candidate_blocks_to_sparse_indices`.
end: Caller-owned scratch, see `candidate_blocks_to_sparse_indices`.
col_indices: Caller-owned scratch, see `sparse_topk_remap`.
kernel_metadata: DeepGEMM schedule from a previous call with the same
candidates and bounds (i.e. another indexer layer in the same
step). When given, the candidate expansion is skipped and
``sparse_indices``/``end`` are assumed to be up to date.
Returns:
The DeepGEMM schedule metadata used, for reuse by later layers.
"""
assert weights.dtype == torch.bfloat16, weights.dtype
if kernel_metadata is None:
candidate_blocks_to_sparse_indices(
candidate_blocks,
cu_seqlen_ks,
cu_seqlen_ke,
candidate_block_size,
sparse_block_kv,
out=(sparse_indices, end),
)
kernel_metadata = get_sparse_mqa_logits_metadata(
cu_seqlen_ks,
cu_seqlen_ke,
k_quant.shape[0],
sparse_indices,
q.dtype,
sparse_block_kv,
)
logits = fp8_fp4_sparse_mqa_logits(
(q, q_scale),
(k_quant, k_scale),
weights,
kernel_metadata,
sparse_indices.shape[1],
sparse_block_kv,
)
sparse_topk_remap(
logits,
sparse_indices,
end,
cu_seqlen_ks,
sparse_block_kv,
topk_tokens,
topk_indices,
col_indices=col_indices,
)
return kernel_metadata
def sparse_mqa_logits_paged_decode(
q: torch.Tensor,
q_scale: torch.Tensor,
kv_cache: torch.Tensor,
weights: torch.Tensor,
context_lens: torch.Tensor,
block_table: torch.Tensor,
row_indices: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
sparse_block_kv: int,
topk_tokens: int,
topk_indices: torch.Tensor,
*,
row_ks: torch.Tensor,
sparse_indices: torch.Tensor,
end: torch.Tensor,
col_indices: torch.Tensor,
kernel_metadata: torch.Tensor | None = None,
) -> torch.Tensor:
"""Sparse-MQA logits + top-k for paged decode rows.
Every query is one row (spec-decode rows are flattened by the metadata
builder), so ``block_table`` is already per row.
Args:
q: [rows, 1, H, D] packed Q values (MXFP4 viewed as int8).
q_scale: [rows, 1, H] int32 packed UE8M0 Q scales.
kv_cache: 4D [num_pages, page_kv, 1, head_bytes] uint8 fused cache
view; the page stride must be 256B-aligned.
weights: [rows, H] bf16 per-head weights.
context_lens: [rows] int32 per-row (compressed) context lengths.
block_table: [rows, P] int32 per-row page table, ``stride(-1) == 1``.
row_indices: [rows] int32 row -> request map (pairing only affects
scheduling).
candidate_blocks: [rows, K] int32 candidate block ids.
candidate_block_size: Positions per candidate block.
sparse_block_kv: Positions per sparse block (8 or 16).
topk_tokens: Number of tokens to keep per row.
topk_indices: [rows, topk_tokens] output buffer.
row_ks: [rows] int32 zeros (paged blocks are context-relative).
sparse_indices: Caller-owned scratch.
end: Caller-owned scratch.
col_indices: Caller-owned scratch.
kernel_metadata: See `sparse_mqa_logits_prefill_chunk`.
Returns:
The DeepGEMM schedule metadata used, for reuse by later layers.
"""
rows = q.shape[0]
page_kv = kv_cache.shape[1]
assert weights.dtype == torch.bfloat16, weights.dtype
assert kv_cache.stride(0) % 256 == 0, (
"paged cache page stride must be 256B-aligned: "
f"shape={tuple(kv_cache.shape)} stride={kv_cache.stride()}"
)
assert page_kv % sparse_block_kv == 0, (page_kv, sparse_block_kv)
assert block_table.shape[0] == rows and block_table.stride(-1) == 1
assert context_lens.shape == (rows,) and context_lens.is_contiguous()
if kernel_metadata is None:
candidate_blocks_to_sparse_indices(
candidate_blocks,
row_ks,
context_lens,
candidate_block_size,
sparse_block_kv,
out=(sparse_indices, end),
)
kernel_metadata = get_paged_sparse_mqa_logits_metadata(
context_lens,
block_table,
row_indices,
page_kv,
sparse_indices,
q.dtype,
sparse_block_kv,
)
logits = fp8_fp4_paged_sparse_mqa_logits(
(q, q_scale),
kv_cache,
weights,
kernel_metadata,
sparse_indices.shape[1],
sparse_block_kv,
)
sparse_topk_remap(
logits,
sparse_indices,
end,
row_ks,
sparse_block_kv,
topk_tokens,
topk_indices,
col_indices=col_indices,
)
return kernel_metadata
File diff suppressed because it is too large Load Diff
@@ -1,231 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Replicated input embedding + its fused gather/norm kernels.
Groups the ``VLLM_REPLICATE_EMBED`` path in one place: the embedding factory,
the predicate that says whether the fusions apply, and the two Triton fusions
the full on-rank table unlocks --
* ``fused_embed_norm``: gather + a chained RMSNorm (e.g. the first decoder
layer's ``input_layernorm``), and
* ``fused_embed_eh_norm``: gather + pos-0 zeroing + enorm/hnorm + cat, the
embed/previous-hidden input norm for a speculative (MTP/eagle) depth layer
(the replicated-table analogue of the model-local ``fused_eh_norm``, which
takes precomputed embeds).
Self-contained (no model-local imports) so it can live under ``layers/``.
"""
import torch
import vllm.envs as envs
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _rms_norm(x, w, eps, HIDDEN_SIZE: tl.constexpr):
x = x.to(tl.float32)
mean_sq = tl.sum(x * x, axis=0) / HIDDEN_SIZE
rrms = tl.rsqrt(mean_sq + eps)
w = w.to(tl.float32)
return (x * rrms) * w
def make_input_embedding(
num_embeddings: int,
embedding_dim: int,
*,
params_dtype: torch.dtype | None = None,
quant_config=None,
prefix: str = "",
tie_word_embeddings: bool = False,
) -> VocabParallelEmbedding:
"""Input token embedding with an optional replicated escape hatch.
``VLLM_REPLICATE_EMBED=1`` builds the embedding with ``disable_tp``: the full
table lives on every rank and the lookup is a local gather with no mask and
no all-reduce, which unlocks the fused gather+norm path. The cost is a full
table per rank at TP>1 (no extra memory at TP=1, where vocab-parallel is
already unsharded). A replicated, unsharded table cannot be tied to a
vocab-parallel ``ParallelLMHead``, so tied word embeddings are rejected at
TP>1 (at TP=1 ``disable_tp`` is a no-op and tying still works).
"""
disable_tp = envs.VLLM_REPLICATE_EMBED
if disable_tp and tie_word_embeddings:
assert get_tensor_model_parallel_world_size() == 1, (
"VLLM_REPLICATE_EMBED is unsupported with tied word embeddings "
"(the replicated table cannot tie to a vocab-parallel lm_head)"
)
return VocabParallelEmbedding(
num_embeddings,
embedding_dim,
params_dtype=params_dtype,
quant_config=quant_config,
prefix=prefix,
disable_tp=disable_tp,
)
def has_full_vocab_on_rank(embedding: torch.nn.Module) -> bool:
"""Whether ``embedding.weight`` is the whole vocab as a plain [V, H] table.
The fused gather kernels index the table directly, so they need every row
on-rank (``disable_tp``, or any TP=1 run) and an unquantized weight.
"""
return getattr(embedding, "tp_size", 0) == 1 and isinstance(
getattr(embedding, "quant_method", None), UnquantizedEmbeddingMethod
)
@triton.jit
def _fused_embed_norm_kernel(
ids_ptr, # [T] token ids
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
table_stride_0,
out_ptr, # [T, H] gathered embedding (the residual stream)
normed_ptr, # [T, H] rmsnorm(out, chain_w) (HAS_NORM only)
chain_w_ptr, # [H] next norm weight (HAS_NORM only)
eps,
H: tl.constexpr,
BLOCK: tl.constexpr,
HAS_NORM: tl.constexpr,
):
tok = tl.program_id(0).to(tl.int64)
off = tl.arange(0, BLOCK)
mask = off < H
row = tl.load(ids_ptr + tok).to(tl.int64)
x = tl.load(table_ptr + row * table_stride_0 + off, mask=mask, other=0.0)
tl.store(out_ptr + tok * H + off, x, mask=mask)
if HAS_NORM:
w = tl.load(chain_w_ptr + off, mask=mask)
y = _rms_norm(x, w, eps, H).to(normed_ptr.dtype.element_ty)
tl.store(normed_ptr + tok * H + off, y, mask=mask)
# Base model fusion
def fused_embed_norm(
input_ids: torch.Tensor,
embed_table: torch.Tensor,
chain_weight: torch.Tensor | None = None,
eps: float = 0.0,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Fused embedding row gather (``embed_table[input_ids]``).
Requires the full vocab on-rank (replicated embedding). When
``chain_weight`` is given, also emits ``rmsnorm(gathered, chain_weight)``
(the first decoder layer's ``input_layernorm``) as a second output in the
same launch, so the returned pair is ``(residual, normed_input)``. Bit-exact
vs a plain gather followed by an ``RMSNorm``.
"""
assert embed_table.ndim == 2, embed_table.shape
ids = input_ids.view(-1)
(t,) = ids.shape
h = embed_table.shape[1]
if chain_weight is not None:
assert chain_weight.shape == (h,), (chain_weight.shape, h)
out = torch.empty((t, h), dtype=embed_table.dtype, device=embed_table.device)
normed = torch.empty_like(out) if chain_weight is not None else None
if t > 0:
block = triton.next_power_of_2(h)
_fused_embed_norm_kernel[(t,)](
ids,
embed_table,
embed_table.stride(0),
out,
normed if normed is not None else out,
chain_weight if chain_weight is not None else embed_table,
eps,
h,
block,
HAS_NORM=chain_weight is not None,
num_warps=min(32, max(4, block // 512)),
)
if normed is not None:
return out, normed
return out
@triton.jit
def _fused_embed_eh_norm_kernel(
pos_ptr,
ids_ptr, # [T] token ids
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
table_stride,
prev_ptr, # [T, H] previous-step hidden
prev_stride,
enorm_w_ptr,
hnorm_w_ptr,
eps,
out_ptr, # [T, 2H]
out_stride,
H: tl.constexpr,
BLOCK: tl.constexpr,
):
"""MTP input fusion with a folded embedding gather: gather
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
cat. Requires the full table on-rank (replicated embedding)."""
tok = tl.program_id(0)
off = tl.arange(0, BLOCK)
mask = off < H
pos = tl.load(pos_ptr + tok)
row = tl.load(ids_ptr + tok).to(tl.int64)
e = tl.load(table_ptr + row * table_stride + off, mask=mask, other=0.0)
e = tl.where(pos == 0, 0.0, e.to(tl.float32))
ew = tl.load(enorm_w_ptr + off, mask=mask)
e_normed = _rms_norm(e, ew, eps, H)
tl.store(out_ptr + tok * out_stride + off, e_normed, mask=mask)
p = tl.load(prev_ptr + tok * prev_stride + off, mask=mask, other=0.0)
hw = tl.load(hnorm_w_ptr + off, mask=mask)
p_normed = _rms_norm(p, hw, eps, H)
tl.store(out_ptr + tok * out_stride + H + off, p_normed, mask=mask)
# MTP fusion
def fused_embed_eh_norm(
positions: torch.Tensor,
input_ids: torch.Tensor,
embed_table: torch.Tensor,
previous_hidden: torch.Tensor,
enorm_w: torch.Tensor,
hnorm_w: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""Fused ``cat([enorm(masked embed_table[ids]), hnorm(prev_hidden)])`` -> [N, 2H].
Folds the embedding row gather into the MTP eh-norm launch; requires the full
table on-rank (replicated embedding). Bit-exact vs gathering ``embed_table[
input_ids]`` and passing it to the model-local ``fused_eh_norm``.
"""
assert previous_hidden.ndim == 2 and embed_table.ndim == 2
n, h = previous_hidden.shape
assert positions.shape == (n,) and input_ids.view(-1).shape == (n,)
assert embed_table.shape[1] == h, (embed_table.shape, h)
assert enorm_w.shape == (h,) and hnorm_w.shape == (h,)
out = torch.empty(
n, 2 * h, dtype=previous_hidden.dtype, device=previous_hidden.device
)
_fused_embed_eh_norm_kernel[(n,)](
positions,
input_ids,
embed_table,
embed_table.stride(0),
previous_hidden,
previous_hidden.stride(0),
enorm_w,
hnorm_w,
eps,
out,
out.stride(0),
h,
triton.next_power_of_2(h),
)
return out
@@ -1,391 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused MoE Triton kernels."""
import functools
import json
import os
import torch
from vllm.logger import init_logger
from vllm.utils.platform_utils import get_device_name_as_file_name
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
# Valid tiles for the aiter a16w-mix int4 kernels (aiter#4646): stage1 requires
# tile_n in {64,128} and tile_k in {128,256} (k_wave=1); stage2 requires tile_n2==128
# and tile_k2 in {128,256}. The pre-#4646 defaults (tile_k=512/64, tile_n2=256) are no
# longer registered and make the port emit an empty-fragment kernel.
_FLYDSL_MOE_DEFAULT_CONFIG = {
1: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
2: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
4: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
8: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
16: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
24: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
32: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
48: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
64: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
128: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
256: {"tile_m": 16, "tile_n": 128, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
512: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
1024: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
2048: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
4096: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
8192: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
}
# aiter#4646's a16w-mix int4 kernels only register a fixed tile set. Tiles from before
# the bump (default config or tuned JSONs) carry tile_k=512/64 and tile_n2=256, which
# emit an empty-fragment kernel (IndexError at emit / wrong output). Snap any resolved
# tiles to the nearest registered value so a stale tuned config can't crash the kernel.
_FLYDSL_INT4_TILE_M = (16, 32, 64, 128)
_FLYDSL_INT4_TILE_N1 = (64, 128)
_FLYDSL_INT4_TILE_K = (128, 256)
def _snap_flydsl_int4_tiles(tile_m, tile_n, tile_k, tile_n2, tile_k2):
def _nearest(v, allowed):
return min(allowed, key=lambda a: (abs(a - v), a))
snapped = (
_nearest(tile_m, _FLYDSL_INT4_TILE_M),
_nearest(tile_n, _FLYDSL_INT4_TILE_N1),
_nearest(tile_k, _FLYDSL_INT4_TILE_K),
128, # stage2 N tile is only registered at 128
_nearest(tile_k2, _FLYDSL_INT4_TILE_K),
)
if snapped != (tile_m, tile_n, tile_k, tile_n2, tile_k2):
logger.warning_once(
"FlyDSL int4 MoE tiles %s are not registered post-aiter#4646; "
"using %s. Retune the config for the a16w-mix kernel.",
(tile_m, tile_n, tile_k, tile_n2, tile_k2),
snapped,
scope="local",
)
return snapped
def moe_sorting(
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
*,
num_experts: int,
model_dim: int,
block_m: int,
):
# Imported lazily so aiter's import-time sort-backend env vars are read
# when this is called, not whenever this module happens to be imported.
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
topk_ids_i32 = topk_ids.to(torch.int32)
topk_w_f32 = topk_weights.to(torch.float32)
sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = (
aiter_moe_sorting(
topk_ids_i32,
topk_w_f32,
num_experts,
model_dim,
torch.float16,
block_m,
)
)
if num_valid_ids.numel() > 1:
num_valid_ids = num_valid_ids[:1].contiguous()
return sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids
def build_routing_buffers(
*,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
num_experts: int,
model_dim: int,
tile_m: int,
):
res = moe_sorting(
topk_ids,
topk_weights,
num_experts=num_experts,
model_dim=model_dim,
block_m=tile_m,
)
if res is None:
raise RuntimeError(
"aiter moe_sorting failed/unavailable; cannot build routing buffers."
)
sorted_token_ids, sorted_weights, sorted_expert_ids, num_valid_ids = res
sorted_token_ids = sorted_token_ids.contiguous()
sorted_weights = sorted_weights.contiguous()
sorted_expert_ids = sorted_expert_ids.contiguous()
sorted_size = int(sorted_token_ids.numel())
blocks = int(sorted_expert_ids.numel())
return (
sorted_token_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
sorted_size,
blocks,
)
@functools.lru_cache
def try_get_optimal_config(num_experts, inter_dim):
device_name = get_device_name_as_file_name()
json_file_name = (
f"E={num_experts},N={inter_dim},device_name={device_name},"
"dtype=int4_w4a16,backend=flydsl.json"
)
config_file_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name
)
if os.path.exists(config_file_path):
with open(config_file_path) as f:
logger.info_once(
"Using tuned FlyDSL MoE config from %s",
config_file_path,
scope="global",
)
tuned_config = json.load(f)
return {int(key): val for key, val in tuned_config.items()}
logger.warning_once(
"Using default FlyDSL MoE config. Performance might be sub-optimal! "
"Config file not found at %s",
config_file_path,
scope="local",
)
return _FLYDSL_MOE_DEFAULT_CONFIG
def fused_flydsl_moe_impl(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
num_experts: int,
inter_dim: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
topk: int = 8,
group_size: int = 32,
doweight_stage1: bool = False,
in_dtype: str = "int4_bf16",
out_dtype: str = "bf16",
scale_is_bf16: bool = True,
tile_m: int | None = None,
tile_n: int | None = None,
tile_k: int | None = None,
tile_n2: int | None = None,
tile_k2: int | None = None,
) -> torch.Tensor:
# Imported lazily -- see the comment in moe_sorting() above.
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
flydsl_a16w4_gemm2,
)
device = hidden_states.device
tokens = hidden_states.shape[0]
model_dim = hidden_states.shape[1]
tuned_config = {}
if tile_m and tile_n and tile_k and tile_n2 and tile_k2:
tuned_config["tile_m"] = tile_m
tuned_config["tile_n"] = tile_n
tuned_config["tile_k"] = tile_k
tuned_config["tile_n2"] = tile_n2
tuned_config["tile_k2"] = tile_k2
else:
tuned_config = try_get_optimal_config(num_experts, inter_dim)
tuned_config = tuned_config[
min(tuned_config.keys(), key=lambda x: abs(x - tokens))
]
out_torch_dtype = torch.bfloat16 if out_dtype == "bf16" else torch.float16
tile_m = tuned_config["tile_m"]
tile_n = tuned_config["tile_n"]
tile_k = tuned_config["tile_k"]
tile_n2 = tuned_config["tile_n2"]
tile_k2 = tuned_config["tile_k2"]
tile_m, tile_n, tile_k, tile_n2, tile_k2 = _snap_flydsl_int4_tiles(
tile_m, tile_n, tile_k, tile_n2, tile_k2
)
routing = build_routing_buffers(
topk_ids=topk_ids,
topk_weights=topk_weights,
num_experts=num_experts,
model_dim=model_dim,
tile_m=tile_m,
)
(
sorted_token_ids,
sorted_weights,
sorted_expert_ids,
num_valid_ids,
sorted_size,
blocks,
) = routing
# aiter#4646 (v0.1.20) removed compile_moe_gemm{1,2} + the flyc.compile pointer ABI
# and replaced them with the a16w-mix wrappers, which compile+launch internally
# (functools-cached) and take torch tensors directly. w_dtype="int4" is the a16wi4
# (W4A16) path. The stage-1 intermediate now lives in sorted
# [sorted_size, inter_dim] layout, and topk weighting is applied in stage-2 via
# sorted_weights (the port has no stage-1 doweight).
sorted_weights_1d = sorted_weights.view(-1).contiguous()
empty_scale = torch.empty((0,), device=device, dtype=torch.uint8)
cumsum_tensor = num_valid_ids.to(torch.int32).contiguous()
m_indices = sorted_token_ids.to(torch.int32).contiguous()
# stage1: bf16 A x int4 W1 + SiLU -> bf16 sorted intermediate.
inter_sorted = torch.empty(
(sorted_size, inter_dim), device=device, dtype=torch.bfloat16
)
w1_scale_u8 = (
w1_scale.view(torch.uint8).contiguous().view(-1)
if w1_scale is not None
else empty_scale
)
flydsl_a16w4_gemm1(
a_bf16=hidden_states.to(torch.bfloat16).contiguous(),
w1_u8=w1.view(torch.uint8).contiguous(),
w1_scale_u8=w1_scale_u8,
sorted_expert_ids=sorted_expert_ids,
cumsum_tensor=cumsum_tensor,
m_indices=m_indices,
inter_sorted_bf16=inter_sorted,
n_tokens=tokens,
NE=num_experts,
D_HIDDEN=model_dim,
D_INTER=inter_dim,
topk=topk,
tile_m=tile_m,
tile_n=tile_n,
tile_k=tile_k,
waves_per_eu=None,
act="silu",
w_dtype="int4",
w_layout="standard",
)
# stage2: bf16 inter x int4 W2 -> weighted reduce into [tokens, model_dim].
out_stage2 = torch.zeros((tokens, model_dim), device=device, dtype=out_torch_dtype)
w2_scale_u8 = (
w2_scale.view(torch.uint8).contiguous().view(-1)
if w2_scale is not None
else empty_scale
)
flydsl_a16w4_gemm2(
inter_sorted_bf16=inter_sorted,
w2_u8=w2.view(torch.uint8).contiguous(),
w2_scale_u8=w2_scale_u8,
sorted_expert_ids=sorted_expert_ids,
cumsum_tensor=cumsum_tensor,
sorted_token_ids=sorted_token_ids,
sorted_weights=sorted_weights_1d,
flat_out=out_stage2.view(-1),
M_logical=tokens,
max_sorted=sorted_size,
NE=num_experts,
D_HIDDEN=model_dim,
D_INTER=inter_dim,
topk=topk,
tile_m=tile_m,
tile_n=tile_n2,
tile_k=tile_k2,
waves_per_eu=None,
w_dtype="int4",
)
return out_stage2
def fused_flydsl_moe_impl_fake(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
num_experts: int,
inter_dim: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
topk: int = 8,
group_size: int = 32,
doweight_stage1: bool = False,
in_dtype: str = "int4_bf16",
out_dtype: str = "bf16",
scale_is_bf16: bool = True,
tile_m: int | None = None,
tile_n: int | None = None,
tile_k: int | None = None,
tile_n2: int | None = None,
tile_k2: int | None = None,
) -> torch.Tensor:
return torch.empty_like(hidden_states)
direct_register_custom_op(
op_name="fused_flydsl_moe_impl",
op_func=fused_flydsl_moe_impl,
fake_impl=fused_flydsl_moe_impl_fake,
)
def fused_flydsl_moe(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
num_experts: int,
inter_dim: int,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
w1_scale: torch.Tensor | None = None,
w2_scale: torch.Tensor | None = None,
topk: int = 8,
group_size: int = 32,
doweight_stage1: bool = False,
in_dtype: str = "int4_bf16",
out_dtype: str = "bf16",
scale_is_bf16: bool = True,
config: dict | None = None,
) -> torch.Tensor:
tile_m = None
tile_n = None
tile_k = None
tile_n2 = None
tile_k2 = None
if config is not None:
tile_m = config.get("tile_m")
tile_n = config.get("tile_n")
tile_k = config.get("tile_k")
tile_n2 = config.get("tile_n2")
tile_k2 = config.get("tile_k2")
return torch.ops.vllm.fused_flydsl_moe_impl(
hidden_states=hidden_states,
w1=w1,
w2=w2,
num_experts=num_experts,
inter_dim=inter_dim,
topk_weights=topk_weights,
topk_ids=topk_ids,
w1_scale=w1_scale,
w2_scale=w2_scale,
topk=topk,
group_size=group_size,
doweight_stage1=doweight_stage1,
in_dtype=in_dtype,
out_dtype=out_dtype,
scale_is_bf16=scale_is_bf16,
tile_m=tile_m,
tile_n=tile_n,
tile_k=tile_k,
tile_n2=tile_n2,
tile_k2=tile_k2,
)
@@ -1,682 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton input-staging kernels for DeepGEMM MegaMoE.
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
global scales, or copy them as BF16, and repack the routing top-k tensors into
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
"""
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kMxfp8Dynamic,
)
from vllm.triton_utils import tl, triton
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
@triton.jit
def _quantize_fp8_groups(
hidden,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
USE_UE8M0: tl.constexpr,
):
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
num_groups: tl.constexpr = BLOCK_K // GROUP_K
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
amax = tl.max(hidden_groups, axis=2)
amax = tl.maximum(amax, 1.0e-4)
scale = amax / 448.0
if USE_UE8M0:
scale_bits = scale.to(tl.uint32, bitcast=True)
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
tl.uint32
)
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
else:
scale_exp = scale.to(tl.uint32, bitcast=True)
quant_scale = scale
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
@triton.jit
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
offsets = tl.arange(0, NUM_BYTES)
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
@triton.jit
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
m_block_id = token_id // SHARED_BLOCK_M
m_in_block = token_id % SHARED_BLOCK_M
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
transposed_m = (
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
)
return m_block_id * aligned_block_m + transposed_m
@triton.jit
def _store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
) -> None:
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
topk_offsets = tl.arange(0, BLOCK_TOPK)
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
if is_padding is not None:
token_is_padding = tl.load(
is_padding + token_id * is_padding_stride_m,
mask=token_mask,
other=True,
)
ids = tl.load(
topk_ids
+ token_id[:, None] * topk_ids_stride_m
+ topk_offsets[None, :] * topk_ids_stride_k,
mask=topk_mask,
other=0,
).to(tl.int64)
ids = tl.where(token_is_padding[:, None], -1, ids)
tl.store(
topk_idx_out
+ token_id[:, None] * topk_idx_stride_m
+ topk_offsets[None, :] * topk_idx_stride_k,
ids,
mask=topk_mask,
)
weights = tl.load(
topk_weights
+ token_id[:, None] * topk_weights_stride_m
+ topk_offsets[None, :] * topk_weights_stride_k,
mask=topk_mask,
other=0.0,
)
weights = tl.where(token_is_padding[:, None], 0.0, weights)
tl.store(
topk_weights_out
+ token_id[:, None] * topk_weights_out_stride_m
+ topk_offsets[None, :] * topk_weights_out_stride_k,
weights,
mask=topk_mask,
)
@triton.jit(do_not_specialize=["num_tokens"])
def _prepare_megamoe_inputs_kernel(
hidden_states,
x_out,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_k: tl.constexpr,
x_sf_stride_m: tl.constexpr,
x_sf_stride_k: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
QUANTIZE: tl.constexpr,
USE_UE8M0: tl.constexpr,
) -> None:
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_block_id = tl.program_id(1)
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
hidden = tl.load(
hidden_states
+ token_id[:, None] * hidden_stride_m
+ k_offsets[None, :] * hidden_stride_k,
mask=k_mask,
other=0.0,
).to(tl.float32)
if not QUANTIZE:
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
hidden.to(x_out.dtype.element_ty),
mask=k_mask,
)
else:
num_groups: tl.constexpr = BLOCK_K // GROUP_K
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
)
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
fp8,
mask=k_mask,
)
if USE_UE8M0:
packed_scale = _pack_bytes(scale_exp, num_groups)
tl.store(
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
packed_scale,
mask=token_mask,
)
else:
sf_offset = tl.arange(0, num_groups)
tl.store(
x_sf
+ token_id[:, None] * x_sf_stride_m
+ k_block_id * x_sf_stride_k
+ sf_offset[None, :],
quant_scale,
mask=token_mask[:, None],
)
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
# in an MN-major layout whose row permutation depends on the MegaMoE
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
# is already resident, avoiding another kernel and temporary tensor.
if USE_UE8M0 and shared_x_sf is not None:
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
tl.store(
shared_x_sf
+ shared_row * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
packed_scale,
mask=token_mask,
)
if k_block_id == 0:
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
def _prepare_megamoe_nvfp4_inputs_kernel(
hidden_states,
x_out,
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_sf_stride_m: tl.constexpr,
shared_x_stride_m: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
l2_activation_scale,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
num_local_experts: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
BLOCK_E: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
SHARED_MXFP8: tl.constexpr,
) -> None:
if l2_alpha is not None:
experts = tl.arange(0, BLOCK_E)
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_offsets = tl.arange(0, BLOCK_K)
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
amax = tl.zeros((BLOCK_M,), tl.float32)
for k_block_id in range(hidden_size // BLOCK_K):
hidden = tl.load(
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
tl.store(x_scales + token_id, x_scale, mask=token_mask)
for k_block_id in range(hidden_size // BLOCK_K):
k_ids = k_block_id * BLOCK_K + k_offsets
hidden = tl.load(
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
)
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
values = tl.reshape(
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
[BLOCK_M, BLOCK_K // 16, 16],
)
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
block_scale = block_scale.to(tl.float8e4nv)
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
magnitude = tl.abs(values)
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
code = (
(magnitude > 0.25).to(tl.int32)
+ (magnitude >= 0.75).to(tl.int32)
+ (magnitude > 1.25).to(tl.int32)
+ (magnitude >= 1.75).to(tl.int32)
+ (magnitude > 2.5).to(tl.int32)
+ (magnitude >= 3.5).to(tl.int32)
+ (magnitude > 5.0).to(tl.int32)
)
code = tl.where((values < 0) & (code != 0), code | 8, code)
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
tl.store(
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
(low | (high << 4)).to(tl.uint8),
mask=token_mask[:, None],
)
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
sf_bytes = tl.reshape(
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
)
tl.store(
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
_pack_bytes(sf_bytes, 4),
mask=token_mask[:, None],
)
if shared_x is not None:
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
if SHARED_MXFP8:
fp8, _, scale_exp = _quantize_fp8_groups(
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
)
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
tl.store(
shared_x_sf
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
_pack_bytes(scale_exp, BLOCK_K // 32),
mask=token_mask,
)
else:
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
scale = hidden_quant.scale
group_shape = scale.group_shape
if (
hidden_quant.dtype != FP8_DTYPE
or not hidden_quant.symmetric
or hidden_quant.scale2 is not None
or scale.static
or not group_shape.is_per_group()
or block_k % group_shape.col != 0
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
):
raise ValueError(
"MegaMoE input staging requires symmetric dynamic fp8 "
f"per-group quantization with a group size dividing {block_k} and "
f"E8M0 or fp32 scales, got {hidden_quant}."
)
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
def prepare_megamoe_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor | None,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
hidden_quant: QuantKey | None = kMxfp8Dynamic,
) -> None:
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
Args:
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Output buffer for the dispatched hidden states.
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
``hidden_quant`` is ``None``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x_sf: Optional output buffer for shared-expert scale factors.
Must be given together with ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
four per int32 into ``x_sf``; fp32 scales are stored directly.
``None`` copies the hidden states into ``x`` unquantized.
"""
num_tokens, hidden_size = hidden_states.shape
if num_tokens == 0:
return
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
block_k = 128
quantize = hidden_quant is not None
if quantize != (x_sf is not None):
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
group_k, use_ue8m0 = (
(block_k, False)
if hidden_quant is None
else _resolve_hidden_quant(hidden_quant, block_k)
)
top_k = topk_ids.shape[1]
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
if (shared_x_sf is None) != (shared_block_m is None):
raise ValueError(
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
)
if shared_x_sf is not None and not use_ue8m0:
raise ValueError(
"MegaMoE shared input staging currently requires "
"UE8M0-packed hidden scales."
)
if shared_x_sf is not None:
assert shared_block_m is not None
if shared_block_m <= 0:
raise ValueError("MegaMoE shared_block_m must be positive.")
expected_sf_k = hidden_size // 128
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
raise ValueError(
"MegaMoE shared_x_sf must have shape "
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
)
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
if shared_x_sf.shape[0] < required_rows:
raise ValueError(
"MegaMoE shared_x_sf has insufficient rows: requires "
f"{required_rows}, got {shared_x_sf.shape[0]}."
)
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
block_m = 8 if num_tokens >= 64 else 1
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
block_topk = triton.next_power_of_2(top_k)
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
_prepare_megamoe_inputs_kernel[grid](
hidden_states,
x,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x.stride(1),
x_sf.stride(0) if x_sf is not None else 0,
x_sf.stride(1) if x_sf is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
padding_stride_m,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
hidden_size,
top_k,
BLOCK_M=block_m,
BLOCK_K=block_k,
GROUP_K=group_k,
BLOCK_TOPK=block_topk,
SHARED_BLOCK_M=shared_block_m or 1,
QUANTIZE=quantize,
USE_UE8M0=use_ue8m0,
num_warps=4,
)
def prepare_megamoe_nvfp4_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor,
x_scales: torch.Tensor,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
w2_global_scale: torch.Tensor | None = None,
l2_alpha: torch.Tensor | None = None,
l2_activation_scale: float = 1.0,
) -> None:
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
quantized to E2M1 with one E4M3 scale per 16 values.
Args:
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
or a BF16 copy of the hidden states.
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
TMA layout for ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
w2_global_scale: Per-local-expert L2 weight global scales.
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
dequantization scale of each local expert. Written even without
local tokens, as other ranks' tokens still reach these experts.
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
"""
num_tokens, hidden_size = hidden_states.shape
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
shared_block_m is None
):
raise ValueError(
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
"for an fp8 shared_x."
)
if (w2_global_scale is None) != (l2_alpha is None):
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
if tensor is not None and tensor.stride(-1) != 1:
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
block_m = 8 if num_tokens >= 64 else 1
top_k = topk_ids.shape[1]
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
# One program even without tokens, to write l2_alpha.
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
_prepare_megamoe_nvfp4_inputs_kernel[grid](
hidden_states,
x.view(torch.uint8),
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x_sf.stride(0),
shared_x.stride(0) if shared_x is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
is_padding.stride(0) if is_padding is not None else 0,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
l2_activation_scale,
hidden_size,
top_k,
num_local_experts,
BLOCK_M=block_m,
BLOCK_K=128,
BLOCK_TOPK=triton.next_power_of_2(top_k),
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
SHARED_BLOCK_M=shared_block_m or 1,
SHARED_MXFP8=shared_mxfp8,
num_warps=4,
)
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
@triton.jit
def _relu_squared_static_fp8_quant_kernel(
x_ptr,
scale_ptr,
output_ptr,
num_elements,
FP8_MIN: tl.constexpr,
FP8_MAX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < num_elements
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
input_is_nan = x != x
relu = tl.maximum(x, 0.0)
# Preserve the standalone ReLU2 kernel's BF16 output boundary.
activated = (relu * relu).to(tl.bfloat16).to(tl.float32)
# Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply.
quantized = activated * (1.0 / tl.load(scale_ptr))
quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN)
# Match the O2 native chain by preserving NaN through the FP8 conversion.
quantized = tl.where(input_is_nan, x, quantized)
tl.store(output_ptr + offsets, quantized, mask=mask)
def relu_squared_static_fp8_quant(
x: torch.Tensor, linear: LinearBase
) -> QuantizedActivation:
"""BF16 ReLU2 followed by static per-tensor FP8 quantization."""
assert x.dtype == torch.bfloat16
assert x.is_contiguous()
scale = linear.input_scale
assert scale.dtype == torch.float32
assert x.device == scale.device
assert scale.numel() == 1
output = torch.empty_like(x, dtype=current_platform.fp8_dtype())
if x.numel() != 0:
block_size = min(triton.next_power_of_2(x.shape[-1]), 2048)
num_warps = min(max(block_size // 256, 1), 4)
grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
_relu_squared_static_fp8_quant_kernel[grid](
x,
scale,
output,
x.numel(),
FP8_MIN=_FP8_MIN,
FP8_MAX=_FP8_MAX,
BLOCK_SIZE=block_size,
num_warps=num_warps,
)
return QuantizedActivation(
data=output,
scale=scale,
orig_dtype=x.dtype,
orig_shape=x.shape,
quant_key=kFp8StaticTensorSym,
)
-405
View File
@@ -1,405 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Top-k kernels for the DSA sparse attention indexer."""
import functools
import torch
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.worker.workspace import current_workspace_manager
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
# cooperative_topk's hard row limit: one cluster wave covers at most this many
# rows. "auto" uses cooperative_topk for every batch within the limit and falls
# back to persistent_topk past it; explicit ``cooperative`` requests are
# validated against the same bound.
AUTO_COOPERATIVE_MAX_ROWS = 64
# ---------------------------------------------------------------------------
# DeepSelect (vllm._deepselect_C)
# ---------------------------------------------------------------------------
# Matches the -1 fill convention used for topk_indices_buffer elsewhere.
IDX_OOB_FILL_VALUE = -1
try:
import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select)
_HAS_DEEP_SELECT_C = True
except ImportError:
_HAS_DEEP_SELECT_C = False
@functools.lru_cache(maxsize=1)
def get_deep_select_stride_requirement() -> tuple[int, int]:
"""Stride alignment requirement (input, output) in bytes."""
return torch.ops.deep_select.get_alignment_requirement()
def is_deep_select_supported(input: torch.Tensor, topk: int) -> bool:
"""Whether the kernel accepts this input (dtype/stride/topk constraints)."""
return (
topk <= 4096
and input.shape[1] < 2**23
and input.dtype in (torch.float32, torch.bfloat16)
and input.stride(1) == 1
and input.stride(0)
* input.element_size()
% get_deep_select_stride_requirement()[0]
== 0
)
def _get_empty_and_aligned_tensor(
dim0: int, dim1: int, device: torch.device, dtype: torch.dtype
) -> torch.Tensor:
"""Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned."""
output_stride_requirement = get_deep_select_stride_requirement()[1] // (
dtype.itemsize
)
assert output_stride_requirement > 0
dim1_rounded = (
(dim1 + output_stride_requirement - 1) // output_stride_requirement
) * output_stride_requirement
return torch.empty((dim0, dim1_rounded), device=device, dtype=dtype)[:, :dim1]
def deep_select_topk(
input: torch.Tensor,
topk: int,
end: torch.Tensor | None = None,
output_idx: torch.Tensor | None = None,
indices_dtype: torch.dtype = torch.int32,
) -> torch.Tensor:
"""Select the top-k indices per row of `input` with DeepSelect.
Args:
input: (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and
stride(0) must be 1024B-aligned.
topk: Number of elements to select per row; must be <= 4096.
end: Optional (num_rows,) int32 tensor with the exclusive right
boundary of each row. Rows with `end[i] < topk` get their
remaining indices filled with -1.
output_idx: Optional preallocated (num_rows, topk) output tensor whose
stride(0) is 32B-aligned (e.g. a slice of a wider buffer).
indices_dtype: Output dtype when `output_idx` is not provided.
Returns:
The (num_rows, topk) indices tensor.
"""
assert input.dim() == 2 and input.stride(1) == 1
assert (
input.stride(0) * input.element_size() % get_deep_select_stride_requirement()[0]
== 0
)
num_rows = input.shape[0]
if output_idx is None:
output_idx = _get_empty_and_aligned_tensor(
num_rows, topk, input.device, indices_dtype
)
else:
assert output_idx.dtype == indices_dtype
assert output_idx.shape[0] >= num_rows and output_idx.shape[1] >= topk
torch.ops.deep_select.topk(
input,
topk,
None, # begin is not supported
end,
False, # sorted_value
False, # sorted_index
None, # output_value
output_idx,
None, # output_idx_offset
IDX_OOB_FILL_VALUE,
float("-inf"), # value_oob_fill_value
False, # return_value
False, # abort_when_nan_found: dummy/capture passes can feed NaN
# logits from uninitialized KV cache; the reference topk kernels do
# not trap on NaN, and trapping would abort CUDA graph capture.
)
return output_idx
# ---------------------------------------------------------------------------
# Backend selection
# ---------------------------------------------------------------------------
@functools.cache
def get_indexer_topk(backend: str) -> "SparseIndexerTopk":
return SparseIndexerTopk(backend)
class SparseIndexerTopk(torch.nn.Module):
"""The sparse indexer's decode top-k stage.
Selects among the available top-k kernels (see
kernel_config.sparse_indexer_topk_backend) and runs the chosen one.
"""
def __init__(self, backend: str | None = None) -> None:
super().__init__()
if backend is None:
backend = (
get_current_vllm_config().kernel_config.sparse_indexer_topk_backend
)
self._backend = backend
self._is_cuda = current_platform.is_cuda()
self._has_deep_select = self._is_cuda and (
current_platform.is_device_capability_family(100)
)
self._has_flashinfer_topk = has_flashinfer()
self._cooperative_capable = self._is_cuda and (
current_platform.has_device_capability(90)
and not current_platform.is_device_capability_family(120)
)
self._is_rocm = current_platform.is_rocm()
self._aiter_enabled = False
self._aiter_capable = False
if self._is_rocm:
from vllm._aiter_ops import rocm_aiter_ops
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
def resolve_backend(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""Resolve the decode top-k implementation from the configured
backend ("auto" = the pre-existing chain, or a validated explicit
value)."""
if self._backend == "auto":
return self._resolve_auto(logits, topk_tokens, num_rows)
failures: list[str] = []
if self._backend == "cooperative":
failures = self._cooperative_constraints(logits, topk_tokens, num_rows)
elif self._backend == "persistent":
if not self._is_cuda:
failures.append("requires a CUDA platform")
if topk_tokens not in (512, 1024, 2048):
failures.append(
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
)
elif self._backend == "deep_select":
if not self._is_cuda:
failures.append("requires a CUDA platform")
elif not self._has_deep_select:
failures.append("requires SM100a/SM103a (10.x device family)")
elif not _HAS_DEEP_SELECT_C:
failures.append("vllm._deepselect_C could not be imported")
elif not is_deep_select_supported(logits, topk_tokens):
failures.append(
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
f" stride={logits.stride()}, topk={topk_tokens}"
)
elif self._backend == "aiter":
if not self._is_rocm:
failures.append("requires a ROCm platform")
elif not self._aiter_enabled:
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
elif not self._aiter_capable:
failures.append(
"AITER has no tuned indexer top-k kernels for this GPU arch"
)
if topk_tokens not in (512, 1024, 2048, 4096):
failures.append(
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
)
elif self._backend == "flashinfer":
if not self._is_cuda:
failures.append("requires a CUDA platform")
if not self._has_flashinfer_topk:
failures.append(
"flashinfer.topk.top_k_ragged_transform is not importable"
)
if logits.dtype != torch.float32 or logits.stride(1) != 1:
failures.append(
f"requires fp32 logits with stride(1) == 1, got "
f"dtype={logits.dtype}, stride={logits.stride()}"
)
if failures:
raise RuntimeError(
f"sparse_indexer_topk_backend='{self._backend}' was requested, but: "
+ "; ".join(failures)
)
return self._backend
def _resolve_auto(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The priority chain: cooperative -> persistent -> per_row.
aiter/deep_select/flashinfer/torch are opt-in only.
cooperative_topk is preferred whenever it is applicable, i.e. within
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
persistent_topk.
aiter not auto-resolved as it does not outperform the per-row on every shape,
hence we delegate to the caller to decide when to use it.
"""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
if self._is_cuda and topk_tokens in (512, 1024, 2048):
return "persistent"
return "per_row"
def _cooperative_constraints(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> list[str]:
"""Unmet constraints of cooperative_topk (empty when applicable)."""
failures = []
if not self._is_cuda:
failures.append("requires a CUDA platform")
if topk_tokens not in (512, 1024, 2048):
failures.append(
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
)
if num_rows > AUTO_COOPERATIVE_MAX_ROWS:
failures.append(
f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}"
)
if logits.stride(0) % 4 != 0:
failures.append(
f"logits.stride(0) must be divisible by 4, got {logits.stride(0)}"
)
if self._is_cuda and not self._cooperative_capable:
failures.append("requires SM90+ and is not supported on the SM12x family")
return failures
@staticmethod
def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor:
"""Per-row exclusive end offsets (int32, (num_rows,)) for top-k
kernels that take ragged lengths (DeepSelect, FlashInfer, torch
reference).
seq_lens is (B, next_n) per-row effective lens for native spec decode
and (B, 1) otherwise, in which case per-row lens are derived the same
way as the other decode top-k kernels.
"""
if seq_lens.numel() == num_rows:
row_ends = seq_lens.reshape(-1)
else:
next_n_offsets = torch.arange(
next_n, dtype=torch.int32, device=seq_lens.device
)
row_ends = (
(seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets)
.clamp_(min=0)
.reshape(-1)
)
assert row_ends.dtype == torch.int32
return row_ends
def forward(
self,
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices)
elif backend == "cooperative":
(topk_workspace,) = current_workspace_manager().get_simultaneous(
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
torch.ops._C.cooperative_topk(
logits,
seq_lens,
topk_indices,
topk_workspace,
topk_tokens,
max_seq_len,
)
elif backend == "persistent":
(topk_workspace,) = current_workspace_manager().get_simultaneous(
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
)
# persistent_topk's last argument is the host-side "max seq_len
# across rows" bound (it gates the sampled_topk path and clamps
# per-row lengths), not the padded row width: logits is sized to
# max_model_len, so passing its width would incorrectly enable the
# long-row kernels on short-context batches.
torch.ops._C.persistent_topk(
logits,
seq_lens,
topk_indices,
topk_workspace,
topk_tokens,
max_seq_len,
)
elif backend == "flashinfer":
# Deferred: importing flashinfer initializes CUDA at import time.
from flashinfer.topk import top_k_ragged_transform
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
offsets = torch.zeros(
logits.shape[0], dtype=torch.int32, device=logits.device
)
# top_k_ragged_transform selects within [0, row_ends[i]) per
# row and -1-fills past the row length.
indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens)
topk_indices.copy_(indices)
elif backend == "torch":
# Debug reference: mask everything past each row's end, then topk.
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
cols = torch.arange(logits.shape[1], device=logits.device)
masked = logits.masked_fill(
cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf")
)
indices = masked.topk(topk_tokens, dim=-1).indices
in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze(
0
) < row_ends.unsqueeze(1)
indices = torch.where(in_range, indices, -1)
topk_indices.copy_(indices)
elif backend == "aiter":
from vllm._aiter_ops import rocm_aiter_ops
# AITER's decode kernel only implements the 1D/next_n == 1 form,
# which is equivalent once the ragged ends are expanded.
rocm_aiter_ops.indexer_top_k_decode(
logits,
1,
self._row_ends(seq_lens, next_n, logits.shape[0]),
topk_indices,
topk_tokens,
)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
@staticmethod
def _per_row(
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
) -> None:
ops.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
@@ -1,741 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from einops import rearrange
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
@triton.jit
def _fwd_diag_kernel(
Q,
K,
V,
Out,
S,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
CBLOCK: tl.constexpr,
):
# This kernel computes the diagonal blocks of the attention matrix
# Each diagonal block represents attention
# where queries attend to keys in the same block
off = tl.program_id(0)
off_bh = off // NUM_BLOCK # batch-head index
off_block = off % NUM_BLOCK # block index within the sequence
off_cblock = tl.program_id(1) # sub-block index within a block
off_h = off_bh % h # head index
# Calculate base offsets for the current batch and head
qk_offset = off_bh * n * d
v_offset = off_bh * n * e
o_offset = off_bh * n * e
# Calculate offsets for the current block
block_offset = off_block * BLOCK
qk_block_offset = block_offset * d
v_block_offset = block_offset * e
o_block_offset = block_offset * e
# Calculate offsets for the current sub-block
cblock_offset = off_cblock * CBLOCK
q_cblock_offset = cblock_offset * d
o_cblock_offset = cblock_offset * e
# Calculate pointers to the query, key, value, and output tensors
Q_block_ptr = (
Q
+ qk_offset
+ qk_block_offset
+ q_cblock_offset
+ tl.arange(0, CBLOCK)[:, None] * d
+ tl.arange(0, d)[None, :]
)
K_trans_block_ptr = (
K
+ qk_offset
+ qk_block_offset
+ tl.arange(0, CBLOCK)[None, :] * d
+ tl.arange(0, d)[:, None]
)
V_block_ptr = (
V
+ v_offset
+ v_block_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, e)[None, :]
)
O_block_ptr = (
Out
+ o_offset
+ o_block_offset
+ o_cblock_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, e)[None, :]
)
# Load the decay rate for the current head
S_block_ptr = S + off_h
s = tl.load(S_block_ptr)
i = off_cblock
q_index = tl.arange(0, CBLOCK) + i * CBLOCK
# Load query values
q = tl.load(Q_block_ptr, mask=block_offset + q_index[:, None] < n, other=0.0).to(
tl.float32
)
# Initialize output accumulator
qkv = tl.zeros([CBLOCK, e], dtype=tl.float32)
# Process all sub-blocks up to and
# including the current one (causal attention)
for j in range(i + 1):
kv_index = tl.arange(0, CBLOCK) + j * CBLOCK
diff = q_index[:, None] - kv_index[None, :]
s_index = s * diff
# Apply causal mask: only attend to positions before the current one
s_index = tl.where(diff >= 0, -s_index, float("-inf"))
decay = tl.exp(s_index)
# Load key and value
k_trans = tl.load(
K_trans_block_ptr,
mask=block_offset + kv_index[None, :] < n,
other=0.0,
).to(tl.float32)
v = tl.load(
V_block_ptr,
mask=block_offset + kv_index[:, None] < n,
other=0.0,
).to(tl.float32)
# Compute attention scores and apply decay
qk = tl.dot(q, k_trans) * decay
# Compute weighted values and accumulate
qkv += tl.dot(qk, v)
# Move to the next sub-block
K_trans_block_ptr += CBLOCK * d
V_block_ptr += CBLOCK * e
# Store the result
tl.store(
O_block_ptr,
qkv.to(O_block_ptr.dtype.element_ty),
mask=block_offset + q_index[:, None] < n,
)
@triton.jit
def _fwd_kv_parallel(
K,
V,
K_decay,
KV,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
D_FBLOCK: tl.constexpr,
E_FBLOCK: tl.constexpr,
NUM_FBLOCK: tl.constexpr,
CBLOCK: tl.constexpr,
NUM_CBLOCK: tl.constexpr,
):
# This kernel computes the key-value outer
# products for each block in parallel
off_bh = tl.program_id(0) # batch-head index
off_block = tl.program_id(1) # block index
off_h = off_bh % h # head index
block_offset = off_block * BLOCK
# Calculate offsets for the current block
k_block_offset = block_offset * d
v_block_offset = block_offset * e
kv_block_offset = off_block * d * e
# Calculate base offsets for the current batch and head
k_offset = off_bh * n * d
v_offset = off_bh * n * e
kv_offset = off_bh * NUM_BLOCK * d * e
# Calculate pointers to the key, value, and key-value tensors
K_trans_block_ptr = (
K
+ k_offset
+ k_block_offset
+ tl.arange(0, CBLOCK)[None, :] * d
+ tl.arange(0, D_FBLOCK)[:, None]
)
V_block_ptr = (
V
+ v_offset
+ v_block_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
KV_block_ptr = (
KV
+ kv_offset
+ kv_block_offset
+ tl.arange(0, D_FBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
# Load the decay factors for the current head and block
k_decay_ptr = K_decay + off_h * BLOCK + tl.arange(0, CBLOCK)
kv_index = tl.arange(0, CBLOCK)
# Initialize the key-value outer product accumulator
kv = tl.zeros([D_FBLOCK, E_FBLOCK], dtype=tl.float32)
# Handle the last block which might be smaller than BLOCK
split_n = n - (NUM_BLOCK - 1) * BLOCK if off_block == NUM_BLOCK - 1 else BLOCK
left_shift = tl.cdiv(split_n, CBLOCK) * CBLOCK - split_n
num_blocks = min(tl.cdiv(split_n, CBLOCK), NUM_CBLOCK)
k_decay_ptr += (NUM_CBLOCK - num_blocks) * CBLOCK
# Process all sub-blocks in the current block
for j in range(num_blocks):
left_bound = (1 - j) * left_shift
# Load key and value, handling boundary conditions
k_trans = tl.load(
K_trans_block_ptr - left_shift * d,
mask=kv_index[None, :] >= left_bound,
other=0.0,
)
v = tl.load(
V_block_ptr - left_shift * e,
mask=kv_index[:, None] >= left_bound,
other=0.0,
)
# Load decay factor and compute weighted key-value outer product
k_decay = tl.load(k_decay_ptr)
# NOTE: Need to add the extra dim here due to AMD MLIR lowering error.
# Please don't move it back until issue is resolved.
# Issue: https://github.com/ROCm/triton/issues/907
k_decay = k_decay[None, :]
kv += tl.dot(k_trans * k_decay, v)
# Move to the next sub-block
K_trans_block_ptr += CBLOCK * d
V_block_ptr += CBLOCK * e
k_decay_ptr += CBLOCK
# Store the result
tl.store(KV_block_ptr, kv.to(KV_block_ptr.dtype.element_ty))
@triton.jit
def _fwd_kv_reduce(
S,
KV,
KV_HISTORY,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
D_FBLOCK: tl.constexpr,
E_FBLOCK: tl.constexpr,
):
# This kernel reduces the key-value outer products
# across blocks and updates the KV history
off_bh = tl.program_id(0) # batch-head index
off_h = off_bh % h # head index
kv_offset = off_bh * NUM_BLOCK * d * e
# Calculate pointer to the key-value tensor
KV_block_ptr = (
KV
+ kv_offset
+ tl.arange(0, D_FBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
# Load the decay rate for the current head
s_ptrs = S + off_h
s = tl.load(s_ptrs)
# Calculate pointer to the key-value history tensor
kv_history_offset = off_bh * d * e
KV_HISTORY_block_ptr = (
KV_HISTORY
+ kv_history_offset
+ tl.arange(0, D_FBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
# Load the previous key-value history
kv_pre = tl.load(KV_HISTORY_block_ptr).to(tl.float32)
# Process all blocks in reverse order to compute the prefix sum
for i in range(NUM_BLOCK):
block_size = min(n - i * BLOCK, BLOCK)
# Compute decay factor for the current block
block_decay = tl.exp(-s.to(tl.float32) * block_size)
# Load the current key-value outer product
kv_cur = tl.load(KV_block_ptr).to(tl.float32)
# Store the previous key-value history to the current block
tl.store(KV_block_ptr, kv_pre.to(KV_block_ptr.dtype.element_ty))
# Update the key-value history with the current block
kv_pre = block_decay * kv_pre + kv_cur
KV_block_ptr += d * e
# Store the updated key-value history
tl.store(KV_HISTORY_block_ptr, kv_pre)
@triton.jit
def _fwd_none_diag_kernel(
Q,
Out,
S,
KV,
b: tl.constexpr,
h: tl.constexpr,
n,
d: tl.constexpr,
e: tl.constexpr,
BLOCK: tl.constexpr,
NUM_BLOCK,
E_FBLOCK: tl.constexpr,
CBLOCK: tl.constexpr,
NUM_CBLOCK: tl.constexpr,
):
# This kernel computes the non-diagonal blocks of the attention matrix
# Each non-diagonal block represents attention
# where queries attend to keys in different blocks
off_bh = tl.program_id(0) # batch-head index
off_h = off_bh % h # head index
off_nc = tl.program_id(1)
off_n = off_nc // NUM_CBLOCK # block index
off_c = off_nc % NUM_CBLOCK # sub-block index
off_e = tl.program_id(2) # output feature block index
n_offset = off_n * BLOCK
c_offset = off_c * CBLOCK
e_offset = off_e * E_FBLOCK
block_offset = n_offset + c_offset
# Calculate offsets for the current batch, head, and block
q_offset = off_bh * n * d + (n_offset + c_offset) * d
o_offset = off_bh * n * e + (n_offset + c_offset) * e + e_offset
kv_offset = off_bh * NUM_BLOCK * d * e + off_n * d * e + e_offset
# Calculate pointers to the query, output, and key-value tensors
Q_block_ptr = (
Q + q_offset + tl.arange(0, CBLOCK)[:, None] * d + tl.arange(0, d)[None, :]
)
O_block_ptr = (
Out
+ o_offset
+ tl.arange(0, CBLOCK)[:, None] * e
+ tl.arange(0, E_FBLOCK)[None, :]
)
KV_block_ptr = (
KV + kv_offset + tl.arange(0, d)[:, None] * e + tl.arange(0, E_FBLOCK)[None, :]
)
# Load the decay rate for the current head
S_block_ptr = S + off_h
s = tl.load(S_block_ptr)
c_array = tl.arange(0, CBLOCK)
# Load the key-value outer product for the current block
kv = tl.load(KV_block_ptr).to(tl.float32)
q_index = block_offset + tl.arange(0, CBLOCK)
# Load query values
q = tl.load(Q_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
# Compute decay factors for the current sub-block
q_decay = tl.exp(-s.to(tl.float32) * (off_c * CBLOCK + c_array[:, None]))
# Compute non-diagonal attention output
qkv_none_diag = tl.dot(q, kv) * q_decay
# Load diagonal attention output (computed by _fwd_diag_kernel)
qkv_diag = tl.load(O_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
# Combine diagonal and non-diagonal attention outputs
qkv = qkv_diag + qkv_none_diag
# Store the result
tl.store(
O_block_ptr, qkv.to(O_block_ptr.dtype.element_ty), mask=q_index[:, None] < n
)
class _attention(torch.autograd.Function):
@staticmethod
def forward(ctx, q, k, v, s, kv_history):
# Forward pass of the lightning attention algorithm
q = q.contiguous()
k = k.contiguous()
v = v.contiguous()
s = s.contiguous()
# Check CUDA compute capability (Ampere+ required for flash attention
# path). Other accelerators (ROCm, XPU) rely on their own Triton
# backend support and skip this check.
if current_platform.is_cuda():
capability = torch.cuda.get_device_capability()
if capability[0] < 8:
raise RuntimeError(
"Flash attention currently only supported",
"for compute capability >= 80",
)
# Get input dimensions
b, h, n, d = q.shape
e = v.shape[-1]
# Initialize output tensor
o = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device)
# Set block sizes
BLOCK = 256
NUM_BLOCK = triton.cdiv(n, BLOCK)
CBLOCK = 32
NUM_CBLOCK = BLOCK // CBLOCK
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
# Compute decay factors for keys
array = torch.arange(0, BLOCK, device=q.device) + 1
k_decay = torch.exp(-s * (BLOCK - array.reshape(1, -1)))
# Step 1: Compute diagonal blocks of attention
grid = (b * h * NUM_BLOCK, NUM_CBLOCK)
_fwd_diag_kernel[grid](
q,
k,
v,
o,
s,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
CBLOCK=CBLOCK,
)
# Set feature block sizes
NUM_FBLOCK = 1
D_FBLOCK = d // NUM_FBLOCK
assert d % NUM_FBLOCK == 0
E_FBLOCK = e // NUM_FBLOCK
assert e % NUM_FBLOCK == 0
CBLOCK = 64
NUM_CBLOCK = BLOCK // CBLOCK
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
# Step 2: Compute key-value outer products for each block in parallel
kv = torch.empty((b, h, NUM_BLOCK, d, e), dtype=torch.float32, device=q.device)
grid = (b * h, NUM_BLOCK)
_fwd_kv_parallel[grid](
k,
v,
k_decay,
kv,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
D_FBLOCK=D_FBLOCK,
E_FBLOCK=E_FBLOCK,
NUM_FBLOCK=NUM_FBLOCK,
CBLOCK=CBLOCK,
NUM_CBLOCK=NUM_CBLOCK,
)
# Step 3: Reduce key-value outer products
# across blocks and update KV history
grid = (b * h, NUM_FBLOCK)
_fwd_kv_reduce[grid](
s,
kv,
kv_history,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
D_FBLOCK=D_FBLOCK,
E_FBLOCK=E_FBLOCK,
)
# Step 4: Compute non-diagonal blocks of attention
grid = (b * h, NUM_BLOCK * NUM_CBLOCK)
_fwd_none_diag_kernel[grid](
q,
o,
s,
kv,
b,
h,
n,
d,
e,
BLOCK=BLOCK,
NUM_BLOCK=NUM_BLOCK,
E_FBLOCK=E_FBLOCK,
CBLOCK=CBLOCK,
NUM_CBLOCK=NUM_CBLOCK,
)
# Save tensors for backward pass
ctx.save_for_backward(q, k, v, s, kv)
ctx.BLOCK = BLOCK
return o, torch.cat([kv, kv_history.unsqueeze(2)], dim=2)
# Apply the lightning attention function
lightning_attention_ = _attention.apply
def lightning_attention(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
ed: torch.Tensor,
block_size: int = 256,
kv_history: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Apply lightning attention algorithm
to compute attention efficiently.
Args:
q: Query tensor of shape [batch, heads, seq_len, dim]
k: Key tensor of shape [batch, heads, seq_len, dim]
v: Value tensor of shape [batch, heads, seq_len, dim_v]
ed: Decay rate tensor of shape [heads]
block_size: Size of blocks for block-sparse attention
kv_history: Optional key-value history from previous computations
Returns:
output: Attention output
kv: Updated key-value history
"""
d = q.shape[-1]
e = v.shape[-1]
if ed.dim() == 1:
ed = ed.view(1, -1, 1, 1)
# Split the computation into chunks for better parallelism
m = 128 if d >= 128 else 64
assert d % m == 0, f"Dimension d ({d}) must be divisible by m ({m})"
arr = [m * i for i in range(d // m + 1)]
if arr[-1] != d:
arr.append(d)
n = len(arr)
output = 0
# Initialize or clone key-value history
if kv_history is None:
kv_history = torch.zeros(
(q.shape[0], q.shape[1], d, e), dtype=torch.float32, device=q.device
)
else:
kv_history = kv_history.clone().contiguous()
# Process each chunk and accumulate results
for i in range(n - 1):
s = arr[i]
e = arr[i + 1]
q1 = q[..., s:e]
k1 = k[..., s:e]
o, kv = lightning_attention_(q1, k1, v, ed, kv_history)
output = output + o
return output, kv
@triton.jit
def _linear_attn_decode_kernel(
q_ptr,
k_ptr,
v_ptr,
kv_cache_ptr,
slope_rate,
slot_idx,
output_ptr,
D: tl.constexpr,
qkv_b_stride,
qkv_h_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
pad_slot_id: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
"""Kernel for linear attention decoding with KV cache.
This kernel computes attention for a single token using the KV cache.
"""
pid_b = tl.program_id(0) # batch index
pid_h = tl.program_id(1) # head index
pid_d = tl.program_id(2) # dimension block index
# Load slot index for the current batch
slot_id = tl.load(slot_idx + pid_b).to(tl.int64)
# Skip if slot_id is PAD_SLOT_ID (padding)
if slot_id == pad_slot_id:
return
batch_id = pid_b
head_id = pid_h
# Load decay rate for the current head
ratio = tl.load(slope_rate + pid_h)
# Calculate offsets for dimensions
qk_d_offsets = tl.arange(0, D)
v_d_offsets = tl.arange(0, BLOCK_SIZE) + pid_d * BLOCK_SIZE
cache_d_offsets = (
qk_d_offsets[:, None] * cache_d0_stride + v_d_offsets[None, :] * cache_d1_stride
)
# Calculate offsets for the current batch and head
q_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
k_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
v_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
cache_offset = slot_id * cache_b_stride + head_id * cache_h_stride
# Create masks for loading tensors
qk_mask = qk_d_offsets < D
v_mask = v_d_offsets < D
# Load query, key, and value tensors
q = tl.load(q_ptr + q_offset + qk_d_offsets, mask=qk_mask, other=0.0)
k = tl.load(k_ptr + k_offset + qk_d_offsets, mask=qk_mask, other=0.0)
v = tl.load(v_ptr + v_offset + v_d_offsets, mask=v_mask, other=0.0)
# Compute key-value outer product
kv_outer = k[:, None] * v[None, :]
kv_mask = qk_mask[:, None] & v_mask[None, :]
# Apply decay to previous KV cache
ratio = tl.exp(-ratio)
kv_ptr = kv_cache_ptr + cache_offset + cache_d_offsets
kv_cache_old = tl.load(kv_ptr, mask=kv_mask, other=0.0)
kv_outer = kv_outer + ratio * kv_cache_old
# Compute attention output
output = q[:, None].to(tl.float32) * kv_outer
output = tl.sum(output, axis=0)
# Update KV cache and store output
tl.store(kv_ptr, kv_outer, mask=kv_mask)
tl.store(output_ptr + q_offset + v_d_offsets, output, mask=v_mask)
def linear_decode_forward_triton(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
kv_caches: torch.Tensor,
slope_rate: torch.Tensor,
slot_idx: torch.Tensor,
BLOCK_SIZE: int = 32,
) -> torch.Tensor:
"""Perform linear attention decoding using Triton kernels.
Args:
q: Query tensor of shape [B, H, 1, D]
k: Key tensor of shape [B, H, 1, D]
v: Value tensor of shape [B, H, 1, D]
kv_caches: Key-value cache tensor
slope_rate: Decay rate tensor
slot_idx: Slot indices for batches
BLOCK_SIZE: Size of blocks for processing
Returns:
output: Attention output tensor
"""
B, H, _, D = q.shape
assert k.shape == (B, H, 1, D)
assert v.shape == (B, H, 1, D)
# Initialize output tensor
output = torch.empty_like(q)
# Set grid dimensions for the kernel
grid = (B, H, D // BLOCK_SIZE)
# Calculate strides for tensors
qkv_b_stride = q.stride(0)
qkv_h_stride = q.stride(1)
cache_b_stride = kv_caches.stride(0)
cache_h_stride = kv_caches.stride(1)
cache_d0_stride = kv_caches.stride(2)
cache_d1_stride = kv_caches.stride(3)
# Launch the kernel
_linear_attn_decode_kernel[grid](
q,
k,
v,
kv_caches,
slope_rate,
slot_idx,
output,
D,
qkv_b_stride,
qkv_h_stride,
cache_b_stride,
cache_h_stride,
cache_d0_stride,
cache_d1_stride,
pad_slot_id=PAD_SLOT_ID,
BLOCK_SIZE=BLOCK_SIZE,
)
# Reshape output and return
output = rearrange(output, "b h n d -> b n (h d)")
return output.squeeze(1).contiguous()
@@ -1,382 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateDtypeCalculator,
MambaStateShapeCalculator,
is_conv_state_dim_first,
)
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
# --8<-- [start:short_conv]
@PluggableLayer.register("short_conv")
class ShortConv(MambaBase, PluggableLayer):
# --8<-- [end:short_conv]
def __init__(
self,
config,
dim: int,
layer_idx: int,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.conv_dim = dim
self.L_cache = config.conv_L_cache
self.bias = config.conv_bias
self.conv = ColumnParallelLinear(
input_size=self.L_cache,
output_size=dim,
bias=self.bias,
prefix=f"{prefix}.conv1d",
)
# unsqueeze to fit conv1d weights shape into the linear weights shape.
# Can't do this in `weight_loader` since it already exists in
# `ColumnParallelLinear` and `set_weight_attrs`
# doesn't allow to override it
self.conv.weight.data = self.conv.weight.data.unsqueeze(1)
self.in_proj = MergedColumnParallelLinear(
input_size=dim,
output_sizes=[dim] * 3,
bias=self.bias,
quant_config=quant_config,
prefix=f"{prefix}.in_proj",
)
self.out_proj = RowParallelLinear(
input_size=dim,
output_size=dim,
bias=self.bias,
quant_config=quant_config,
prefix=f"{prefix}.out_proj",
)
vllm_config = get_current_vllm_config()
self.num_spec = vllm_config.num_speculative_tokens
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
self.kv_cache = (torch.tensor([]),)
self.model_config = model_config
self.cache_config = cache_config
self.prefix = prefix
def forward_native(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
# for causal conv can be plugged in here later.
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_fn_cpu as causal_conv1d_torch,
)
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_cpu,
causal_conv1d_update_torch,
)
from vllm.platforms import CpuArchEnum, current_platform
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
assert attn_metadata is None or isinstance(
attn_metadata, ShortConvAttentionMetadata
)
BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1)
# (dim, kernel_size) — same reshape as forward_cuda
conv_weights = self.conv.weight.view(
self.conv.weight.size(0), self.conv.weight.size(2)
)
if attn_metadata is None:
# Profile run — output value doesn't matter
Bx = (B * x).contiguous()
output_tensor, _ = self.out_proj(C * Bx)
output[: hidden_states.shape[0]] = output_tensor
return
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
) # (num_blocks, dim, state_len)
num_prefills = attn_metadata.num_prefills
num_decodes = attn_metadata.num_decode_tokens
num_prefill_tokens = attn_metadata.num_prefill_tokens
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
num_actual_tokens = num_decodes + num_prefill_tokens
B_d, B_p = torch.split(
B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
C_d, C_p = torch.split(
C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
x_d, x_p = torch.split(
x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
)
conv_output_list = []
if has_prefill:
assert attn_metadata.state_indices_tensor_p is not None
Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens)
out_p = causal_conv1d_torch(
Bx_p,
conv_weights,
self.conv.bias,
conv_state,
attn_metadata.query_start_loc_p,
attn_metadata.state_indices_tensor_p.flatten(),
attn_metadata.has_initial_states_p,
activation=None,
).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim)
conv_output_list.append(C_p * out_p)
if has_decode:
assert attn_metadata.state_indices_tensor_d is not None
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
Bx_d = B_d * x_d # (num_decodes, dim)
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
conv_state_view = conv_state[state_indices_d].contiguous()
out_d = causal_conv1d_update_torch(
Bx_d.unsqueeze(-1),
conv_state_view,
conv_weights,
self.conv.bias,
activation=None,
).squeeze(-1)
conv_state[state_indices_d] = conv_state_view
else:
out_d = causal_conv1d_update_cpu(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_d,
)
conv_output_list.insert(0, C_d * out_d)
hidden_states_out = torch.vstack(conv_output_list)
output[:num_actual_tokens], _ = self.out_proj(hidden_states_out)
def forward(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
torch.ops.vllm.short_conv(
hidden_states,
output,
self.prefix,
)
def forward_cuda(
self,
hidden_states: torch.Tensor,
output: torch.Tensor,
):
forward_context = get_forward_context()
# ShortConvAttentionMetadata contains metadata necessary for the
# short_conv triton kernels to operate in continuous batching and in
# chunked prefill modes; they are computed at top-level model forward
# since they stay the same and reused for all mamba layers in the same
# iteration.
attn_metadata_raw = forward_context.attn_metadata
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
has_initial_states_p = attn_metadata.has_initial_states_p
query_start_loc_p = attn_metadata.query_start_loc_p
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1)
conv_weights = self.conv.weight.view(
self.conv.weight.size(0), self.conv.weight.size(2)
)
if attn_metadata is None:
# V1 profile run
Bx = (B * x).contiguous()
hidden_states = C * Bx
contextualized_states, _ = self.out_proj(hidden_states)
return contextualized_states
num_prefills = attn_metadata.num_prefills # request count
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
has_prefill = num_prefills > 0
has_decode = num_decodes > 0
num_actual_tokens = num_decodes + num_prefill_tokens
# NOTE: V1 puts decode before prefill
# Separate prefill and decode by splitting varlen input
# Split along token dimension
B_d, B_p = torch.split(
B[:num_actual_tokens],
[num_decodes, num_prefill_tokens],
dim=0,
)
C_d, C_p = torch.split(
C[:num_actual_tokens],
[num_decodes, num_prefill_tokens],
dim=0,
)
x_d, x_p = torch.split(
x[:num_actual_tokens],
[num_decodes, num_prefill_tokens],
dim=0,
)
conv_output_list = []
if has_prefill:
Bx_p = (B_p * x_p).transpose(0, 1)
Bx = causal_conv1d_fn(
Bx_p,
conv_weights,
self.conv.bias,
activation=None,
conv_states=conv_state,
has_initial_state=has_initial_states_p,
cache_indices=state_indices_tensor_p,
metadata=attn_metadata,
query_start_loc=query_start_loc_p,
).transpose(0, 1)[:num_prefill_tokens]
y = C_p * Bx
conv_output_list.append(y)
if has_decode:
Bx_d = (B_d * x_d).contiguous()
if num_accepted_tokens is not None:
# Speculative decode: the verify step feeds >1 query token per
# decode request, so use the spec-aware conv update path
# (mirrors mamba_mixer2). state_indices_tensor_d is
# (num_decodes, 1 + num_spec_tokens) here.
assert state_indices_tensor_d is not None
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
)
else:
# Non-spec decode
Bx = causal_conv1d_update(
Bx_d,
conv_state,
conv_weights,
self.conv.bias,
activation=None,
conv_state_indices=state_indices_tensor_d,
)
y = C_d * Bx
conv_output_list.insert(0, y)
# Merge prefill and decode outputs before passing to gated MLP
hidden_states = torch.vstack(conv_output_list)
# Final linear projection
output[:num_actual_tokens], _ = self.out_proj(hidden_states)
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
assert self.model_config is not None
assert self.cache_config is not None
return MambaStateDtypeCalculator.short_conv_state_dtype(
self.model_config.dtype,
self.cache_config.mamba_cache_dtype,
)
def get_state_shape(self) -> tuple[tuple[int, ...]]:
return MambaStateShapeCalculator.short_conv_state_shape(
tp_world_size=get_tensor_model_parallel_world_size(),
intermediate_size=self.conv_dim,
conv_kernel=self.L_cache,
num_spec=self.num_spec,
)
@property
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.SHORT_CONV
@eager_break_during_capture
def short_conv(
hidden_states: torch.Tensor,
output: torch.Tensor,
layer_name: str,
) -> None:
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
if not current_platform.is_cpu():
self.forward_cuda(hidden_states=hidden_states, output=output)
else:
self.forward_native(hidden_states=hidden_states, output=output)
direct_register_custom_op(
op_name="short_conv",
op_func=short_conv,
mutates_args=["output"],
)
@@ -1,10 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.minimax_rms_norm.rms_norm_tp import (
MiniMaxText01RMSNormTP,
)
__all__ = [
"MiniMaxText01RMSNormTP",
]
@@ -1,316 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import array
import contextlib
import ctypes
import struct
import sys
import threading
import torch
try:
from cuda.bindings import runtime as cudart
except ImportError:
from cuda import cudart
_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment
_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t)
# ---------------------------------------------------------------------------
# CUDA helpers
# ---------------------------------------------------------------------------
def _check(error):
"""Raise on CUDA runtime error."""
success = getattr(cudart.cudaError_t, "cudaSuccess", None) or cudart.cudaError_t(0)
if error != success:
raise RuntimeError(f"CUDA runtime error: {error}")
# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so
# copy the raw handle bytes through getPtr() instead. This is safe because the
# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]).
def _ipc_handle_to_bytes(handle) -> bytes:
return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE)
def _ipc_handle_from_bytes(raw: bytes):
assert len(raw) == _IPC_HANDLE_SIZE, len(raw)
handle = cudart.cudaIpcMemHandle_t()
ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE)
return handle
def _cuda_malloc(size: int):
aligned = ((size + _ALIGN - 1) >> 21) << 21
err, ptr = cudart.cudaMalloc(aligned)
_check(err)
return ptr, aligned
def _cuda_free(ptr: int):
if ptr:
_check(cudart.cudaFree(ptr)[0])
def _cuda_memset_zero(ptr: int, size: int):
_check(cudart.cudaMemset(ptr, 0, size)[0])
def _cuda_memcpy_d2d(dst: int, src: int, size: int):
_check(
cudart.cudaMemcpy(
dst, src, size, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice
)[0]
)
# ---------------------------------------------------------------------------
# IPC buffer
# ---------------------------------------------------------------------------
class IpcBuffer:
"""Allocates CUDA device memory and exchanges IPC handles with all ranks
so that every rank holds a valid device pointer to every other rank's buffer.
"""
def __init__(self, rank: int, world_size: int, size: int, process_group=None):
self.rank = rank
self.world_size = world_size
self.peer_ptrs: list[int] = [0] * world_size
self.local_ptr: int = 0
self._alive = False
if size <= 0:
return
self.local_ptr, _ = _cuda_malloc(size)
_cuda_memset_zero(self.local_ptr, size)
self._alive = True
# --- exchange IPC handles via torch.distributed ---
err, local_handle = cudart.cudaIpcGetMemHandle(self.local_ptr)
_check(err)
all_handles: list[bytes | None] = [None] * world_size
torch.distributed.all_gather_object(
all_handles, _ipc_handle_to_bytes(local_handle), group=process_group
)
for r in range(world_size):
if r == rank:
self.peer_ptrs[r] = self.local_ptr
else:
raw = all_handles[r]
assert raw is not None
handle = _ipc_handle_from_bytes(raw)
err, ptr = cudart.cudaIpcOpenMemHandle(
handle, cudart.cudaIpcMemLazyEnablePeerAccess
)
_check(err)
self.peer_ptrs[r] = ptr
def serialize(self) -> list[int]:
"""Return peer pointers as a list of int64 values (one per rank)."""
raw = b""
for ptr in self.peer_ptrs:
raw += struct.pack("P", ptr)
return array.array("Q", raw).tolist()
def cleanup(self):
if not self._alive:
return
self._alive = False
for r in range(self.world_size):
if self.peer_ptrs[r] == 0:
continue
if r == self.rank:
_cuda_free(self.peer_ptrs[r])
else:
with contextlib.suppress(RuntimeError):
_check(cudart.cudaIpcCloseMemHandle(self.peer_ptrs[r])[0])
self.peer_ptrs[r] = 0
self.local_ptr = 0
def __del__(self):
if not sys.is_finalizing():
self.cleanup()
# ---------------------------------------------------------------------------
# Lamport negative-zero initialization
# ---------------------------------------------------------------------------
def _lamport_fill_neg_zero(device_ptr: int, size_bytes: int):
"""Fill device memory with IEEE-754 negative zero (-0.0f = 0x80000000).
This is the "slot empty" sentinel for the Lamport protocol: the kernel
spin-waits until a value is *not* negative zero.
"""
if size_bytes == 0 or device_ptr == 0:
return
n_floats = size_bytes // 4
# torch preserves -0.0 in IEEE-754
fill = torch.full((n_floats,), -0.0, dtype=torch.float32, device="cuda")
_cuda_memcpy_d2d(device_ptr, fill.data_ptr(), size_bytes)
del fill
# ---------------------------------------------------------------------------
# LamportWorkspace — the main class
# ---------------------------------------------------------------------------
class LamportWorkspace:
"""Self-contained workspace for Lamport-based cross-GPU AllReduce.
Parameters
----------
rank : int
Local rank (0-based).
world_size : int
Total number of ranks in the TP group.
comm_size : int
Size in bytes of *one* Lamport buffer slot. The total IPC allocation
per rank is ``3 * comm_size`` (triple-buffering). Must be large enough
to hold the per-slot data written by the kernel. Use
``compute_comm_size_for_minimax()`` for a safe default.
process_group : optional
``torch.distributed`` process group for IPC handle exchange.
``None`` uses the default group.
"""
def __init__(self, rank: int, world_size: int, comm_size: int, process_group=None):
assert world_size >= 2, "Lamport workspace requires at least 2 ranks"
assert comm_size > 0, "comm_size must be positive"
self.rank = rank
self.world_size = world_size
self.comm_size = comm_size
# 1) Lamport triple-buffer (the only IPC memory the kernel reads/writes)
lamport_total = 3 * comm_size
self._lamport = IpcBuffer(rank, world_size, lamport_total, process_group)
_lamport_fill_neg_zero(self._lamport.local_ptr, lamport_total)
# 2) flag_buffer on device: int32[3] = {counter, unused, lamport_flag}
# counter — used for block-level sync inside the kernel
# unused — reserved (index 1)
# lamport_flag — triple-buffer rotation index (0 → 1 → 2 → 0 …)
self._flag_buf = torch.zeros(3, dtype=torch.int32, device="cuda")
# 3) layout_buffer on device: int64[2] = {clear_size, comm_size}
# clear_size — bytes to clear from *previous* slot (set by kernel)
# comm_size — size of one triple-buffer slot
self._layout_buf = torch.tensor(
[0, comm_size], dtype=torch.int64, device="cuda"
)
# 4) Assemble device-side void* pointer array
N = world_size
ptrs: list[int] = []
ptrs += [0] * N # [0 .. N-1] ipc_buffers (placeholder)
ptrs += [0] * N # [N .. 2N-1] ipc_barriers (placeholder)
ptrs += self._lamport.serialize() # [2N .. 3N-1] lamport peer ptrs
ptrs.append(self._flag_buf.data_ptr()) # [3N] flag_buffer
ptrs.append(self._layout_buf.data_ptr()) # [3N+1] layout_buffer
self._workspace = torch.tensor(ptrs, dtype=torch.int64, device="cuda")
@property
def workspace(self) -> torch.Tensor:
"""Device tensor (int64) that can be passed to the kernel
as ``void** workspace``."""
return self._workspace
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
@staticmethod
def compute_comm_size_for_minimax(
max_tokens: int,
world_size: int,
fused_qk: bool = True,
) -> int:
"""Return a safe ``comm_size`` (in bytes) for MiniMaxReduceRMSKernel.
The kernel stores per-token variance scalars in the Lamport buffer:
- single-matrix path: ``world_size × max_tokens × 4`` bytes per slot
- fused Q+K path: ``world_size × 2 × ceil(max_tokens/4) × 16`` bytes per slot
The returned value is rounded up to 2 MiB alignment.
"""
if fused_qk:
groups = (max_tokens + 3) // 4
slot_bytes = world_size * 2 * groups * 16 # 16 = sizeof(float4)
else:
slot_bytes = world_size * max_tokens * 4 # 4 = sizeof(float)
return ((slot_bytes + _ALIGN - 1) >> 21) << 21
def cleanup(self):
if hasattr(self, "_lamport"):
self._lamport.cleanup()
def __del__(self):
if not sys.is_finalizing():
self.cleanup()
def __repr__(self):
return (
f"LamportWorkspace(rank={self.rank}, world_size={self.world_size}, "
f"comm_size={self.comm_size})"
)
# ---------------------------------------------------------------------------
# Cached convenience function (mirrors TRT-LLM's get_allreduce_workspace)
# ---------------------------------------------------------------------------
_cache_lock = threading.Lock()
_workspace_cache: dict = {}
def get_allreduce_workspace(
rank: int,
world_size: int,
comm_size: int | None = None,
max_tokens: int = 16384,
process_group=None,
) -> torch.Tensor:
"""Return a cached workspace tensor for the given (rank, world_size) pair.
On first call the workspace is allocated and IPC handles are exchanged;
subsequent calls with the same arguments return the cached tensor.
Parameters
----------
rank, world_size : int
TP rank and TP size.
comm_size : int, optional
Explicit slot size in bytes. If ``None``, computed automatically
from ``max_tokens`` and ``world_size`` (fused Q+K path).
max_tokens : int
Maximum number of tokens per batch (used when ``comm_size is None``).
process_group : optional
``torch.distributed`` process group.
"""
if comm_size is None:
comm_size = LamportWorkspace.compute_comm_size_for_minimax(
max_tokens, world_size, fused_qk=True
)
pg_id = id(process_group) if process_group is not None else 0
key = (rank, world_size, comm_size, pg_id)
with _cache_lock:
if key not in _workspace_cache:
ws = LamportWorkspace(rank, world_size, comm_size, process_group)
_workspace_cache[key] = ws
return _workspace_cache[key].workspace
@@ -1,421 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
# Max number of tokens supported by the Lamport fused allreduce+RMSNorm kernel.
# Larger batches fall back to the eager allreduce + RMSNorm path.
MINIMAX_QK_NORM_MAX_TOKEN_NUM = 2048
_MINIMAX_FUSED_AR_RMS_QK = getattr(torch.ops._C, "minimax_allreduce_rms_qk", None)
def _all_reduce_variance(var: torch.Tensor) -> torch.Tensor:
"""All-reduce a per-token variance tensor across the TP group.
Variance is accumulated in fp32 for numerical stability. The FlashInfer
fused all-reduce caches a single global workspace keyed to the model's
16-bit activation dtype (``use_fp32_lamport=False``); routing an fp32
reduction through it would read against a mismatched workspace and corrupt
the result. FlashInfer's fast-path only triggers for 2D inputs, so reducing
a flattened (1D) view keeps these fp32 reductions on custom all-reduce /
pynccl, both of which handle fp32 correctly.
"""
return tensor_model_parallel_all_reduce(var.flatten()).view_as(var)
@triton.jit
def _minimax_qk_var_kernel(
qkv_ptr, # [num_tokens, hidden], 16-bit activations
var_ptr, # [num_tokens, 2], fp32
row_stride, # element stride between tokens in qkv
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
kv_size: tl.constexpr,
BLOCK: tl.constexpr,
):
"""TP-pre stage: per-token mean-of-squares for the q and k segments.
Accumulates in fp32 while reading the 16-bit qkv in place, so no fp32
copy of q/k is materialized. ``var[:, 0]`` is the q variance and
``var[:, 1]`` the k variance; both are the local-shard means, ready for
the all-reduce that follows.
"""
token = tl.program_id(0)
base = qkv_ptr + token * row_stride
q_acc = 0.0
for off in range(0, q_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < q_size
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
q_acc += tl.sum(x * x, axis=0)
k_acc = 0.0
for off in range(0, kv_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < kv_size
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
k_acc += tl.sum(x * x, axis=0)
tl.store(var_ptr + token * 2 + 0, q_acc / q_size)
tl.store(var_ptr + token * 2 + 1, k_acc / kv_size)
@triton.jit
def _minimax_rms_apply_kernel(
qkv_ptr, # [num_tokens, hidden]
var_ptr, # [num_tokens, 2], fp32, all-reduced sum of per-shard means
q_w_ptr, # [q_size], q per-channel weight
k_w_ptr, # [kv_size], k per-channel weight
q_out_ptr, # [num_tokens, q_size], contiguous
k_out_ptr, # [num_tokens, kv_size], contiguous
row_stride, # element stride between tokens in qkv
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
kv_size: tl.constexpr,
tp_world: tl.constexpr, # folds the post-all-reduce /tp_world into rsqrt
eps: tl.constexpr,
BLOCK: tl.constexpr,
):
"""TP-post stage: ``x * rsqrt(var / tp_world + eps) * weight``.
A single program normalizes both the q and k segments of one token, so q
and k share one launch instead of two. The all-reduce yields the sum of
per-shard means, so the ``/ tp_world`` that recovers the global
mean-of-squares is folded into the ``rsqrt`` here rather than run as a
separate elementwise pass over the ``[num_tokens, 2]`` variance tensor.
"""
token = tl.program_id(0)
base = qkv_ptr + token * row_stride
q_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 0) / tp_world + eps)
q_out_row = q_out_ptr + token * q_size
for off in range(0, q_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < q_size
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
w = tl.load(q_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
y = x * q_inv * w
tl.store(q_out_row + idx, y.to(q_out_ptr.dtype.element_ty), mask=mask)
k_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 1) / tp_world + eps)
k_out_row = k_out_ptr + token * kv_size
for off in range(0, kv_size, BLOCK):
idx = off + tl.arange(0, BLOCK)
mask = idx < kv_size
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
w = tl.load(k_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
y = x * k_inv * w
tl.store(k_out_row + idx, y.to(k_out_ptr.dtype.element_ty), mask=mask)
def _minimax_qk_norm_tp_eager(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_world: int,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pure-torch reference path used when Triton is unavailable."""
q, k, _ = qkv.split([q_size, kv_size, kv_size], dim=-1)
orig_dtype = q.dtype
q = q.to(torch.float32)
k = k.to(torch.float32)
q_var = q.pow(2).mean(dim=-1, keepdim=True)
k_var = k.pow(2).mean(dim=-1, keepdim=True)
qk_var = torch.cat([q_var, k_var], dim=-1)
qk_var = _all_reduce_variance(qk_var) / tp_world
q_var, k_var = qk_var.chunk(2, dim=-1)
q = q * torch.rsqrt(q_var + eps) * q_weight
k = k * torch.rsqrt(k_var + eps) * k_weight
return q.to(orig_dtype), k.to(orig_dtype)
def _minimax_qk_norm_tp_fallback(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_rank: int,
tp_world: int,
eps: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""All-reduce + QK RMSNorm without the Lamport fused kernel.
The all-reduce is a TP communication barrier and cannot live inside a
single kernel, so the eager-torch path is split into two Triton kernels
around it: a variance reduction before the all-reduce and a normalize
after. Compared to the ``torch.compile`` path this avoids materializing
fp32 copies of q/k and the ``cat``/``chunk`` temporaries.
"""
if not HAS_TRITON:
return _minimax_qk_norm_tp_eager(
qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps
)
num_tokens = qkv.shape[0]
row_stride = qkv.stride(0)
BLOCK = 1024
grid = (num_tokens,)
qk_var = torch.empty(num_tokens, 2, dtype=torch.float32, device=qkv.device)
_minimax_qk_var_kernel[grid](
qkv, qk_var, row_stride, q_size=q_size, kv_size=kv_size, BLOCK=BLOCK
)
# All-reduce sums the per-shard means; the /tp_world that turns this back
# into the global mean is folded into the apply kernel's rsqrt below.
qk_var = _all_reduce_variance(qk_var)
q_out = torch.empty(num_tokens, q_size, dtype=qkv.dtype, device=qkv.device)
k_out = torch.empty(num_tokens, kv_size, dtype=qkv.dtype, device=qkv.device)
_minimax_rms_apply_kernel[grid](
qkv,
qk_var,
q_weight,
k_weight,
q_out,
k_out,
row_stride,
q_size=q_size,
kv_size=kv_size,
tp_world=tp_world,
eps=eps,
BLOCK=BLOCK,
)
return q_out, k_out
def _minimax_qk_norm_fusion(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_rank: int,
tp_world: int,
eps: float,
workspace: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
if (
workspace is not None
and tp_world > 1
and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM
and _MINIMAX_FUSED_AR_RMS_QK is not None
):
return _MINIMAX_FUSED_AR_RMS_QK(
qkv,
q_weight,
k_weight,
workspace,
q_size,
kv_size,
tp_rank,
tp_world,
eps,
)
# ROCm AITER fused QK-norm + AllReduce; skips unaligned shapes.
if tp_world > 1 and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM:
pack_size = 16 // qkv.element_size()
warp_work_size = 32 * pack_size
if q_size % warp_work_size == 0 and kv_size % warp_work_size == 0:
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
if aiter_ar is not None:
q_out, k_out, _ = aiter_ar.aiter_ca.custom_fused_qknorm_ar(
qkv, q_weight, k_weight, eps
)
return q_out, k_out
return _minimax_qk_norm_tp_fallback(
qkv, q_weight, k_weight, q_size, kv_size, tp_rank, tp_world, eps
)
def _minimax_qk_norm_fusion_fake(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
q_size: int,
kv_size: int,
tp_rank: int,
tp_world: int,
eps: float,
workspace: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert qkv.ndim == 2
num_tokens = qkv.shape[0]
return (
torch.empty([num_tokens, q_size], dtype=qkv.dtype, device=qkv.device),
torch.empty([num_tokens, kv_size], dtype=qkv.dtype, device=qkv.device),
)
direct_register_custom_op(
op_name="minimax_qk_norm_fusion",
op_func=_minimax_qk_norm_fusion,
fake_impl=_minimax_qk_norm_fusion_fake,
mutates_args=[],
)
@CustomOp.register("minimax_text01_rmsnorm_tp")
class MiniMaxText01RMSNormTP(CustomOp):
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
*,
weight_shard_world_size: int | None = None,
weight_shard_rank: int | None = None,
) -> None:
super().__init__()
self.tp_world = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
self.weight_shard_world = weight_shard_world_size or self.tp_world
self.weight_shard_rank = (
self.tp_rank if weight_shard_rank is None else weight_shard_rank
)
self.weight = nn.Parameter(torch.ones(hidden_size // self.weight_shard_world))
self.weight.weight_loader = partial(
self.weight_loader,
shard_world_size=self.weight_shard_world,
shard_rank=self.weight_shard_rank,
)
self.variance_epsilon = eps
self.workspace = None
if _MINIMAX_FUSED_AR_RMS_QK is not None and self.tp_world > 1:
from .lamport_workspace import (
get_allreduce_workspace,
)
# The Lamport workspace exchanges CUDA IPC handles and enables peer
# access between GPUs. This requires P2P (IPC peer access) to be
# available; on topologies where it is not (e.g. consumer PCIe cards
# with P2P disabled in the driver), allocation raises. Fall back to
# the eager allreduce + RMSNorm path instead of failing model load.
try:
self.workspace = get_allreduce_workspace(
rank=self.tp_rank,
world_size=self.tp_world,
max_tokens=MINIMAX_QK_NORM_MAX_TOKEN_NUM,
process_group=get_tp_group().cpu_group,
)
except Exception as e:
logger.warning_once(
"Failed to initialize MiniMax fused allreduce+RMSNorm "
"Lamport workspace: %s. This is expected on GPUs without "
"P2P (IPC peer access) support. Falling back to the eager "
"allreduce + RMSNorm path.",
e,
)
self.workspace = None
@staticmethod
def weight_loader(
param: nn.Parameter,
loaded_weight: torch.Tensor,
shard_world_size: int | None = None,
shard_rank: int | None = None,
) -> None:
if shard_world_size is None:
shard_world_size = get_tensor_model_parallel_world_size()
if shard_rank is None:
shard_rank = get_tensor_model_parallel_rank()
shard_size = loaded_weight.shape[0] // shard_world_size
shard = slice(shard_rank * shard_size, (shard_rank + 1) * shard_size)
param.data.copy_(loaded_weight[shard])
def _forward(
self,
x: torch.Tensor,
) -> torch.Tensor:
orig_dtype = x.dtype
x = x.to(torch.float32)
variance = x.pow(2).mean(dim=-1, keepdim=True, dtype=torch.float32)
if self.tp_world > 1:
variance = _all_reduce_variance(variance) / self.tp_world
x = x * torch.rsqrt(variance + self.variance_epsilon)
x = (x * self.weight).to(orig_dtype)
return x
def forward(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
assert residual is None, "RMSNorm does not support residual connection."
return self._forward(x)
@staticmethod
def forward_qk(
q_norm: "MiniMaxText01RMSNormTP",
k_norm: "MiniMaxText01RMSNormTP",
q: torch.Tensor,
k: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
orig_dtype = q.dtype
q = q.to(torch.float32)
k = k.to(torch.float32)
q_var = q.pow(2).mean(dim=-1, keepdim=True)
k_var = k.pow(2).mean(dim=-1, keepdim=True)
q = q * torch.rsqrt(q_var + q_norm.variance_epsilon) * q_norm.weight
k = k * torch.rsqrt(k_var + k_norm.variance_epsilon) * k_norm.weight
q = q.to(orig_dtype)
k = k.to(orig_dtype)
return q, k
@staticmethod
def forward_qkv(
q_norm: "MiniMaxText01RMSNormTP",
k_norm: "MiniMaxText01RMSNormTP",
qkv: torch.Tensor,
q_size: int,
kv_size: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
assert qkv.ndim == 2
assert q_norm.variance_epsilon == k_norm.variance_epsilon
# Case 0: tp_size=1
if get_tensor_model_parallel_world_size() == 1:
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
q, k = MiniMaxText01RMSNormTP.forward_qk(q_norm, k_norm, q, k)
return q, k, v
# Case : tp_size>1
q, k = torch.ops.vllm.minimax_qk_norm_fusion(
qkv,
q_norm.weight,
k_norm.weight,
q_size,
kv_size,
q_norm.tp_rank,
q_norm.tp_world,
q_norm.variance_epsilon,
q_norm.workspace,
)
_, _, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
return q, k, v
+2 -70
View File
@@ -115,77 +115,9 @@ _SPECULATIVE_DECODING_MODELS = {
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
}
_TRANSFORMERS_SUPPORTED_MODELS = {
# Text generation models
"ArceeForCausalLM": ("transformers", "TransformersForCausalLM"),
"CwmForCausalLM": ("transformers", "TransformersForCausalLM"),
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"),
"PhiForCausalLM": ("transformers", "TransformersForCausalLM"),
"SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"),
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
# Multimodal models
"Emu3ForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
"HunYuanVLForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
"VibeVoiceAsrForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
}
_TRANSFORMERS_SUPPORTED_MODELS = {}
_TRANSFORMERS_BACKEND_MODELS = {
# Text generation models
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
"TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
# Multimodal models
"TransformersMultiModalForCausalLM": (
"transformers",
"TransformersMultiModalForCausalLM",
),
"TransformersMultiModalMoEForCausalLM": (
"transformers",
"TransformersMultiModalMoEForCausalLM",
),
# Embedding models
"TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
"TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
"TransformersMultiModalEmbeddingModel": (
"transformers",
"TransformersMultiModalEmbeddingModel",
),
# Sequence classification models
"TransformersForSequenceClassification": (
"transformers",
"TransformersForSequenceClassification",
),
"TransformersMoEForSequenceClassification": (
"transformers",
"TransformersMoEForSequenceClassification",
),
"TransformersMultiModalForSequenceClassification": (
"transformers",
"TransformersMultiModalForSequenceClassification",
),
}
_TRANSFORMERS_BACKEND_MODELS = {}
_VLLM_MODELS = {
**_TEXT_GENERATION_MODELS,
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Wrapper around `transformers` models."""
from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.moe import MoEMixin
from vllm.model_executor.models.transformers.multimodal import (
MultiModalDummyInputsBuilder,
MultiModalMixin,
MultiModalProcessingInfo,
MultiModalProcessor,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.attention import Attention, MLAAttention
def check_sinks(
module: "torch.nn.Module",
self_attn: "Attention | MLAAttention",
s_aux: "torch.Tensor | None",
):
"""Fail loudly if the model applies a sink the attention layer will not.
Only the attention impl can fold a sink into the softmax denominator, so a sink
that never reached `Attention` is dropped and every softmax is subtly wrong.
"""
if s_aux is None or getattr(self_attn, "has_sink", False):
return
raise ValueError(
f"{type(module).__name__} applies attention sinks, but they were not passed "
f"to {type(self_attn).__name__}, so the output would be wrong. Either the "
"Transformers modeling backend could not find the parameter holding them, or "
"vLLM does not support sinks for this kind of attention. Please open an issue "
"at https://github.com/vllm-project/vllm/issues/new"
)
def vllm_attention_forward(
# Transformers args
module: "torch.nn.Module",
query: "torch.Tensor",
key: "torch.Tensor",
value: "torch.Tensor",
attention_mask: "torch.Tensor",
**kwargs,
):
self_attn = getattr(module, VLLM_ATTN_ATTR)
check_sinks(module, self_attn, kwargs.get("s_aux"))
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
# Pad `value` up to the query/key head size when it is smaller (expanded
# MLA). A larger last dim just means `value` isn't split per head, e.g.
# packed grouped/multi-query projections, and needs no padding.
pad_value = head_dim_v < head_dim_qk
if pad_value:
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
value = value.reshape(hidden, -1)
attn_output = self_attn.forward(query, key, value)
if pad_value:
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
attn_output = attn_output.reshape(hidden, -1)
return attn_output, None
def vllm_mla_attention_forward(
# Transformers args
module: "torch.nn.Module",
query: "torch.Tensor",
kv_c_normed: "torch.Tensor",
k_pe: "torch.Tensor",
attention_mask: "torch.Tensor",
**kwargs,
):
self_attn = getattr(module, VLLM_ATTN_ATTR)
check_sinks(module, self_attn, kwargs.get("s_aux"))
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
query = query.transpose(1, 2).flatten(0, 1)
num_tokens, num_heads = query.shape[:2]
# [batch=1, num_tokens, kv_lora_rank] -> [num_tokens, kv_lora_rank]
kv_c_normed = kv_c_normed.reshape(-1, kv_c_normed.shape[-1])
# [batch=1, heads=1, num_tokens, qk_rope] -> [num_tokens, 1, qk_rope]
k_pe = k_pe.reshape(-1, 1, k_pe.shape[-1])
attn_output = self_attn.forward(
query,
kv_c_normed,
k_pe,
output_shape=(num_tokens, num_heads * self_attn.v_head_dim),
)
return attn_output, None
ALL_ATTENTION_FUNCTIONS.register("vllm", vllm_attention_forward)
ALL_ATTENTION_FUNCTIONS.register("vllm_mla", vllm_mla_attention_forward)
# Text only models
class TransformersForCausalLM(CausalMixin): ...
class TransformersMoEForCausalLM(MoEMixin, CausalMixin): ...
# Multimodal models
@MULTIMODAL_REGISTRY.register_processor(
MultiModalProcessor,
info=MultiModalProcessingInfo,
dummy_inputs=MultiModalDummyInputsBuilder,
)
class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ...
@MULTIMODAL_REGISTRY.register_processor(
MultiModalProcessor,
info=MultiModalProcessingInfo,
dummy_inputs=MultiModalDummyInputsBuilder,
)
class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ...
def __getattr__(name: str):
"""Handle imports of non-existent classes with a helpful error message."""
if name not in globals():
raise AttributeError(
"The Transformers modeling backend does not currently have a class to "
f"handle the requested model type: {name}. Please open an issue at "
"https://github.com/vllm-project/vllm/issues/new"
)
return globals()[name]
@@ -1,860 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend base class."""
import os
from collections.abc import Callable, Iterable
from contextlib import contextmanager
from functools import cached_property
from itertools import chain
from operator import attrgetter
from typing import TYPE_CHECKING, Any, NamedTuple
import regex as re
import torch
import transformers
from packaging.version import Version
from torch import nn
from transformers import AutoModel, PreTrainedConfig
from transformers.conversion_mapping import (
WeightRenaming,
get_checkpoint_conversion_mapping,
get_model_conversion_mapping,
)
from vllm.compilation.decorators import DynamicArgDims, support_torch_compile
from vllm.config.utils import getattr_iter
from vllm.distributed import get_pp_group, get_tp_group
from vllm.distributed.utils import get_pp_indices
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention, EncoderOnlyAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.models.interfaces import (
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
SupportsQuant,
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
from vllm.model_executor.models.transformers.fusers.attention import (
VLLM_ATTN_IMPL,
VLLM_MLA_ATTN_IMPL,
)
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
disable_noop_dynamic_rope,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
named_state,
replace_conv_class,
replace_embedding_class,
replace_layernorm_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
ShardId,
WeightsMapper,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
if TYPE_CHECKING:
from transformers import PreTrainedModel
from vllm.config import VllmConfig
try:
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
except ImportError: # MLA support removed from this build
MLAAttention = type("AbsentMLAAttention", (), {})
logger = init_logger(__name__)
VLLM_ATTN_ATTR = "attn"
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
attention module. `"attn"` is chosen deliberately to match the mappings generated by
`maybe_remap_kv_scale_name`."""
class PreTrainedModelClasses(NamedTuple):
decoder: type["PreTrainedModel"]
encoders: dict[str, type["PreTrainedModel"]]
"""Modality -> encoder class, for each modality that has one."""
class Base(
nn.Module,
VllmModel,
SupportsQuant,
SupportsLoRA,
SupportsPP,
SupportsEagle,
SupportsEagle3,
):
hf_to_vllm_mapper: WeightsMapper
# TODO transformers will have a util to get these
embedding_modules = {"embed_tokens": "input_embeddings"}
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
nn.Module.__init__(self)
logger.info("Using Transformers modeling backend.")
self.vllm_config = vllm_config
self.config = vllm_config.model_config.hf_config
self.text_config = self.config.get_text_config()
self.cache_config = vllm_config.cache_config
self.compilation_config = vllm_config.compilation_config
self.device_config = vllm_config.device_config
self.model_config = vllm_config.model_config
self.parallel_config = vllm_config.parallel_config
self.quant_config = vllm_config.quant_config
self.pp_group = get_pp_group()
self.tp_group = get_tp_group()
# Attrs for weight loading (see self.load_weights)
self.ignore_unexpected_prefixes: list[str] = []
"""Ignore unexpected weights whose qualname starts with these prefixes."""
self.ignore_unexpected_suffixes: list[str] = []
"""Ignore unexpected weights whose qualname ends with these suffixes."""
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
self.fusers: dict[str, list[BaseFuser]] = {}
"""Module qualname -> the fusers applied to it, populated
by `recursive_replace` for `_create_attention_instances`."""
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
"""`layer_idx` -> the qualname and fuser of the module computing that
layer's attention, populated by `recursive_replace` for
`_create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
"""Target class for Eagle3 aux hidden state recording."""
self._layer_names: dict[int, str] = {}
"""Mapping from layer index to layer name for Eagle3."""
self._output_aux_hidden_states_kwargs: dict[str, bool] = {}
"""Kwargs to pass to model forward for Eagle3 aux hidden states."""
if self.quant_config:
quant_method_name = self.quant_config.get_name()
# Check for unsupported quantization methods.
if quant_method_name in ("mxfp4", "gpt_oss_mxfp4"):
raise NotImplementedError(
"Transformers modeling backend does "
"not support MXFP4 quantization yet."
)
self._patch_config()
self._decorate_for_torch_compile()
# Init on "meta" to delay allocating GPU tensors
with (
self._mark_model_components(vllm_config),
init_on_device_without_buffers("meta"),
):
from_config_kwargs = self._from_config_kwargs
self.model: PreTrainedModel = AutoModel.from_config(**from_config_kwargs)
# Create weight name to module qualname mapper
self._create_hf_to_vllm_mapper()
# Remove layers not on this pipeline parallel rank
self.pipeline_parallel()
# Skip the device sync of dynamic rope where it can never rescale
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation
self._create_attention_instances(prefix)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
# Upcast weights Transformers always keeps in fp32
self.keep_in_fp32(self.model)
# Pipeline parallel intermediate tensors
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.text_config.hidden_size
)
def _patch_config(self):
"""Patch the config to ensure that the model is created correctly:
- Sets the attention implementation to "vllm" so the attention instances from
`_create_attention_instances` are used
- Sets the dtype to the default torch dtype set by vLLM because Transformers
uses the config dtype when creating the model
"""
self.text_config._attn_implementation = VLLM_ATTN_IMPL
self.config.dtype = torch.get_default_dtype()
@contextmanager
def _mark_model_components(self, vllm_config: "VllmConfig"):
"""Mark language model and tower submodules as `self.model` is created.
Nothing to do in `Base`, `MultiModalMixin` will override."""
yield
@cached_property
def _from_config_kwargs(self) -> dict[str, Any]:
"""The kwargs used to create `self.model`."""
return dict(
config=self.config,
dtype=self.model_config.dtype,
trust_remote_code=self.model_config.trust_remote_code,
)
def _find_encoder_classes(
self, model: "PreTrainedModel"
) -> dict[str, type["PreTrainedModel"]]:
"""Find the encoder class of each modality `model` has one for.
Text models have none. Multi-modal ones override this.
"""
return {}
@cached_property
def _pre_trained_model_classes(self) -> PreTrainedModelClasses:
"""The decoder class and the encoder class of each modality that has one.
Both come from a single throwaway model, since building one is not cheap.
"""
with torch.device("meta"):
model: PreTrainedModel = AutoModel.from_config(**self._from_config_kwargs)
model_classes = PreTrainedModelClasses(
decoder=type(model.get_decoder()),
encoders=self._find_encoder_classes(model),
)
del model
logger.debug("Identified model classes as: %s", model_classes)
return model_classes
def _decorate_cls_for_torch_compile(
self,
cls: type["PreTrainedModel"],
dynamic_arg_dims: DynamicArgDims | None,
enable_if: Callable[["VllmConfig"], bool],
is_encoder: bool,
):
"""Decorate `cls` to indicate to vLLM that it supports torch compile.
Args:
cls: The PreTrainedModel class to decorate.
dynamic_arg_dims: A mapping from argument name to the dynamic dimensions
of the argument. If None, default dynamic arg dims will be used. See
[`support_torch_compile`][vllm.compilation.decorators.support_torch_compile]
for more details.
enable_if: A function which takes in the vLLM config and returns whether
torch compile should be enabled for this class.
is_encoder: Whether the class being decorated is an encoder.
"""
logger.debug(
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
cls.__name__,
"encoder" if is_encoder else "decoder",
dynamic_arg_dims,
)
support_torch_compile(
dynamic_arg_dims=dynamic_arg_dims,
enable_if=enable_if,
is_encoder=is_encoder,
use_aot_compile=False,
)(cls)
def _decorate_for_torch_compile(self):
"""Decorate the model's decoder class to indicate to vLLM that it
supports torch compile if `can_enable_torch_compile` is True."""
self._decorate_cls_for_torch_compile(
cls=self._pre_trained_model_classes.decoder,
# Applied to a PreTrainedModel so the batch dimension will exist
dynamic_arg_dims=DynamicArgDims(
input_ids=1, # shape: [1, seq_len]
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
),
enable_if=can_enable_torch_compile,
is_encoder=False,
)
def _create_hf_to_vllm_mapper(self):
"""Create a WeightsMapper to map checkpoint weight names to module qualnames.
This handles:
- Transformers weight renaming from `WeightRenaming`
- Checkpoints saved with a base model prefix that is not `model`
- Checkpoints saved with no base model prefix
- Any quantization config specific mappings
"""
orig_to_new_renaming: list[WeightRenaming] = []
orig_to_new_regex: dict[re.Pattern, str | None] = {}
mappings = get_model_conversion_mapping(self.model)
# Ensure layers that are part of the model, but not AutoModel, get their mapping
instantiated = {type(m).__name__ for m in self.model.modules()}
for arch in self.config.architectures or []:
if arch not in instantiated:
mappings.extend(get_checkpoint_conversion_mapping(arch) or [])
for mapping in mappings:
# Handle weights which have been renamed in Transformers
if isinstance(mapping, WeightRenaming):
orig_to_new_renaming.append(mapping)
# TODO: Handle WeightConverter to enable layer merging
# Handle unexpected weights which should be ignored
if self.model._keys_to_ignore_on_load_unexpected is not None:
for key in self.model._keys_to_ignore_on_load_unexpected:
orig_to_new_regex[re.compile(key)] = None
# Standardise base model prefix
bmp = self.model.base_model_prefix
expected_bmp = r"model.\1"
# Handle checkpoints saved with different base model prefix
if bmp and bmp != "model":
different_bmp_pattern = re.compile(rf"^{bmp}\.(.+)")
orig_to_new_regex[different_bmp_pattern] = expected_bmp
# Handle direct children of self.model which were saved without the model prefix
direct_children = chain(
self.model.named_children(),
self.model.named_parameters(recurse=False),
self.model.named_buffers(recurse=False),
)
model_children = "|".join(name for name, _ in direct_children)
missing_bmp_pattern = re.compile(rf"^(?!model\.)(({model_children}).*)")
orig_to_new_regex[missing_bmp_pattern] = expected_bmp
# Handle weights saved as direct children of self.model which no longer are
unexpected_bmp_pattern = re.compile(rf"^(model\.)((?!{model_children}).+)")
orig_to_new_regex[unexpected_bmp_pattern] = r"\2"
# Handle lm_head which was saved inside the base model
nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)")
orig_to_new_regex[nested_lm_head_pattern] = r"\2"
self.hf_to_vllm_mapper = WeightsMapper(
orig_to_new_renaming=orig_to_new_renaming,
orig_to_new_regex=orig_to_new_regex,
)
# Apply mapping to quantization config if needed
self._maybe_apply_model_mapping()
def _get_tie_word_embeddings(self):
"""Check if the model has tied word embeddings."""
# Models created with Transformers v4 and v5 will store this in different places
tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False)
tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False)
return tie_word_embeddings_v4 or tie_word_embeddings_v5
def pipeline_parallel(self):
"""Apply the model's pipeline parallelization plan."""
if self.pp_group.world_size <= 1:
return
if self.model.supports_pp_plan:
module = self.model
names = list(module._pp_plan.keys())
else:
module = self.model.get_decoder()
has_parameters = lambda m: next(m.parameters(), None) is not None
names = [n for n, c in module.named_children() if has_parameters(c)]
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
logger.warning_once(
"%s does not define a pipeline parallel plan. The Transformers "
"modeling backend will infer the split from the layers of %s in order "
"of declaration and keep parameter-free modules on every rank. This "
"may fail if the model's structure is non-standard. %s",
type(self.model).__name__,
type(module).__name__,
tip,
)
module_lists = []
module_list_idx = None
for i, name in enumerate(names):
# attrgetter in case the module is nested (e.g. "text_model.layers")
if isinstance(attrgetter(name)(module), nn.ModuleList):
module_lists.append(name)
module_list_idx = i
if len(module_lists) > 1:
raise ValueError(
"Pipeline parallel of models with multiple `ModuleList`s "
"in the base model are not supported yet!"
)
if module_list_idx is None:
raise ValueError(f"Could not find `ModuleList` in {type(module)}")
# Layers before module list
for name in names[:module_list_idx]:
if self.pp_group.is_first_rank or (
self._get_tie_word_embeddings() and self.pp_group.is_last_rank
):
continue
# attrsetter in case the module is nested (e.g. "text_model.embed_tokens")
attrsetter(name)(module, PPMissingLayer())
# Module list
start_layer, end_layer = get_pp_indices(
self.text_config.num_hidden_layers,
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
layers_name = names[module_list_idx]
# attrgetter in case the module is nested (e.g. "text_model.layers")
layers = attrgetter(layers_name)(module)
for i in range(len(layers)):
if start_layer <= i and i < end_layer:
continue
layers[i] = PPMissingLayer()
# Layers after module list
for name in names[module_list_idx + 1 :]:
# Modules that should be on last rank
if not self.pp_group.is_last_rank:
# attrsetter in case the module is nested (e.g. "text_model.norm")
attrsetter(name)(module, PPMissingLayer())
def _vocab_embeddings(self) -> set[nn.Embedding]:
"""The `nn.Embedding`s in `self.model` which hold a vocab table."""
def vocab_sizes(config: PreTrainedConfig):
for key, value in vars(config).items():
if isinstance(value, PreTrainedConfig):
yield from vocab_sizes(value)
elif "vocab_size" in key and isinstance(value, int):
yield value
sizes = set(vocab_sizes(self.config))
# `get_input_embeddings` may return a module which composes the `nn.Embedding`,
# or a `PPMissingLayer` if the embeddings are not on this pipeline stage
embeddings = {
module
for module in self.model.get_input_embeddings().modules()
if isinstance(module, nn.Embedding)
}
embeddings.update(
module
for module in self.model.modules()
if isinstance(module, nn.Embedding) and module.num_embeddings in sizes
)
return embeddings
def recursive_replace(self):
"""Recursively replace modules in the model as needed.
Currently, this replaces:
- GLUs with a fused `MergedColumnParallelLinear` + `...AndMul`
- Attention QKV projections with a fused `QKVParallelLinear` + split
- `nn.Linear` with vLLM's tensor parallel linear classes
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
- `nn.LayerNorm` with vLLM's `LayerNorm`
"""
tp_plan = self.model.tp_plan or {}
if not tp_plan and self.tp_group.world_size > 1:
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
logger.warning_once(
"%s does not define a tensor parallel plan. The Transformers modeling "
"backend will shard the model the best it can during graph fusion and "
"replicate the rest. This may be suboptimal or fail if the model does "
"not fuse cleanly. %s",
type(self.model).__name__,
tip,
)
# Prefix the patterns because we always start from `self.model`
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
# Detect fusable patterns once per module class (cached, so this is cheap)
fusers = Fusers(self.model, self.vllm_config)
vocab_embeddings = self._vocab_embeddings()
orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {}
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
"""Register a fused layer's mappings just before it is built."""
self.fusers.setdefault(prefix, []).append(fuser)
if (
isinstance(fuser, AttentionFuser)
and (index := fuser.layer_index(module)) is not None
):
if index in self.attention_fusers:
other, _ = self.attention_fusers[index]
raise ValueError(
f"Layer {index} dispatches attention from more than one "
f"module ({other} and {prefix}), which one vLLM attention "
"layer cannot serve."
)
self.attention_fusers[index] = (prefix, fuser)
orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix))
packed_modules_mapping = fuser.packed_modules_mapping
self.packed_modules_mapping.update(packed_modules_mapping)
if self.quant_config is not None:
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
new_module = child_module
qual_name = maybe_prefix(prefix, child_name)
if (
isinstance(module, nn.ModuleList)
and len(module) == self.text_config.num_hidden_layers
):
# Populate Eagle3 attrs
self._target_class = type(child_module)
layer_name = qual_name.removeprefix("model.")
self._layer_names[int(child_name)] = layer_name
# MTP weights should not be loaded into the base model
num_hidden_layers = self.text_config.num_hidden_layers
names = (
"n_predict", # Override from SpeculativeConfig
"num_nextn_predict_layers", # Most models
"mtp_num_hidden_layers", # Qwen 3.5
)
n_predict = getattr_iter(self.text_config, names, 0)
for i in range(num_hidden_layers, num_hidden_layers + n_predict):
mtp_prefix = f"{prefix}.{i}."
if mtp_prefix not in self.ignore_unexpected_prefixes:
self.ignore_unexpected_prefixes.append(mtp_prefix)
# Replace modules as needed
if isinstance(child_module, nn.Linear):
generator = (p for p in tp_plan if re.match(p, qual_name))
pattern = next(generator, None)
# Some weight loaders expect all linear layers to inherit
# LinearBase, so we set a default style which causes any
# unspecified layers to be replaced with ReplicatedLinear
style = tp_plan.get(pattern, "replicate")
new_module = replace_linear_class(
child_module, style, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
new_module = replace_conv_class(child_module)
elif child_module in vocab_embeddings:
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, nn.LayerNorm):
new_module = replace_layernorm_class(child_module)
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
# MoERunner can contain aliases of shared experts and gates,
# so we don't want to recurse into it and break weight loading.
_recursive_replace(child_module, prefix=qual_name)
if new_module is not child_module:
setattr(module, child_name, new_module)
log_replacement(qual_name, child_module, new_module)
_recursive_replace(self.model, prefix="model")
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
def _create_attention_instances(self, prefix: str = ""):
"""Create `Attention` instances to inform KV cache allocation."""
text_config = self.text_config
attn_cls = self._get_attn_cls()
# kv_lora_rank indicates that this is an MLA model
if getattr(text_config, "kv_lora_rank", None) is not None:
if attn_cls is MLAAttention:
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
else:
# MLA model not using MLAAttention: recompute head_size for full attn
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0)
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
self.model_config.model_arch_config.head_size = qk_head_dim
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
layer_types = getattr(text_config, "layer_types", None)
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
if i not in self.attention_fusers:
layer = (
f"{i} ({layer_types[i]})"
if layer_types and i < len(layer_types)
else str(i)
)
raise ValueError(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."
)
attn_prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(attn_prefix)
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
arch_config = self.model_config.model_arch_config[i]
num_heads = self.model_config.get_num_attention_heads(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
if (scale := attn_fuser.scale(attn_module)) is None:
# Default to Llama scale if AttentionFuser couldn't identify it
scale = head_size**-0.5
kwargs = dict(
num_heads=num_heads,
scale=scale,
cache_config=self.cache_config,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
)
if attn_cls is MLAAttention:
mla_fuser = next(
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
None,
)
if mla_fuser is None:
raise ValueError(
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
"of the model did, so its latent KV cache cannot be allocated."
)
from vllm.model_executor.layers.attention.mla_attention import (
get_mla_dims,
)
dims = get_mla_dims(self.model_config)
kwargs.update(
qk_nope_head_dim=dims.qk_nope_head_dim,
qk_rope_head_dim=dims.qk_rope_head_dim,
v_head_dim=dims.v_head_dim,
q_lora_rank=dims.q_lora_rank,
kv_lora_rank=dims.kv_lora_rank,
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
)
else:
kwargs.update(
head_size=head_size,
num_kv_heads=self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
),
logits_soft_cap=logits_soft_cap,
)
# Handle interleaved sliding window attention
if layer_types and layer_types[i] == "sliding_attention":
kwargs["per_layer_sliding_window"] = text_config.sliding_window
# Handle attention sinks
if (sinks := attn_fuser.sinks(attn_module)) is not None:
kwargs["sinks"] = sinks
attn_instance = attn_cls(**kwargs)
# Attach the instance to the module it serves, which is the only way it
# is in the module tree at all: a plain dict is invisible to
# `nn.Module.__setattr__`. Being in the tree runs its
# `process_weights_after_loading` and moves its KV-scale buffers with
# `model.to(device)`, and lets the attention interface read it off the
# module rather than index by `module.layer_idx`, an `int` that bakes
# layer identity into the traced graph and so costs one compiled
# artifact per layer.
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
def _get_attn_cls(self) -> type[AttentionLayerBase]:
"""Return the `Attention` class to use for this model's layers."""
# In encoder models, the attention layers will have `is_causal=False`
is_encoder = lambda module: not getattr(module, "is_causal", True)
has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
is_multimodal = lambda config: config != config.get_text_config()
# vLLM does not support encoder-decoder models, so if any encoder layer is
# found in a text only model, we assume the whole model is an encoder model
if has_encoder(self.model) and not is_multimodal(self.config):
return EncoderOnlyAttention
if self.model_config.use_mla:
if any(
isinstance(fuser, MLAFuser)
for fusers in self.fusers.values()
for fuser in fusers
):
return MLAAttention
logger.warning_once(
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
"MLA attention layers. Falling back to full attention with a padded "
"`value` head dimension."
)
os.environ["VLLM_MLA_DISABLE"] = "1"
return Attention
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
"""If a `parameter` is on the `meta` device, then its parent
`module` is the original module created by:
```python
with torch.device("meta"):
self.model: "PreTrainedModel" = AutoModel.from_config(...)
```
"""
dtype = dtype or self.model_config.dtype
device = self.device_config.device
def _init_parameters(module: nn.Module):
for name, param in module.named_parameters(recurse=False):
# Already on device, nothing to do
if param.device != torch.device("meta"):
continue
# Already a vLLM parameter, nothing to do
if hasattr(param, "weight_loader"):
continue
data = torch.empty_like(param.data, dtype=dtype, device=device)
setattr(module, name, nn.Parameter(data=data))
for child in module.children():
_init_parameters(child)
_init_parameters(module)
def keep_in_fp32(self, module: nn.Module):
"""Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would."""
if self.model_config.dtype not in (torch.float16, torch.bfloat16):
return
fragments = getattr(module, "_keep_in_fp32_modules_strict", None)
if not fragments:
return
pattern = re.compile("|".join(re.escape(f) for f in fragments))
for name, tensor in named_state(module):
if not hasattr(tensor, "weight_loader") and pattern.search(name):
tensor.data = tensor.data.to(torch.float32)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.get_input_embeddings()(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | IntermediateTensors:
if not self.pp_group.is_first_rank:
assert intermediate_tensors is not None
input_ids = None
inputs_embeds = intermediate_tensors["hidden_states"]
# Add batch dimension before entering Transformers model
if input_ids is not None and input_ids.ndim == 1:
# [seq_len] -> [1, seq_len]
input_ids = input_ids[None, ...]
if inputs_embeds is not None and inputs_embeds.ndim == 2:
# [seq_len, hidden_size] -> [1, seq_len, hidden_size]
inputs_embeds = inputs_embeds[None, ...]
if positions.ndim == 1:
# [seq_len] -> [1, seq_len]
positions = positions[None, ...]
# Transformers models expect either input_ids or inputs_embeds, but not both
if input_ids is not None and inputs_embeds is not None:
input_ids = None
outputs = self.model(
input_ids=input_ids,
inputs_embeds=inputs_embeds,
use_cache=False,
position_ids=positions,
return_dict=False,
**self._output_aux_hidden_states_kwargs,
**kwargs,
)
# Remove batch dimension after exiting Transformers model
hidden_states = outputs[0][0, ...]
if self._output_aux_hidden_states_kwargs:
aux_hidden_states = [x[0][0, ...] for x in outputs[1:]]
if not self.pp_group.is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
if self._output_aux_hidden_states_kwargs and len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@staticmethod
def check_version(min_version: str, feature: str):
installed = Version(transformers.__version__)
required = Version(min_version)
if installed < required:
raise ImportError(
f"Transformers modeling backend requires transformers>={required} "
f"for {feature}, but got {installed}"
)
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
from transformers.utils.output_capturing import (
OutputRecorder,
maybe_install_capturing_hooks,
)
# The default value in PreTrainedModel is None
if self.model._can_record_outputs is None:
self.model._can_record_outputs = {}
target_class = self._target_class
for layer in layers:
# layer - 1 because we want the input to the layer
layer_name = self._layer_names[layer - 1]
layer_key = f"aux_hidden_state_{layer}"
aux_hidden_state_i = OutputRecorder(target_class, layer_name=layer_name)
self.model._can_record_outputs[layer_key] = aux_hidden_state_i
self._output_aux_hidden_states_kwargs[f"output_{layer_key}"] = True
# Ensure that the capture hooks are installed before dynamo traces the model
maybe_install_capturing_hooks(self.model)
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
num_layers = self.text_config.num_hidden_layers
return (2, num_layers // 2, num_layers - 3)
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixin for causal language models."""
from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
from .base import Base
if TYPE_CHECKING:
from vllm.config import VllmConfig
class CausalMixin(VllmModelForTextGeneration, Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip VllmModelForTextGeneration.__init__ and call the next class in MRO
super(VllmModelForTextGeneration, self).__init__(
vllm_config=vllm_config, prefix=prefix
)
tie_word_embeddings = self._get_tie_word_embeddings()
if self.pp_group.is_last_rank:
self.lm_head = ParallelLMHead(
self.text_config.vocab_size,
self.text_config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if tie_word_embeddings:
for module in self.model.get_input_embeddings().modules():
if isinstance(module, VocabParallelEmbedding):
self.lm_head = self.lm_head.tie_weights(module)
break
self.logits_processor = LogitsProcessor(
self.text_config.vocab_size,
scale=getattr(self.text_config, "logit_scale", 1.0),
soft_cap=getattr(self.text_config, "final_logit_softcapping", None),
)
else:
self.lm_head = PPMissingLayer()
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
lm_head_bias = set()
load_device = self.vllm_config.load_config.device or self.device_config.device
def auto_load_lm_head_bias(weights):
for name, weight in weights:
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
# load_weights is called outside vLLM's device context,
# so we must enter it again when registering the bias.
with torch.device(load_device):
self.lm_head._register_bias()
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
lm_head_bias.add(name)
else:
yield name, weight
return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias
def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
return logits
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser detection for the Transformers modeling backend."""
from collections import UserDict
from typing import TYPE_CHECKING, TypeVar
from cachetools import cached
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
AttentionFuser,
BaseFuser,
GLUFuser,
MergedColumnParallelFuser,
MLAFuser,
PackedQKVFuser,
QKVFuser,
RewriteFuser,
RMSNormFuser,
)
from vllm.model_executor.models.transformers.fx_utils import trace
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
F = TypeVar("F", bound=BaseFuser)
def key(module: nn.Module) -> tuple:
"""Cache key for `get_fusers`. Considers module type and its immediate children."""
return (type(module), tuple(name for name, _ in module.named_children()))
FUSERS: tuple[type[BaseFuser], ...] = (
# Order these by priority, e.g. QKV would be preferred over MergedColumnParallel
MLAFuser,
GLUFuser,
QKVFuser,
PackedQKVFuser,
MergedColumnParallelFuser,
RMSNormFuser,
# Put fusers that don't redefine forward after this comment
AttentionFuser,
)
"""Every fuser, in priority order: those that redefine the forward first, then those
that leave it alone. A new fuser is added here."""
@cached(cache={}, key=key)
def get_fusers(module: nn.Module) -> list[BaseFuser]:
"""Every fuser that could apply to `module`'s class (cached), in `FUSERS` order."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
is_leaf = next(module.children(), None) is None
graph = trace(module) if n_linear >= 2 or is_leaf else None
# Every redefining candidate that matches is cached, not just the first:
# instances of this class can be shaped heterogeneously, so which one is
# usable is decided per instance in Fusers.__getitem__, not here. A forward
# can only be rewritten from a trace of it, so each starts fresh regardless.
fusers: list[BaseFuser] = []
for fuser_cls in FUSERS:
if fuser_cls.redefines_forward and graph is None:
continue
if (fuser := fuser_cls.match(graph, module)) is None:
continue
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
logger.debug(
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
continue
fusers.append(fuser)
# A norm we could not match structurally is left unfused; flag likely misses.
if (
module.__class__.__name__.endswith("RMSNorm")
and graph is not None
and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
):
logger.warning_once(
"%s looks like an RMSNorm but its computation did not match the "
"expected pattern, so it was left unfused.",
module.__class__.__name__,
)
return fusers
def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
"""The first `fuser_cls` that applies to `module`, if one does."""
return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
class Fusers(UserDict):
"""Mapping from module class and shape to fusers, for all fusable modules."""
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
self.vllm_config = vllm_config
super().__init__({key(m): get_fusers(m) for m in model.modules()})
def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
"""The fusers this instance can take, in the order to apply them."""
chosen: list[BaseFuser] = []
redefined = False
for fuser in self.data.get(key(m), ()):
# Only one candidate may rewrite m's forward; take the first
# (highest-priority) one that validates for this instance.
if fuser.redefines_forward and redefined:
continue
if not fuser.validate(m, self.vllm_config):
continue
redefined = redefined or fuser.redefines_forward
chosen.append(fuser)
return chosen
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
StackedFuser,
)
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.merged_column import (
MergedColumnParallelFuser,
)
from vllm.model_executor.models.transformers.fusers.mla import MLAFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"AttentionFuser",
"BaseFuser",
"RewriteFuser",
"StackedFuser",
"GLUFuser",
"MergedColumnParallelFuser",
"MLAFuser",
"MoEBlockFuser",
"PackedQKVFuser",
"QKVFuser",
"RMSNormFuser",
]
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention fuser: the module that dispatches to the attention interface."""
import ast
import inspect
import textwrap
from collections.abc import Callable
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
import torch
from torch import fx, nn
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.utils import set_weight_attrs
if TYPE_CHECKING:
from vllm.config import VllmConfig
VLLM_ATTN_IMPL = "vllm"
VLLM_MLA_ATTN_IMPL = "vllm_mla"
def _is_interface_lookup(node: ast.expr | None) -> bool:
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
if isinstance(node, ast.Call):
node = node.func
if not isinstance(node, (ast.Attribute, ast.Subscript)):
return False
return (
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
)
@cache
def interface_call(forward: Callable) -> ast.Call | None:
"""The attention interface call in `forward`, if it makes exactly one."""
try:
source = inspect.getsource(inspect.unwrap(forward))
tree = ast.parse(textwrap.dedent(source))
except (AttributeError, OSError, SyntaxError, TypeError):
return None
# Find the names of the local variables that read from the interface lookup
names: set[str] = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
continue
if not _is_interface_lookup(node.value):
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
names.update(t.id for t in targets if isinstance(t, ast.Name))
# Find the calls to those names, and return the one if there is exactly one
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in names
]
return calls[0] if len(calls) == 1 else None
def _resolve(node: ast.expr, module: nn.Module) -> object:
"""The value of `node` on `module`, for literals and `self.<attr>`."""
if isinstance(node, ast.Constant):
return node.value
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
return getattr(module, node.attr, None)
return None
@dataclass
class AttentionFuser(BaseFuser):
"""A module that dispatches through the Transformers attention interface."""
redefines_forward: ClassVar[bool] = False
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None = None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
s_aux_expr: ast.expr | None = None
"""Source of the `s_aux=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@classmethod
def match(
cls, graph: fx.Graph | None, module: nn.Module
) -> "AttentionFuser | None":
if (call := interface_call(type(module).forward)) is None:
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"]
s_aux_expr = s_aux[0] if len(s_aux) == 1 else None
return cls(
source_cls=type(module).__name__,
scale_expr=scale_expr,
s_aux_expr=s_aux_expr,
)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
config = getattr(module, "config", None)
# Only patched in the text config, this excludes attention based mm encoders
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
if (sinks := self.sinks(module)) is not None:
assert isinstance(self.s_aux_expr, ast.Attribute)
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
device = vllm_config.device_config.device
data = torch.empty(size, dtype=sinks.dtype, device=device)
sinks_param = nn.Parameter(data, requires_grad=False)
set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)})
setattr(module, self.s_aux_expr.attr, sinks_param)
return module
def layer_index(self, module: nn.Module) -> int | None:
"""The layer `module` computes attention for, if it declares one."""
layer_idx = getattr(module, "layer_idx", None)
return layer_idx if isinstance(layer_idx, int) else None
def scale(self, module: nn.Module) -> float | None:
"""The softmax scale `module` passes to the interface, or `None`."""
if self.scale_expr is None:
return None
scale = _resolve(self.scale_expr, module)
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
expression = ast.unparse(self.scale_expr)
raise ValueError(
f"Cannot resolve attention scaling expression {expression!r} in "
f"{type(module).__name__}."
)
return float(scale)
def sinks(self, module: nn.Module) -> nn.Parameter | None:
"""The per-head sink tensor `module` passes to the interface, or `None`."""
if self.s_aux_expr is None:
return None
s_aux = _resolve(self.s_aux_expr, module)
if not isinstance(s_aux, (nn.Parameter, type(None))):
expression = ast.unparse(self.s_aux_expr)
raise ValueError(
f"Cannot resolve attention s_aux expression {expression!r} in "
f"{type(module).__name__}."
)
return s_aux
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base classes for the Transformers backend fusers."""
import ast
import types
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.transformers.fx_utils import (
aliasing_names,
bypass_existence_guard,
replace_expr,
self_call_and_refs,
written_names,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
redefines_forward: ClassVar[bool] = True
"""Whether `fuse` gives the module a different forward,
by rewriting its source or by returning a different module in its place."""
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
@classmethod
@abstractmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "BaseFuser | None":
"""Match the pattern in `graph`, returning a fuser if found."""
@abstractmethod
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether this fuser can be applied to this `module` instance."""
@abstractmethod
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built)."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
(none unless it stacks weights)."""
return {}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`packed_modules_mapping` entries this fuser contributes (none unless
it stacks weights)."""
return {}
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
"""The head size of `module`, which head counts are derived from.
Prefer the module's own `head_dim`, which Transformers sets per instance:
the model-wide value is the largest head size across layers, so on a
heterogeneous checkpoint it would divide out the wrong number of heads, and
it is the text head size, so it does not describe a vision tower at all.
"""
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
def local_output_sizes(merged_name: str) -> str:
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
merged = f"self.{merged_name}"
return f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
@dataclass
class RewriteFuser(BaseFuser):
"""A fuser that rewrites the module's forward and rebinds it.
`match` and `update_forward` analyse the class once; `fuse` swaps the
submodules and binds the compiled forward on an instance in place, so it
keeps its class and any attribute the fusion does not consume.
"""
source_cls: str
"""Class of the HF module the fused projections belonged to (for logging)."""
fused_forward: Callable = field(init=False, repr=False)
"""The compiled rewritten forward, set by `update_forward`."""
@abstractmethod
def update_forward(self, module: nn.Module) -> None:
"""Rewrite and compile `type(module)`'s forward source.
Raises if the source does not admit the rewrite (fusion is then skipped).
"""
@abstractmethod
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace `module`'s submodules with their vLLM equivalents."""
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@dataclass
class StackedFuser(RewriteFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it."""
merged_cls_name: ClassVar[str]
"""Name of the vLLM class the merged projection becomes (for logging)."""
@property
@abstractmethod
def merged_name(self) -> str:
"""Attribute name of the merged module created by `update_attrs`."""
def info(self, name: str) -> str:
sources = " + ".join(shard for shard, _ in self.shards)
return (
f"Fused: {sources} ({name}: {self.source_cls}) -> "
f"{self.merged_name} ({self.merged_cls_name})"
)
@property
@abstractmethod
def shards(self) -> list[tuple[str, ShardId]]:
"""Each projection's original name and its shard id in the merged module.
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
so only this exact layer is remapped, never a same-named projection
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
merged = maybe_prefix(prefix, self.merged_name)
return {
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`{merged_name: [projection names]}` so quantization can unpack the
fused layer into its per-shard configs."""
return {self.merged_name: [name for name, _ in self.shards]}
def _unguarded_calls(
self, funcdef: ast.FunctionDef, names: Iterable[str]
) -> list[ast.Call]:
"""One `self.<name>(arg)` call per projection, existence guards folded.
`update_attrs` deletes the projections it merges, so any reference to one beyond
its call site must be a guard on its existence; folding those to their constant
value keeps the rewritten forward off a name that no longer exists. A reference
that is not such a guard raises, so fusion is skipped."""
calls = []
for name in names:
call, refs = self_call_and_refs(funcdef, name)
for ref in refs:
bypass_existence_guard(funcdef, ref, name)
calls.append(call)
return calls
def _check_input_stable(
self,
funcdef: ast.FunctionDef,
module: nn.Module,
calls: list[ast.Call],
block: list[ast.stmt],
indices: list[int],
) -> None:
"""Raise unless hoisting the merged GEMM preserves what it reads.
Fusing moves every projection to one call at `min(indices)`, so the
merged GEMM reads the input once, up front, where the last of `calls`
would have read it later. That holds only if nothing in between changes
the input, and a change need not name it: writing any view that shares
its storage changes it too. Both halves of the check over-approximate,
since a false hit costs a fusion while a miss returns wrong numbers.
"""
arg_names = {
node.id
for node in ast.walk(calls[0].args[0])
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
}
# Protect the names the input reads and anything that may alias them.
tracked = aliasing_names(funcdef, arg_names, module)
# Any of them rebound, written through, or passed to a call that writes
# its argument would leave the hoisted GEMM reading a different value.
region = block[min(indices) : max(indices) + 1]
if tracked & written_names(region):
raise ValueError("projection input is rebound or mutated before all calls")
def _splice_merged_split(
self,
funcdef: ast.FunctionDef,
calls: list[ast.Call],
block: list[ast.stmt],
index: int,
) -> None:
"""Insert `temps = self.<merged_name>(arg).split(sizes, -1)` at
`block[index]` and replace each of `calls` with its temp name.
`calls` must share one input argument; `block[index]` must be where
they are (or would be) evaluated. Raises if a generated temporary
would shadow an existing name in `funcdef`.
"""
temps = [f"_vllm_merged_{i}" for i in range(len(calls))]
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
if names & set(temps):
raise ValueError("fused temporaries would shadow existing names")
targets = ", ".join(temps)
sections = local_output_sizes(self.merged_name)
source = f"{targets} = self.{self.merged_name}(__arg__).split({sections}, -1)"
assign = ast.parse(source).body[0]
arg = next(
node
for node in ast.walk(assign)
if isinstance(node, ast.Name) and node.id == "__arg__"
)
replace_expr(assign, arg, calls[0].args[0])
ast.copy_location(assign, block[index])
block.insert(index, assign)
for call, temp in zip(calls, temps):
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
@@ -1,191 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLU projection fuser: `act(gate(x)) * up(x)` -> a fused gate/up linear."""
import ast
import operator
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from transformers.activations import ACT2CLS
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
from vllm.model_executor.models.transformers.fusers.merged_column import (
MergedColumnParallelFuser,
)
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
find_node,
is_linear,
peel,
recover_forward,
replace_expr,
single_self_call,
)
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
CLS2ACT: dict[type, list[str]] = {}
for _act_name, _act_cls in ACT2CLS.items():
if isinstance(_act_cls, tuple):
_act_cls = _act_cls[0]
CLS2ACT.setdefault(_act_cls, []).append(_act_name)
ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys())
@dataclass
class GLUFuser(MergedColumnParallelFuser):
"""Fuser for the GLU pattern `act(gate(x)) * up(x)`."""
act_name: str
down_name: str | None
merged_name: ClassVar[str] = "gate_up_proj"
@property
def gate_name(self) -> str:
return self.linear_names[0]
@property
def up_name(self) -> str:
return self.linear_names[1]
@classmethod
def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool:
"""Is node `act(gate(x))` where `gate` is linear and `act` is not linear."""
return (
node.op == "call_module"
and not is_linear(node, module)
and len(node.args) == 1
and isinstance(node.args[0], fx.Node)
and is_linear(node.args[0], module)
)
@classmethod
def _get_glu_nodes(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node, fx.Node] | None:
"""Search graph for the GLU pattern `act(gate(x)) * up(x)`."""
for mul in graph.nodes:
if (
mul.op == "call_function"
and mul.target == operator.mul
and len(mul.args) == 2
and all(isinstance(arg, fx.Node) for arg in mul.args)
):
a, b = mul.args
if cls._is_act_of_gate(a, module) and is_linear(b, module):
act, gate, up = a, a.args[0], b
elif cls._is_act_of_gate(b, module) and is_linear(a, module):
act, gate, up = b, b.args[0], a
else:
continue
if (
all(len(args) == 1 for args in (gate.args, up.args))
and isinstance(x := gate.args[0], fx.Node)
and x is up.args[0]
):
return act, gate, up, mul
return None
@staticmethod
def _get_act_and_mul_name(act: nn.Module) -> str | None:
"""Get the name of `act` if it has an `...AndMul` equivalent."""
for name in CLS2ACT.get(type(act), []):
if name in ACT_AND_MUL_NAMES:
return name
# nn.GELU is not in ACT2CLS, but could be in model code
if type(act) is nn.GELU:
return "gelu_pytorch_tanh" if act.approximate == "tanh" else "gelu"
return None
@classmethod
def _get_act_and_mul(cls, act: nn.Module) -> nn.Module:
"""Get the `...AndMul` equivalent of a Transformers activation module."""
if name := cls._get_act_and_mul_name(act):
return get_act_and_mul_fn(name)
raise ValueError(f"No AndMul equivalent for {type(act)}")
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "GLUFuser | None":
if (glu_nodes := cls._get_glu_nodes(graph, module)) is None:
return None
act_node, gate_node, up_node, mul_node = glu_nodes
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
down_node = find_node(graph, predicate)
return cls(
source_cls=type(module).__name__,
act_name=act_node.target,
linear_names=(gate_node.target, up_node.target),
down_name=down_node.target if down_node is not None else None,
)
def update_forward(self, module: nn.Module) -> None:
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
funcdef, fn = recover_forward(type(module))
act_call = single_self_call(funcdef, self.act_name)
calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name))
gate_call, up_call = calls
if act_call.args[0] is not gate_call:
raise ValueError("activation does not directly wrap the gate")
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
raise ValueError("gate and up inputs are written differently")
muls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.BinOp)
and isinstance(node.op, ast.Mult)
and {id(node.left), id(node.right)} == {id(act_call), id(up_call)}
]
if len(muls) != 1:
raise ValueError("no multiply of the activation and up projection")
# act(gate(x)) * up(x) -> act(gate_up(x))
assert isinstance(gate_call.func, ast.Attribute)
gate_call.func.attr = self.merged_name
replace_expr(funcdef, muls[0], act_call)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
if not super().validate(module, vllm_config):
return False
# An AndMul kernel splits its input in half, so gate and up must match.
gate = module.get_submodule(self.gate_name)
up = module.get_submodule(self.up_name)
if gate.out_features != up.out_features:
logger.debug("%s and %s differ in size; skipping fusion", gate, up)
return False
act = module.get_submodule(self.act_name)
if self._get_act_and_mul_name(act) is None:
logger.debug("%s has no AndMul equivalent; skipping fusion", act)
return False
return True
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
super().update_attrs(module, prefix, vllm_config)
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
setattr(module, self.act_name, act_fn)
# If there is a down projection, we know it must be rowwise.
if self.down_name is not None:
down_prefix = maybe_prefix(prefix, self.down_name)
down = module.get_submodule(self.down_name)
new_down = replace_linear_class(
down, "rowwise", vllm_config.quant_config, prefix=down_prefix
)
setattr(module, self.down_name, new_down)
log_replacement(down_prefix, down, new_down)
@@ -1,145 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser for parallel linear projections."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
block_chain,
compile_forward,
is_linear,
recover_forward,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class MergedColumnParallelFuser(StackedFuser):
"""Fuser for merging column-parallel linear projections."""
linear_names: tuple[str, ...]
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
@property
def merged_name(self) -> str:
"""Programmatic name for the merged projection, based on the original names."""
# len is used to disambiguate names like `a_b` + `c` vs `a` + `b_c`
parts = "_".join(f"{len(name)}_{name}" for name in self.linear_names)
return f"merged_proj_{parts}"
@property
def shards(self) -> list[tuple[str, ShardId]]:
return [(name, index) for index, name in enumerate(self.linear_names)]
@staticmethod
def _names(group: list[fx.Node]) -> tuple[str, ...] | None:
names = tuple(str(node.target) for node in group)
if len(set(names)) != len(names) or any("." in name for name in names):
return None
return names
@classmethod
def match(
cls, graph: fx.Graph, module: nn.Module
) -> "MergedColumnParallelFuser | None":
"""Fuse the module's sibling linears when there is only one such group."""
by_input: dict[fx.Node, list[fx.Node]] = {}
for node in graph.nodes:
if (
is_linear(node, module)
and len(node.args) == 1
and not node.kwargs
and isinstance(node.args[0], fx.Node)
# Like QKVFuser/PackedQKVFuser: this fuser has no head or
# TP-replication awareness, so it must not absorb a QKV-shaped
# pattern (e.g. behind a norm) a more specific fuser declined.
and node.args[0].op == "placeholder"
):
by_input.setdefault(node.args[0], []).append(node)
# A group whose members are not distinct direct children is dropped:
# the source rewrite addresses each projection as `self.<name>` once.
groups = [
(group, names)
for group in by_input.values()
if len(group) >= 2 and (names := cls._names(group)) is not None
]
if len(groups) > 1:
logger.debug(
"%s has %d fusable sibling-linear groups; skipping fusion "
"since which one to merge is ambiguous",
type(module).__name__,
len(groups),
)
if len(groups) != 1:
return None
_, names = groups[0]
fuser = cls(source_cls=type(module).__name__, linear_names=names)
return None if hasattr(module, fuser.merged_name) else fuser
def update_forward(self, module: nn.Module) -> None:
"""Replace the parallel calls with one merged call and split."""
funcdef, fn = recover_forward(type(module))
calls = self._unguarded_calls(funcdef, self.linear_names)
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
raise ValueError("parallel linears read different inputs")
chains = [block_chain(funcdef.body, call) for call in calls]
if any(not chain for chain in chains):
raise ValueError("parallel linear calls not found in the function body")
blocks = [chain[-1] for chain in chains]
if len(set(id(block) for block, _ in blocks)) != 1:
raise ValueError("parallel linear calls are in different blocks")
block = blocks[0][0]
indices = [index for _, index in blocks]
index = min(indices)
self._check_input_stable(funcdef, module, calls, block, indices)
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
self._splice_merged_split(funcdef, calls, block, index)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Check that the parallel linears are compatible for merging."""
linear_layers = [module.get_submodule(name) for name in self.linear_names]
tp_size = vllm_config.parallel_config.tensor_parallel_size
return (
len(linear_layers) >= 2
and len(set(linear.in_features for linear in linear_layers)) == 1
and len(set(linear.bias is None for linear in linear_layers)) == 1
and all(linear.out_features % tp_size == 0 for linear in linear_layers)
)
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace the module's parallel linears with one merged projection."""
linear_modules = [module.get_submodule(name) for name in self.linear_names]
merged = MergedColumnParallelLinear(
input_size=linear_modules[0].in_features,
output_sizes=[linear.out_features for linear in linear_modules],
bias=linear_modules[0].bias is not None,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
)
setattr(module, self.merged_name, merged)
for name in self.linear_names:
delattr(module, name)
logger.debug(
"%s -> %s: %s",
", ".join(f"{n}: {m}" for n, m in zip(self.linear_names, linear_modules)),
self.merged_name,
merged,
)
@@ -1,303 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MLA fuser: adapt a Transformers MLA attention module for vLLM's MLA layer."""
import ast
import inspect
import textwrap
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
downstream_linear,
is_linear,
recover_forward,
replace_expr,
returned_linear,
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
Style,
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
def _consumes_placeholder(node: fx.Node) -> bool:
"""Whether `node` is a linear applied directly to a `forward` input."""
return (
len(node.args) == 1
and isinstance(node.args[0], fx.Node)
and node.args[0].op == "placeholder"
)
def _norm_size(norm: nn.Module) -> int:
weight = getattr(norm, "weight", None)
return weight.numel() if weight is not None else -1
def _is_rms_norm(module: nn.Module) -> bool:
"""Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher."""
# Go via `get_fuser` because it caches
from vllm.model_executor.models.transformers.fuser import get_fuser
return get_fuser(module, RMSNormFuser) is not None
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
"""Index in `funcdef.body` of the top-level statement containing `node`."""
for index, stmt in enumerate(funcdef.body):
if any(child is node for child in ast.walk(stmt)):
return index
raise ValueError("node is not in the function body")
def _single_expand_call(
funcdef: ast.FunctionDef, module: nn.Module, kv_b_proj_name: str
) -> ast.Call:
"""The KV expansion call `self.<method>(kv_c_normed, k_pe)`: the unique
two-argument call to a method of `module` with the expansion's signature.
The expansion is `kv_c_normed, k_pe -> key, value`: two inputs, every
`return` a pair, and it applies `kv_b_proj` -- the projection `MLAAttention`
owns after the bypass, absorbed into the attention computation.
"""
def is_expansion_method(name: str) -> bool:
method = getattr(type(module), name, None)
if method is None:
return False
code = getattr(inspect.unwrap(method), "__code__", None)
if (
code is None
or code.co_argcount != 3 # self, kv_c_normed, k_pe
or kv_b_proj_name not in code.co_names
):
return False
try:
tree = ast.parse(textwrap.dedent(inspect.getsource(method)))
except (OSError, TypeError):
return False
returns = [node for node in ast.walk(tree) if isinstance(node, ast.Return)]
return bool(returns) and all(
isinstance(ret.value, ast.Tuple) and len(ret.value.elts) == 2
for ret in returns
)
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == "self"
and len(node.args) == 2
and not node.keywords
and is_expansion_method(node.func.attr)
]
if len(calls) != 1:
raise ValueError("expected exactly one KV expansion method call")
return calls[0]
@dataclass
class MLAFuser(StackedFuser):
"""Fuser for the MLA attention pattern."""
q_proj_name: str | None
q_a_proj_name: str | None
q_a_layernorm_name: str | None
q_b_proj_name: str | None
kv_a_proj_name: str
kv_a_layernorm_name: str
kv_b_proj_name: str
o_proj_name: str | None
merged_name: ClassVar[str] = "fused_qkv_a_proj"
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
def info(self, name: str) -> str:
info_str = (
f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)"
)
if self.q_a_proj_name is not None:
info_str += "; " + super().info(name).removeprefix("Fused: ")
return info_str
@property
def shards(self) -> list[tuple[str, ShardId]]:
"""`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection."""
if self.q_a_proj_name is None:
return []
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
if self.q_a_proj_name is None:
return {}
return super().packed_modules_mapping
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None":
# Find all `rms_norm(linear(placeholder))` chains.
chains = []
for node in graph.nodes:
if node.op != "call_module" or is_linear(node, module) or not node.args:
continue
source = upstream_linear(node.args[0], module)
if source is None or not _consumes_placeholder(source):
continue
if not _is_rms_norm(module.get_submodule(node.target)):
continue
chains.append((source, node))
# Tell the chains apart by width.
def is_kv_a(chain) -> bool:
source, rms_norm = chain
source_mod = module.get_submodule(source.target)
rms_norm_mod = module.get_submodule(rms_norm.target)
return source_mod.out_features != _norm_size(rms_norm_mod)
kv_a_chains = [chain for chain in chains if is_kv_a(chain)]
q_a_chains = [chain for chain in chains if not is_kv_a(chain)]
# Exactly one KV chain is MLA's signature. The query chain is optional.
if len(kv_a_chains) != 1 or len(q_a_chains) > 1:
return None
kv_a_proj, kv_a_layernorm = kv_a_chains[0]
# Linear children claimed for a role so far; the rest resolve by elimination.
claimed_linears = {kv_a_proj.target}
q_proj_name = q_a_proj = q_a_layernorm = q_b_proj = None
if q_a_chains:
# Find `q_b_proj(q_a_layernorm(q_a_proj(placeholder)))`.
q_a_proj, q_a_layernorm = q_a_chains[0]
q_b_proj = downstream_linear(q_a_layernorm, module)
if q_b_proj is None:
return None
claimed_linears |= {q_a_proj.target, q_b_proj.target}
else:
# Find `q_proj(placeholder)`.
placeholder_linears = {
node.target
for node in graph.nodes
if is_linear(node, module) and _consumes_placeholder(node)
}
if len(q_proj_candidates := placeholder_linears - claimed_linears) != 1:
return None
q_proj_name = next(iter(q_proj_candidates))
claimed_linears.add(q_proj_name)
# Find `kv_b_proj(kv_a_layernorm(...))`.
kv_b_proj = downstream_linear(kv_a_layernorm, module)
if kv_b_proj is None or kv_b_proj.target in claimed_linears:
return None
claimed_linears.add(kv_b_proj.target)
# Find `o_proj` if it is returned by the forward graph.
o_proj_name = returned_linear(graph, module)
if o_proj_name in claimed_linears:
o_proj_name = None
return cls(
source_cls=type(module).__name__,
q_proj_name=q_proj_name,
q_a_proj_name=q_a_proj.target if q_a_proj else None,
q_a_layernorm_name=q_a_layernorm.target if q_a_layernorm else None,
q_b_proj_name=q_b_proj.target if q_b_proj else None,
kv_a_proj_name=kv_a_proj.target,
kv_a_layernorm_name=kv_a_layernorm.target,
kv_b_proj_name=kv_b_proj.target,
o_proj_name=o_proj_name,
)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
return vllm_config.model_config.use_mla
def update_forward(self, module: nn.Module) -> None:
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
attention interface unexpanded."""
funcdef, fn = recover_forward(type(module))
if (q_a_proj_name := self.q_a_proj_name) is not None:
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
# The fused call is inserted at the top-level statement preceding both.
names = [q_a_proj_name, self.kv_a_proj_name]
calls = self._unguarded_calls(funcdef, names)
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
raise ValueError("down-projections read different inputs")
indices = [_top_level_index(funcdef, call) for call in calls]
self._check_input_stable(funcdef, module, calls, funcdef.body, indices)
self._splice_merged_split(funcdef, calls, funcdef.body, min(indices))
# Transformers expands the latent into full key/value in a dedicated method.
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
# so replace the expansion call with its own arguments so `kv_c_normed, k_pe`
# flow to the interface in place of `key, value`.
expand_call = _single_expand_call(funcdef, module, self.kv_b_proj_name)
replace_expr(
funcdef, expand_call, ast.Tuple(elts=list(expand_call.args), ctx=ast.Load())
)
self.fused_forward = compile_forward(funcdef, fn)
def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"):
quant_config = vllm_config.quant_config
def replace_linear_by_name(name: str, style: Style):
linear = module.get_submodule(name)
replacement = replace_linear_class(
linear, style, quant_config, prefix=maybe_prefix(prefix, name)
)
setattr(module, name, replacement)
log_replacement(maybe_prefix(prefix, name), linear, replacement)
if (q_a_proj_name := self.q_a_proj_name) is not None:
# `match` sets the q-LoRA names together, or none of them
assert self.q_b_proj_name is not None
q_a = module.get_submodule(q_a_proj_name)
kv_a = module.get_submodule(self.kv_a_proj_name)
merged = MergedColumnParallelLinear(
input_size=q_a.in_features,
output_sizes=[q_a.out_features, kv_a.out_features],
bias=q_a.bias is not None,
quant_config=quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
disable_tp=True,
)
logger.debug(
"%s: %s, %s: %s -> %s: %s",
q_a_proj_name,
q_a,
self.kv_a_proj_name,
kv_a,
self.merged_name,
merged,
)
setattr(module, self.merged_name, merged)
# The rewritten forward calls the merged projection instead.
delattr(module, q_a_proj_name)
delattr(module, self.kv_a_proj_name)
replace_linear_by_name(self.q_b_proj_name, "colwise")
else:
assert self.q_proj_name is not None
replace_linear_by_name(self.kv_a_proj_name, "replicate")
replace_linear_by_name(self.q_proj_name, "colwise")
replace_linear_by_name(self.kv_b_proj_name, "colwise")
# MLAAttention calls kv_b_proj and expects vLLM's default return_bias=True
module.get_submodule(self.kv_b_proj_name).return_bias = True
if self.o_proj_name is not None:
replace_linear_by_name(self.o_proj_name, "rowwise")
@@ -1,439 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
import ast
import functools
import inspect
import textwrap
import types
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
import torch
from torch import fx, nn
from typing_extensions import TypeIs
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
is_complete,
is_op,
output_value,
peel,
trace,
)
from vllm.model_executor.models.transformers.utils import named_state
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
logger = init_logger(__name__)
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
stack = list(ast.iter_child_nodes(node))
while stack:
child = stack.pop()
if isinstance(child, ast.Return):
yield child
elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
stack.extend(ast.iter_child_nodes(child))
def _returns_tuple(cls: type[nn.Module]) -> bool:
"""Does `cls.forward()` return a tuple?"""
try:
source = textwrap.dedent(inspect.getsource(inspect.unwrap(cls.forward)))
forward = ast.parse(source).body[0]
except (OSError, SyntaxError, TypeError, IndexError):
return True
# Names bound to a tuple literal, e.g. `out = hidden, logits` then `return out`.
tuple_names = {
target.id
for node in ast.walk(forward)
if isinstance(node, ast.Assign) and isinstance(node.value, ast.Tuple)
for target in node.targets
if isinstance(target, ast.Name)
}
def yields_tuple(value: ast.expr | None) -> bool:
if isinstance(value, ast.Tuple):
return True
if isinstance(value, ast.Name):
return value.id in tuple_names
if isinstance(value, ast.IfExp):
return yields_tuple(value.body) or yields_tuple(value.orelse)
return False
return any(yields_tuple(ret.value) for ret in _own_returns(forward))
def _is_scalar_gate(module: nn.Module) -> bool:
"""A linear projecting to a single logit (the shared-expert sigmoid gate)."""
weight = getattr(module, "weight", None)
return (
isinstance(module, nn.Linear)
and weight is not None
and weight.ndim == 2
and weight.shape[0] == 1
)
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
"""The floating dtype `nodes` cast to, if any.
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it."""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
name = str(node.target).rsplit(".", 1)[-1]
if name == "float":
return torch.float32
if name in ("to", "type"):
for arg in (*node.args[1:], *node.kwargs.values()):
if isinstance(arg, torch.dtype) and arg.is_floating_point:
return arg
return None
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
"""Returns the set of nodes reachable from `node` by following `key` edges."""
seen: set[fx.Node] = set()
stack = [node]
while stack:
n = stack.pop()
if n in seen:
continue
seen.add(n)
stack.extend(getattr(n, key))
return seen
def _renormalizes(topk: fx.Node) -> bool:
"""Are the top-k weights divided by their own sum?
Every HF router spells renormalization as a division by a sum over the
selection, optionally offset by an epsilon."""
downstream = _reaches(topk, "users")
for node in downstream:
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
continue
denominator = node.args[1]
if not isinstance(denominator, fx.Node):
continue
cone = _reaches(denominator, "all_input_nodes")
if any(is_op(n, "sum") and n in downstream for n in cone):
return True
return False
def _unwrap(node: object, moe_block: nn.Module) -> object:
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
while True:
node = peel(node)
if (
is_op(node, "reshape")
or is_op(node, "view")
or (
isinstance(node, fx.Node)
and node.op == "call_module"
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
)
):
node = node.args[0]
else:
return node
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
"""Is `node` a child module of `moe_block` called on just `x`?"""
return (
isinstance(node, fx.Node)
and node.op == "call_module"
and len(node.args) == 1
and not node.kwargs
and _unwrap(node.args[0], moe_block) is x
)
class SharedExpertMLP(nn.Module):
"""Wraps an HF shared expert, applying the output gating it is paired with."""
def __init__(self, shared_experts: nn.Module, gate: nn.Module | None = None):
super().__init__()
self.shared_experts = shared_experts
self.gate = gate
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
out = self.shared_experts(hidden_states)
if self.gate is not None:
out = torch.sigmoid(self.gate(hidden_states)[0]) * out
return out
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
"""Standard MoE block forward.
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
orig_shape = hidden_states.shape
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
num_tokens = hidden_states.shape[0]
is_sequence_parallel = self.experts.moe_config.is_sequence_parallel
if is_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
out = self.experts(hidden_states, router_logits=hidden_states)
if is_sequence_parallel:
out = tensor_model_parallel_all_gather(out, 0)[:num_tokens]
return out.reshape(orig_shape)
@dataclass
class MoEBlockFuser:
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
gate_name: str
scoring_func: str | None
"""`None` if routing stays in the Transformers gate."""
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
renormalize: bool | None = None
@staticmethod
def _match_router(
gate: nn.Module,
) -> tuple[str, torch.dtype | None, bool | None] | None:
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
`softmax`/`sigmoid`.
Returns the scoring function, the dtype the router computes in, and
whether the top-k weights are renormalized (`None` if the trace was cut
short before it could say)."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
graph = trace(gate)
if graph is None:
return None
# The routing top-k is the last one; any earlier one scores expert groups.
topks = [node for node in graph.nodes if is_op(node, "topk")]
if not topks:
return None
topk = topks[-1]
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
post_topk = False
# Find all scoring functions that reach the top-k node.
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
if not scorers:
# Find all scoring functions that are users of the top-k node.
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
post_topk = True
if len(scorers) != 1:
return None
scorer = scorers[0]
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
is_softmax = is_op(scorer, "softmax")
# vLLM always scores every expert, so matching a router that softmaxes
# the selection alone takes renormalizing. Otherwise look for the
# division, reading its absence as "no" only from a complete trace.
renormalize: bool | None
if (post_topk and is_softmax) or _renormalizes(topk):
renormalize = True
else:
renormalize = False if is_complete(graph) else None
scoring_func = "softmax" if is_softmax else "sigmoid"
return scoring_func, _forced_dtype(logits_cone), renormalize
@staticmethod
def _match_experts(
graph: fx.Graph, experts_name: str, moe_block: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
being the block's input.
Returns the experts call, `x` and the gate call."""
experts = find_node(
graph, lambda n: n.op == "call_module" and n.target == experts_name
)
if experts is None:
return None
# Bind by signature so keyword calls match too. Fail closed on any other
# argument: the rewritten forward would drop it.
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
try:
bound = forward.bind(*experts.args, **experts.kwargs)
except TypeError:
return None
if len(bound.arguments) != 3:
return None
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
return None
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
return None
gate = ids.args[0]
if not (
_reads_input(gate, x, moe_block)
and ids.args == (gate, 2)
and weights.args == (gate, 1)
):
return None
return experts, x, gate
@staticmethod
def _match_shared_experts(
node: object, x: fx.Node, moe_block: nn.Module
) -> tuple[str, str | None] | None:
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
Returns the shared expert's name and its optional gate's name."""
if _reads_input(node, x, moe_block):
return node.target, None
if not (is_op(node, "mul") and len(node.args) == 2):
return None
for gated, shared in (node.args, node.args[::-1]):
gated, shared = peel(gated), peel(shared)
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
continue
gate = peel(gated.args[0])
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
moe_block.get_submodule(gate.target)
):
return shared.target, gate.target
return None
@classmethod
def match(
cls,
moe_block: nn.Module,
experts_name: str,
divides_shared_output: bool = False,
) -> "MoEBlockFuser | None":
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
reshapes, casts and `nn.Identity`.
Args:
moe_block: The MoE block.
experts_name: The name of the experts in `moe_block`.
divides_shared_output: Whether vLLM routing would divide the shared
expert's output by `routed_scaling_factor` (fp16 overflow
protection), which Transformers decoder layers do not undo.
Returns:
A fuser if more than the experts can run in vLLM, else `None`.
"""
# Standard MoE block returns a single tensor.
if _returns_tuple(type(moe_block)):
return None
if (graph := trace(moe_block)) is None:
return None
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
return None
experts, x, gate = matched
unwrap = functools.partial(_unwrap, moe_block=moe_block)
gate_name = gate.target
router = cls._match_router(getattr(moe_block, gate_name))
scoring_func, router_dtype, renormalize = router or (None, None, None)
def is_routed(n: object) -> bool:
return n is experts or (
is_op(n, "mul")
and not isinstance(n.args[1], fx.Node)
and unwrap(n.args[0]) is experts
)
# Output: the experts' output, plus the shared expert's if any.
routed = out = unwrap(output_value(graph))
shared = None
if not is_routed(out):
if not (is_op(out, "add") and len(out.args) == 2):
return None
routed, shared = map(unwrap, out.args)
if is_routed(shared):
routed, shared = shared, routed
if not is_routed(routed):
return None
shared_name = shared_gate_name = None
if shared is not None:
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
return None
shared_name, shared_gate_name = match
# Fail closed: `rewrite_forward` runs only the experts and the detected
# shared expert, so any other stateful child would be dropped.
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
if scoring_func is not None and shared_name and divides_shared_output:
logger.warning_once(
"%s could be fused but routing it in vLLM would apply "
"`routed_scaling_factor` by dividing the shared expert output, "
"which only fp16 overflow protection expects the decoder layer to "
"compensate for. Falling back to routing in Transformers; run in "
"bfloat16 to fuse it.",
type(moe_block).__name__,
)
scoring_func = router_dtype = renormalize = None
# Without vLLM routing, only a shared expert can move to vLLM, and only
# if the block does not scale the experts' output (vLLM routing does).
if scoring_func is None and (shared_name is None or routed is not experts):
return None
return cls(
gate_name,
scoring_func,
shared_name,
shared_gate_name,
router_dtype,
renormalize,
)
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
) -> GateLinear:
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
hf_gate = getattr(moe_block, self.gate_name)
num_experts, hidden_size = hf_gate.weight.shape
gate = GateLinear(
hidden_size,
num_experts,
bias=False,
out_dtype=out_dtype or self.router_dtype,
prefix=maybe_prefix(prefix, self.gate_name),
)
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
gate.register_buffer("e_score_correction_bias", bias)
setattr(moe_block, self.gate_name, gate)
return gate
def shared_experts(
self, moe_block: nn.Module, prefix: str
) -> SharedExpertMLP | None:
"""Build the HF shared expert (and its optional gate)
as a `SharedExpertMLP` for vLLM's fused MoE."""
if self.shared_name is None:
return None
shared_experts = getattr(moe_block, self.shared_name)
gate = None
if self.shared_gate_name is not None:
hf_gate = getattr(moe_block, self.shared_gate_name)
gate = ReplicatedLinear(
hf_gate.in_features,
hf_gate.out_features,
bias=hf_gate.bias is not None,
prefix=maybe_prefix(prefix, self.shared_gate_name),
)
setattr(moe_block, self.shared_gate_name, gate)
return SharedExpertMLP(shared_experts, gate)
def rewrite_forward(self, moe_block: nn.Module) -> None:
"""Rewrite `moe_block.forward` to route through vLLM's fused MoE."""
moe_block.forward = types.MethodType(_moe_block_forward, moe_block)
@@ -1,164 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
RewriteFuser,
fused_head_size,
local_output_sizes,
)
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
is_method,
recover_forward,
returned_linear,
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class PackedQKVFuser(RewriteFuser):
"""Fuser for attention with q, k and v packed into one projection."""
qkv_name: str
o_name: str | None
q_size: int
kv_size: int
def info(self, name: str) -> str:
return (
f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear"
)
@staticmethod
def _packed_sizes(node: fx.Node) -> tuple[int, int] | None:
"""`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one."""
if not is_method(node, "split") or len(node.args) < 2:
return None
sizes = node.args[1]
if not isinstance(sizes, (tuple, list)) or len(sizes) != 3:
return None
if not all(isinstance(size, int) for size in sizes):
return None
q_size, k_size, v_size = sizes
if k_size != v_size or q_size < k_size:
return None
return q_size, k_size
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None":
for node in graph.nodes:
if (sizes := cls._packed_sizes(node)) is None:
continue
q_size, kv_size = sizes
qkv_node = upstream_linear(node.args[0], module)
if qkv_node is None:
continue
qkv_name = str(qkv_node.target)
# The split must consume the whole projection.
if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size:
continue
# o_proj produces the module's output and consumes the query width.
o_name = returned_linear(graph, module)
if o_name == qkv_name or (
o_name is not None
and module.get_submodule(o_name).in_features != q_size
):
o_name = None
return cls(
source_cls=type(module).__name__,
qkv_name=qkv_name,
o_name=o_name,
q_size=q_size,
kv_size=kv_size,
)
return None
def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call:
"""The unique `self.<qkv_name>(...)....split((a, b, c), ...)` call."""
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "split"
and node.args
and isinstance(node.args[0], (ast.Tuple, ast.List))
and len(node.args[0].elts) == 3
and any(
isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name
for inner in ast.walk(node.func.value)
)
]
if len(calls) != 1:
raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits")
return calls[0]
def update_forward(self, module: nn.Module) -> None:
"""Rewrite the split sizes to the sharded projection's per-rank widths."""
funcdef, fn = recover_forward(type(module))
split = self._split_call(funcdef)
# (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes]
sections = local_output_sizes(self.qkv_name)
split.args[0] = ast.parse(sections, mode="eval").body
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible with a head-sharded packed GEMM."""
head_size = fused_head_size(module, vllm_config)
qkv = module.get_submodule(self.qkv_name)
compatible = (
self.q_size % head_size == 0
and self.kv_size % head_size == 0
and qkv.out_features == self.q_size + 2 * self.kv_size
)
if not compatible:
logger.debug("%s is not compatible with packed QKV fusion", type(module))
return compatible
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = fused_head_size(module, vllm_config)
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
qkv = module.get_submodule(self.qkv_name)
merged = QKVParallelLinear(
hidden_size=qkv.in_features,
head_size=head_size,
total_num_heads=self.q_size // head_size,
total_num_kv_heads=self.kv_size // head_size,
bias=qkv.bias is not None,
quant_config=quant_config,
prefix=qkv_prefix,
return_bias=False,
)
setattr(module, self.qkv_name, merged)
log_replacement(qkv_prefix, qkv, merged)
# If there is an output projection, we know it must be rowwise.
if self.o_name is not None:
o_prefix = maybe_prefix(prefix, self.o_name)
o_proj = module.get_submodule(self.o_name)
new_o = replace_linear_class(
o_proj, "rowwise", quant_config, prefix=o_prefix
)
setattr(module, self.o_name, new_o)
log_replacement(o_prefix, o_proj, new_o)
@@ -1,209 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""QKV projection fuser: `q(x), k(x), v(x)` -> a fused qkv linear + split."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
StackedFuser,
fused_head_size,
)
from vllm.model_executor.models.transformers.fx_utils import (
block_chain,
compile_forward,
is_linear,
recover_forward,
returned_linear,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class QKVFuser(StackedFuser):
"""Fuser for the attention QKV pattern `q(x), k(x), v(x)`."""
q_name: str
k_name: str
v_name: str
o_name: str | None
merged_name: ClassVar[str] = "qkv_proj"
merged_cls_name: ClassVar[str] = "QKVParallelLinear"
@property
def shards(self) -> list[tuple[str, ShardId]]:
return [(self.q_name, "q"), (self.k_name, "k"), (self.v_name, "v")]
@classmethod
def _get_qkv_nodes(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
"""Search `graph` for the QKV pattern `q(x), k(x), v(x)`."""
by_input: dict[fx.Node, list[fx.Node]] = {}
for node in graph.nodes:
if (
is_linear(node, module)
and len(node.args) == 1
and not node.kwargs
and isinstance(node.args[0], fx.Node)
and node.args[0].op == "placeholder"
):
by_input.setdefault(node.args[0], []).append(node)
triples = [nodes for nodes in by_input.values() if len(nodes) == 3]
if len(triples) != 1:
return None
q_node, k_node, v_node = nodes = triples[0]
outs = [module.get_submodule(node.target).out_features for node in nodes]
if len(set(outs)) == 2:
# q is identified as the larger projection (GQA)
(q_node,) = (n for n, out in zip(nodes, outs) if outs.count(out) == 1)
k_node, v_node = (n for n, out in zip(nodes, outs) if outs.count(out) == 2)
if module.get_submodule(q_node.target).out_features != max(outs):
return None
elif len(set(outs)) != 1:
return None
return q_node, k_node, v_node
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "QKVFuser | None":
if (qkv_nodes := cls._get_qkv_nodes(graph, module)) is None:
return None
q, k, v = qkv_nodes
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
# o_proj produces the module's output.
o_name = returned_linear(graph, module)
# o_proj must be compatible with the q/k/v projections.
if o_name in names.values() or (
o_name is not None
and module.get_submodule(o_name).in_features
!= module.get_submodule(q.target).out_features
):
o_name = None
names["o_name"] = o_name
return cls(source_cls=type(module).__name__, **names)
def update_forward(self, module: nn.Module) -> None:
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source.
A projection may be guarded by an existence check -- a
`self.<proj> is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare
truthiness test -- which is folded to its constant value (see
`bypass_existence_guard`). The calls may sit in different branches, so the
fused GEMM is inserted before the earliest of them, in the innermost block
that dominates all three.
"""
funcdef, fn = recover_forward(type(module))
proj_names = (self.q_name, self.k_name, self.v_name)
calls = self._unguarded_calls(funcdef, proj_names)
arg_dumps = {ast.dump(call.args[0]) for call in calls}
if len(arg_dumps) != 1:
raise ValueError("projection inputs are written differently")
# The trace may be partial, so prove projection exclusivity in source:
# no other linear child may consume the same input (else the matched
# three may not be q, k and v)
other_linears = {
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
} - set(proj_names)
for node in ast.walk(funcdef):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr in other_linears
and any(ast.dump(arg) in arg_dumps for arg in node.args)
):
raise ValueError("another linear consumes the same input")
# Insert the fused GEMM before the earliest call, in the innermost block
# common to all three (the calls may be split across branches).
chains = [block_chain(funcdef.body, call) for call in calls]
if any(not chain for chain in chains):
raise ValueError("projection calls not found in the function body")
depth = 0
for level in zip(*chains):
if len({id(block) for block, _ in level}) != 1:
break
depth += 1
block = chains[0][depth - 1][0]
indices = [chain[depth - 1][1] for chain in chains]
insert_index = min(indices)
self._check_input_stable(funcdef, module, calls, block, indices)
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
self._splice_merged_split(funcdef, calls, block, insert_index)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
v = module.get_submodule(self.v_name)
head_size = fused_head_size(module, vllm_config)
compatible = (
q.in_features == k.in_features == v.in_features
and len({proj.bias is None for proj in (q, k, v)}) == 1
and k.out_features == v.out_features
and q.out_features % head_size == 0
and k.out_features % head_size == 0
)
if not compatible:
logger.debug("%s is not compatible with QKV fusion", type(module))
return compatible
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = fused_head_size(module, vllm_config)
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
merged = QKVParallelLinear(
hidden_size=q.in_features,
head_size=head_size,
total_num_heads=q.out_features // head_size,
total_num_kv_heads=k.out_features // head_size,
bias=q.bias is not None,
quant_config=quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
)
logger.debug(
"%s: %s, %s: %s, %s: %s -> %s: %s",
self.q_name,
q,
self.k_name,
k,
self.v_name,
module.get_submodule(self.v_name),
self.merged_name,
merged,
)
setattr(module, self.merged_name, merged)
# Drop the consumed submodules so their (meta) params are not expected.
for name in (self.q_name, self.k_name, self.v_name):
delattr(module, name)
# If there is an output projection, we know it must be rowwise.
if self.o_name is not None:
o_proj_prefix = maybe_prefix(prefix, self.o_name)
o_proj = module.get_submodule(self.o_name)
new_o = replace_linear_class(
o_proj, "rowwise", quant_config, prefix=o_proj_prefix
)
setattr(module, self.o_name, new_o)
log_replacement(o_proj_prefix, o_proj, new_o)
@@ -1,297 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RMSNorm fuser: detect the norm structurally and swap in vLLM's fused RMSNorm."""
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
from torch import fx, nn
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
)
from vllm.distributed.parallel_state import model_parallel_is_initialized
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
forward_input_count,
is_op,
output_value,
peel,
trace,
)
from vllm.model_executor.models.transformers.layers import (
GemmaRMSNorm,
RMSNorm,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
def _operand(node: fx.Node, index: int, name: str) -> object | None:
"""Operand `index` of `node`, whether it was passed positionally or as `name`."""
if len(node.args) > index:
return node.args[index]
return node.kwargs.get(name)
def _is_squared(node: object, x: fx.Node) -> bool:
"""`x**2`, `x.square()` or `x * x`, through any dtype casts."""
node = peel(node)
if is_op(node, "pow"):
return (
peel(_operand(node, 0, "input")) is x and _operand(node, 1, "exponent") == 2
)
if is_op(node, "square"):
return peel(_operand(node, 0, "input")) is x
if is_op(node, "mul"):
return (
peel(_operand(node, 0, "input")) is x
and peel(_operand(node, 1, "other")) is x
)
return False
def _is_inverse_sqrt(node: object) -> bool:
"""Detect `rsqrt(v)`, or the `pow(v, -0.5)` / `v ** -0.5` spelling of it."""
if is_op(node, "rsqrt"):
return True
if is_op(node, "pow"):
return _operand(node, 1, "exponent") == -0.5
return False
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
"""`eps` from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
add = peel(_operand(rsqrt, 0, "input"))
if not is_op(add, "add"):
return None
operands = [_operand(add, 0, "input"), _operand(add, 1, "other")]
consts = [a for a in operands if isinstance(a, (int, float))]
nodes = [a for a in operands if isinstance(a, fx.Node)]
if len(consts) != 1 or len(nodes) != 1:
return None
mean = peel(nodes[0])
if not is_op(mean, "mean"):
return None
if not _is_squared(_operand(mean, 0, "input"), x):
return None
return float(consts[0])
def _is_one_plus(node: object) -> bool:
"""`1 + weight` in either operand order (marks a zero-centered weight)."""
node = peel(node)
if not is_op(node, "add"):
return False
return any(
isinstance(a, (int, float)) and a == 1
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
)
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
"""Does the forward compute anything after `node` before returning?"""
value = output_value(graph)
return value is not None and peel(value) is not node
class TPAwareNormMixin(nn.Module):
"""Mixin for RMSNorms that reconstructs a TP-sharded input before normalizing."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if model_parallel_is_initialized():
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
else:
self.tp_size, self.tp_rank = 1, 0
def forward(
self, x: torch.Tensor, residual: torch.Tensor | None = None
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if self.tp_size > 1 and x.shape[-1] < (full := self.weight.shape[0]):
if x.shape[-1] * self.tp_size != full:
raise ValueError(
f"Cannot gather norm of width {full}: a TP-sharded input of "
f"width {x.shape[-1]} does not tile it evenly across "
f"{self.tp_size} ranks (replicated or uneven sharding)."
)
x = tensor_model_parallel_all_gather(x.contiguous())
x = super().forward(x)
splits = split_tensor_along_last_dim(x, num_partitions=self.tp_size)
return splits[self.tp_rank]
return super().forward(x, residual)
class TPAwareRMSNorm(TPAwareNormMixin, RMSNorm):
"""`RMSNorm` that reconstructs a TP-sharded input before normalizing."""
class TPAwareGemmaRMSNorm(TPAwareNormMixin, GemmaRMSNorm):
"""`GemmaRMSNorm` that reconstructs a TP-sharded input before normalizing."""
@dataclass
class RMSNormFuser(BaseFuser):
"""Fuser for RMSNorm patterns, including Gemma-style zero-centered weights."""
zero_centered: bool
"""Gemma-style `(1 + weight)` scaling (weight initialised at zero)."""
source_cls: str
"""Class name of the norm this was matched from (for logging)."""
eps_attr: str | None = None
"""Attribute holding `eps`, read per instance in `fuse`."""
eps: float | None = None
"""`eps` itself, when it is not held in an attribute (see `_eps_source`)."""
def info(self, name: str) -> str:
norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm"
return f"Fused: {name} ({self.source_cls}) -> {norm} (CustomOp)"
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "RMSNormFuser | None":
"""Match a graph to the RMSNorm pattern, returning a fuser if found."""
if forward_input_count(type(module)) != 1:
return None
x = find_node(graph, lambda n: n.op == "placeholder")
if x is None:
return None
# Handle native torch `rms_norm` op.
rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm"))
if rms_norm is not None and peel(_operand(rms_norm, 0, "input")) is x:
if _has_trailing_compute(graph, rms_norm):
return None
eps_attr, eps = cls._eps_source(graph, module)
return cls(
zero_centered=False,
source_cls=type(module).__name__,
eps_attr=eps_attr,
eps=eps,
)
# Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern.
# The rsqrt over the mean-square variance is the spine of the norm.
rsqrt = None
for node in graph.nodes:
if _is_inverse_sqrt(node) and _variance_eps(node, x) is not None:
rsqrt = node
break
if rsqrt is None:
return None
# The `x * rsqrt(...)` normalize multiply.
normalize = find_node(
graph,
lambda n: is_op(n, "mul")
and rsqrt in map(peel, (_operand(n, 0, "input"), _operand(n, 1, "other"))),
)
if normalize is None:
return None
# An optional later `weight * normalized` (or `(1 + weight) * normalized`).
tail, zero_centered = normalize, False
for node in graph.nodes:
if not is_op(node, "mul") or node is normalize:
continue
operands = [
peel(a)
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
if isinstance(a, fx.Node)
]
if len(operands) == 2 and normalize in operands:
weight = next(o for o in operands if o is not normalize)
tail, zero_centered = node, _is_one_plus(weight)
break
# The norm must be the last compute in forward, or it is not a pure norm.
if _has_trailing_compute(graph, tail):
return None
eps_attr, eps = cls._eps_source(graph, module)
return cls(
zero_centered=zero_centered,
source_cls=type(module).__name__,
eps_attr=eps_attr,
eps=eps,
)
@classmethod
def _eps_source(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[str | None, float | None]:
"""Where `fuse` should read `eps` from, resolved once per class."""
eps = cls._eps_from_graph(graph)
if eps is None:
return None, None
# Whatever supplied the constant must still equal it.
candidates = {
name: value
for name, value in vars(module).items()
if isinstance(value, float) and value == eps
}
# Use unique markers and retrace to verify exactly which attribute is eps.
markers = {float(-index - 1): name for index, name in enumerate(candidates)}
marked = None
if markers:
try:
for marker, name in markers.items():
setattr(module, name, marker)
if (remarked := trace(module)) is not None:
marked = cls._eps_from_graph(remarked)
finally:
for name, value in candidates.items():
setattr(module, name, value)
if marked is not None and (name := markers.get(marked)) is not None:
return name, None
logger.debug_once(
"%s does not hold its eps (%s) in an attribute. Every instance in this "
"model will use the value traced from this instance. If this is not "
"desired, consider storing and reading eps using attribute of %s.",
type(module).__name__,
eps,
type(module).__name__,
)
return None, eps
@staticmethod
def _eps_from_graph(graph: fx.Graph) -> float | None:
"""Extract the `eps` constant from the graph, if present."""
if (x := find_node(graph, lambda n: n.op == "placeholder")) is None:
return None
fused = find_node(graph, lambda n: is_op(n, "rms_norm"))
if fused is not None and peel(_operand(fused, 0, "input")) is x:
args, kwargs = fused.args, fused.kwargs
eps = args[3] if len(args) > 3 else kwargs.get("eps")
return float(eps) if isinstance(eps, (int, float)) else None
for node in graph.nodes:
if _is_inverse_sqrt(node) and (eps := _variance_eps(node, x)) is not None:
return eps
return None
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
return True
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
weight = getattr(module, "weight", None)
hidden_size = weight.size(0) if weight is not None else 0
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
if not isinstance(eps, (int, float)):
# If eps was not detected, match torch behaviour.
config_dtype = vllm_config.model_config.dtype
dtype = weight.dtype if weight is not None else config_dtype
eps = torch.finfo(dtype).eps
if self.zero_centered:
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
return TPAwareRMSNorm(
hidden_size=hidden_size,
eps=eps,
has_weight=weight is not None,
dtype=weight.dtype if weight is not None else None,
)
@@ -1,818 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fx tracing and forward-source rewriting for the Transformers backend fusers.
A small engine, independent of any particular pattern: trace a module's forward
with `torch.fx` (tolerating a partial graph), inspect the resulting nodes, and
rewrite the forward's *source* (AST) so only matched calls change while the rest
stays live Python. `fusion.py` builds the concrete fusion patterns on top.
"""
import ast
import contextlib
import inspect
import operator
import textwrap
from collections.abc import Callable
from itertools import chain
from typing import Any
from unittest import mock
import torch
from torch import fx, nn
from torch.nn import functional as F
from typing_extensions import TypeIs
from vllm.logger import init_logger
logger = init_logger(__name__)
_UNKNOWN: Any = object()
"""Sentinel meta value for proxies whose concrete value could not be inferred.
Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`)."""
_MODULE_CALL = nn.Module.__call__
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
`call_module` nodes; meta execution must call modules for real."""
_MODULE_GETATTR = nn.Module.__getattr__
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
recurse until the stack runs out, truncating the graph."""
@contextlib.contextmanager
def _unpatched_modules():
"""Restores the module hooks fx patches, so meta execution records nothing."""
with (
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
):
yield
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
return isinstance(node, fx.Node) and node.meta.get("leaf_call", False)
def _reference_weight(module: nn.Module) -> torch.Tensor | None:
"""A weight whose trailing dim is the module's hidden size.
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
for child in module.modules():
if isinstance(child, nn.Linear):
return child.weight
for param in module.parameters():
if param.ndim in (1, 2):
return param
return None
class _MetaProxy(fx.Proxy):
"""Proxy carrying the meta-tensor value of the traced expression.
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
executing each op on the meta values, so PyTorch's meta kernels are the
single source of shape inference — no per-op rules."""
meta: Any = _UNKNOWN
def __len__(self) -> int:
if self.meta is not _UNKNOWN:
return len(self.meta)
return super().__len__() # type: ignore[misc]
def __getattr__(self, k: str) -> "_MetaAttribute":
return _MetaAttribute(self, k)
def __setitem__(self, key: object, value: object) -> None:
# Record `obj[key] = value` so tracing continues past in-place writes
# into containers (e.g. Gemma 4's `shared_kv_states[...] = ...`).
self.tracer.create_proxy(
"call_function", operator.setitem, (self, key, value), {}
)
class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
`Tracer.proxy`, so the meta value must be grafted on here too."""
def __init__(self, root: fx.Proxy, attr: str):
super().__init__(root, attr)
root_meta = getattr(root, "meta", _UNKNOWN)
if root_meta is not _UNKNOWN:
with contextlib.suppress(Exception):
self.meta = getattr(root_meta, attr)
class _AllLeafTracer(fx.Tracer):
"""Tracer that treats every submodule as a leaf.
Each child stays one `call_module` node, so matching sees the module's own
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
Every traced op is also executed on meta tensors (see `_MetaProxy`) so
shape unpacks and `*`-splats trace through; anything else untraceable ends
the trace early and the partial graph is matched.
"""
varkw: str | None = None
"""Name of the traced forward's `**kwargs` parameter, if any."""
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
return True
def proxy(self, node: fx.Node) -> fx.Proxy:
return _MetaProxy(node, self)
def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs):
proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs)
if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN:
# A failure stays _UNKNOWN; only fatal if a shape question is asked.
with contextlib.suppress(Exception):
proxy.meta = self._infer_meta(kind, target, args, kwargs)
return proxy
def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict):
"""Execute the op on meta tensors; PyTorch infers the output value."""
if kind == "placeholder":
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states.
weight = _reference_weight(self.root)
if str(target) == "hidden_states" and weight is not None:
return torch.empty(
1, 8, weight.shape[-1], dtype=weight.dtype, device="meta"
)
return _UNKNOWN
if kind == "get_attr":
with _unpatched_modules():
value = operator.attrgetter(str(target))(self.root)
if isinstance(value, torch.Tensor):
value = torch.empty_like(value, device="meta")
return value
unknown = False
def meta_of(arg: object) -> object:
nonlocal unknown
if isinstance(arg, fx.Proxy):
meta = getattr(arg, "meta", _UNKNOWN)
unknown = unknown or meta is _UNKNOWN
return meta
return arg
meta_args = fx.node.map_aggregate(args, meta_of)
meta_kwargs = fx.node.map_aggregate(kwargs, meta_of)
if unknown:
return _UNKNOWN
if kind == "call_function":
assert callable(target)
return target(*meta_args, **meta_kwargs)
if kind == "call_method":
receiver, *rest = meta_args
return getattr(receiver, str(target))(*rest, **meta_kwargs)
if kind == "call_module":
# Run the child's forward with all its state on "meta", without
# mutating it (at match time params may be meta but buffers real).
child = self.root.get_submodule(str(target))
state = {
name: torch.empty_like(tensor, device="meta")
for name, tensor in chain(
child.named_parameters(), child.named_buffers()
)
}
with _unpatched_modules():
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
return _UNKNOWN
def _is_varkw(self, node: object) -> bool:
return (
isinstance(node, fx.Node)
and node.op == "placeholder"
and str(node.target).lstrip("*") == self.varkw
)
def iter(self, obj: fx.Proxy):
# Assume kwargs is always empty to simplify tracing.
node = obj.node
if self._is_varkw(node) or (
node.op == "call_method"
and node.target == "keys"
and self._is_varkw(node.args[0])
):
return iter(())
meta = getattr(obj, "meta", _UNKNOWN)
if meta is _UNKNOWN:
return super().iter(obj)
return iter([obj[i] for i in range(len(meta))])
def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable:
"""Wrap any callable so tracing records it as one opaque `call_function` node.
Lets the trace continue past untraceable bodies. Only the proxy arguments carry into
the node's dataflow; the rest are dropped rather than lifted into the graph.
`length` declares how many values the callable returns, so unpacking its result also
traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough.
"""
def leaf(*args, **kwargs):
proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy))
if not proxies:
return fn(*args, **kwargs)
proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {})
proxy.node.meta["leaf_call"] = True
if length is not None:
# The body never executes, so fabricate a value of the declared length.
proxy.meta = (_UNKNOWN,) * length
return proxy
return leaf
def _leaf_attention_interfaces():
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
`vllm_attention_function` needs runtime context so it is untraceable.
Every interface returns `(attn_output, attn_weights)`."""
from transformers.modeling_utils import AttentionInterface
original = AttentionInterface.get_interface
def get_interface(self, *args, **kwargs):
return _as_leaf_call(original(self, *args, **kwargs), length=2)
return mock.patch.object(AttentionInterface, "get_interface", get_interface)
def trace(module: nn.Module) -> fx.Graph | None:
"""Trace `module.forward`, returning the partial graph on failure."""
parameters = forward_parameters(type(module))
# vLLM never passes `past_key_values` so it is always the default value of `None`.
# Make this concrete to simplify tracing.
concrete_args = None
if "past_key_values" in parameters:
concrete_args = {"past_key_values": None}
# Get the name of the kwargs parameter passed to module.forward (usually "kwargs")
tracer = _AllLeafTracer()
tracer.varkw = next(
(
p.name
for p in parameters.values()
if p.kind is inspect.Parameter.VAR_KEYWORD
),
None,
)
try:
with _leaf_attention_interfaces():
return tracer.trace(module, concrete_args=concrete_args)
except Exception as exc:
logger.debug("Could not fully trace %s: %s", type(module), exc)
return getattr(tracer, "graph", None)
def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
"""Parse the source of `cls.forward`, ready for rewriting."""
fn = inspect.unwrap(cls.forward)
if fn.__code__.co_freevars:
raise ValueError("forward is a closure")
tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
funcdef = tree.body[0]
if not isinstance(funcdef, ast.FunctionDef):
raise ValueError("source is not a plain function definition")
# `fn` is already unwrapped; don't re-apply its decorators
funcdef.decorator_list.clear()
# Annotations may not evaluate outside the defining module (e.g. with
# postponed evaluation); they're not needed at runtime
funcdef.returns = None
args = funcdef.args
for arg in (
*args.posonlyargs,
*args.args,
*args.kwonlyargs,
*filter(None, (args.vararg, args.kwarg)),
):
arg.annotation = None
# Recompiling outside the class body would break name mangling
for node in ast.walk(funcdef):
name = getattr(node, "attr", None) or getattr(node, "id", None)
if name and name.startswith("__") and not name.endswith("__"):
raise ValueError(f"{name} would be name mangled")
return funcdef, fn
def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
"""`cls.forward`'s signature parameters, or empty if uninspectable."""
try:
return dict(inspect.signature(cls.forward).parameters)
except (TypeError, ValueError):
return {}
def forward_input_count(cls: type[nn.Module]) -> int:
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
any `*args`/`**kwargs`. Read from the signature, so it is independent of
whether the trace completes (unlike counting placeholders)."""
params = list(forward_parameters(cls).values())
if not params:
return 1 # uninspectable: assume a single input and let matching decide
fixed = (
inspect.Parameter.POSITIONAL_ONLY,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
)
return sum(1 for p in params[1:] if p.kind in fixed)
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
"""Compile `funcdef` in `fn`'s module so tracebacks point at the source."""
module = ast.Module(body=[funcdef], type_ignores=[])
ast.fix_missing_locations(module)
ast.increment_lineno(module, fn.__code__.co_firstlineno - 1)
code = compile(module, fn.__code__.co_filename, "exec")
namespace: dict = {}
exec(code, fn.__globals__, namespace)
return namespace[funcdef.name]
def self_call_and_refs(
funcdef: ast.FunctionDef, name: str
) -> tuple[ast.Call, list[ast.Attribute]]:
"""The unique `self.<name>(arg)` call, plus every other `self.<name>` reference.
Raises unless exactly one single-argument `self.<name>(arg)` call exists, so
one fx `call_module` node maps to one syntactic call site. The remaining
references (e.g. `is not None` guards) are returned for the caller to resolve.
"""
uses = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Attribute)
and node.attr == name
and isinstance(node.value, ast.Name)
and node.value.id == "self"
]
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and any(node.func is use for use in uses)
and len(node.args) == 1
and not isinstance(node.args[0], ast.Starred)
and not node.keywords
]
if len(calls) != 1:
raise ValueError(f"{name} is not called exactly once as self.{name}(arg)")
call = calls[0]
other_refs = [use for use in uses if use is not call.func]
return call, other_refs
def single_self_call(funcdef: ast.FunctionDef, name: str) -> ast.Call:
"""The unique `self.<name>(arg)` call in `funcdef`.
Raises unless `name` appears exactly once, as such a call, so the source
rewrite agrees with the fx match.
"""
call, other_refs = self_call_and_refs(funcdef, name)
if other_refs:
raise ValueError(f"{name} is referenced {len(other_refs) + 1} times")
return call
def _is_none(node: ast.expr) -> bool:
return isinstance(node, ast.Constant) and node.value is None
def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
"""Is `ref` used only for its truth value (a test, or a `not` operand)?
In these positions the object's identity never escapes, so a reference that
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
yield an operand, so the module could escape (`x and self.<name>`)."""
for node in ast.walk(funcdef):
if (
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
and node.test is ref
):
return True
if (
isinstance(node, ast.UnaryOp)
and isinstance(node.op, ast.Not)
and node.operand is ref
):
return True
return False
def bypass_existence_guard(
funcdef: ast.FunctionDef, ref: ast.Attribute, name: str
) -> None:
"""Fold a guard on `self.<name>`'s existence to its constant value.
The fuser deletes `self.<name>` and binds only instances where it exists as a
truthy `nn.Linear` (guaranteed by the match, the cache key, and `validate`),
so a guard testing its presence is a fusion invariant. Two forms are folded:
- an identity `None` check `self.<name> is None` -> `False`,
`self.<name> is not None` -> `True` (either operand order). `==`/`!=` are
*not* folded: `is_linear` accepts `nn.Linear` subclasses, which may override
`__eq__`/`__ne__`, so equality is not guaranteed to track identity.
- a bare truthiness test (`if self.<name>:`, `... if self.<name> else ...`,
`not self.<name>`): the reference itself to `True`.
Any other surviving reference escapes the projection's value, which no longer
exists after fusion, so refuse rather than change semantics."""
for node in ast.walk(funcdef):
# `self.<name> is (not) None`, either operand order.
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
continue
(op,), (right,) = node.ops, node.comparators
if not isinstance(op, (ast.Is, ast.IsNot)):
continue
if (ref is node.left and _is_none(right)) or (
ref is right and _is_none(node.left)
):
value = isinstance(op, ast.IsNot)
replace_expr(funcdef, node, ast.copy_location(ast.Constant(value), node))
return
# A bare truthiness test (statement `if`, ternary, or `not`); an `nn.Linear`
# is always truthy, so the reference folds to `True`.
if _in_boolean_context(funcdef, ref):
replace_expr(funcdef, ref, ast.copy_location(ast.Constant(True), ref))
return
raise ValueError(f"{name} is referenced outside an existence guard")
def block_chain(
block: list[ast.stmt], node: ast.AST
) -> list[tuple[list[ast.stmt], int]]:
"""Path of (statement list, index) pairs from `block` down to `node`.
Each pair names a nested block and the index in it of the statement
containing `node`; the last pair is `node`'s innermost block. Empty if
`node` is not in `block`. The longest common prefix of several nodes' chains
is the innermost block that dominates them all.
"""
for index, stmt in enumerate(block):
if not any(child is node for child in ast.walk(stmt)):
continue
child_blocks = [
getattr(stmt, fld, None) for fld in ("body", "orelse", "finalbody")
]
child_blocks += [h.body for h in getattr(stmt, "handlers", [])]
child_blocks += [c.body for c in getattr(stmt, "cases", [])]
for child_block in child_blocks:
if (
isinstance(child_block, list)
and child_block
and (tail := block_chain(child_block, node))
):
return [(block, index), *tail]
return [(block, index)]
return []
def _base_name(node: ast.expr) -> str | None:
"""The root `Name` of an attribute/subscript chain (`a.b[c]` -> `a`)."""
while isinstance(node, (ast.Attribute, ast.Subscript)):
node = node.value
return node.id if isinstance(node, ast.Name) else None
def _rebound_names(region: list[ast.stmt]) -> set[str]:
"""Names rebound outright within `region` (`x = ...`, `del x`).
These are `Name` nodes in a `Store`/`Del` context."""
return {
node.id
for stmt in region
for node in ast.walk(stmt)
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del))
}
def _inplace_target(node: ast.AST) -> str | None:
"""Base name `node` mutates in place, if any.
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
plain `Name` store (see `_rebound_names`)."""
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
node.ctx, (ast.Store, ast.Del)
):
return _base_name(node)
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr.endswith("_")
and not node.func.attr.endswith("__")
):
return _base_name(node.func.value)
return None
def _mutated_names(region: list[ast.stmt]) -> set[str]:
"""Names mutated in place within `region` (see `_inplace_target`)."""
return {
base
for stmt in region
for node in ast.walk(stmt)
if (base := _inplace_target(node)) is not None
}
def _call_name(call: ast.Call) -> str:
"""The called function's own name, without its qualifier."""
if isinstance(call.func, ast.Attribute):
return call.func.attr
return call.func.id if isinstance(call.func, ast.Name) else ""
def _arg_names(call: ast.Call) -> set[str]:
"""Every name `call` reads through its arguments."""
reads = [*call.args, *(kw.value for kw in call.keywords)]
return {
node.id
for arg in reads
for node in ast.walk(arg)
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
}
def _writes_through_args(call: ast.Call) -> bool:
"""Does `call` write through an argument rather than return a new value?
Torch spells this three ways: an `out=` destination, `inplace=True`, and the
trailing-underscore convention (`torch.relu_(x)`). The underscore is checked
on the function's own name, so it catches the free-function form that
`_inplace_target` cannot (there the base name is the module, `torch`).
"""
for keyword in call.keywords:
if keyword.arg == "out":
return True
if keyword.arg == "inplace" and not (
isinstance(keyword.value, ast.Constant) and keyword.value.value is False
):
return True
name = _call_name(call)
return name.endswith("_") and not name.endswith("__")
def written_names(region: list[ast.stmt]) -> set[str]:
"""Names `region` may rebind or write through.
Beyond plain rebinding and writes through the name itself (`_rebound_names`,
`_mutated_names`), a call can write through an argument without naming it as
a target: `F.relu(x, inplace=True)`, `torch.add(x, y, out=x)`,
`torch.relu_(x)`. A statement-level call whose result is discarded is treated
the same way, since it can only be there for a side effect.
"""
written = _rebound_names(region) | _mutated_names(region)
for stmt in region:
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
written |= _arg_names(stmt.value)
for node in ast.walk(stmt):
if isinstance(node, ast.Call) and _writes_through_args(node):
written |= _arg_names(node)
return written
_METADATA_ATTRS = frozenset(
{"shape", "dtype", "device", "ndim", "size", "dim", "numel", "item"}
)
def _aliasing_reads(node: ast.expr) -> set[str]:
"""Names `node` reads in a way that could yield a view of them.
Metadata (`x.shape`, `x.size(0)`, `x.dtype`) is ints and tuples sharing no
storage with `x`, so reaching a name only through metadata cannot alias it.
Shape arithmetic is pervasive between projections, so propagating through it
would taint nearly every later name.
"""
names: set[str] = set()
stack: list[ast.AST] = [node]
while stack:
current = stack.pop()
if isinstance(current, ast.Attribute) and current.attr in _METADATA_ATTRS:
continue
if (
isinstance(current, ast.Call)
and isinstance(current.func, ast.Attribute)
and current.func.attr in _METADATA_ATTRS
):
stack.extend(current.args)
stack.extend(keyword.value for keyword in current.keywords)
continue
if isinstance(current, ast.Name) and isinstance(current.ctx, ast.Load):
names.add(current.id)
stack.extend(ast.iter_child_nodes(current))
return names
def _returns_fresh(node: ast.expr, module: nn.Module) -> bool:
"""Does evaluating `node` allocate, rather than return a view of its input?
A `nn.Linear` writes its output into new storage, so its result aliases
nothing that went in, and a chain of view ops on top of it aliases only that
fresh tensor. Walking the receiver chain down to a `self.<linear>(...)` call
therefore proves the value cannot alias the projection input. A literal
aliases nothing either, and a branch or sequence is fresh when every part is
-- which is how a guarded projection (`p(x) if p is not None else None`)
stays untracked. A call on an attribute that is missing or `None` cannot run
at all, so a branch selecting a different configuration is fresh too.
"""
if isinstance(node, ast.Constant):
return True
if isinstance(node, ast.IfExp):
if isinstance(node.test, ast.Constant):
# A folded existence guard leaves the untaken branch as dead code.
taken = node.body if node.test.value else node.orelse
return _returns_fresh(taken, module)
return _returns_fresh(node.body, module) and _returns_fresh(node.orelse, module)
if isinstance(node, (ast.Tuple, ast.List)):
return all(_returns_fresh(elt, module) for elt in node.elts)
while isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
func = node.func
if isinstance(func.value, ast.Name) and func.value.id == "self":
attribute = getattr(module, func.attr, None)
if attribute is None or isinstance(attribute, nn.Linear):
# A missing or `None` attribute cannot be called, so the branch
# selecting it never runs (`q_proj` on a q-LoRA layer).
return True
# Anything else can only return storage it was handed, so it is
# fresh exactly when its inputs are (`q_norm(q_proj(x))`).
return bool(node.args) and all(
_returns_fresh(arg, module) for arg in node.args
)
node = func.value
return False
def aliasing_names(
funcdef: ast.FunctionDef, seed: set[str], module: nn.Module
) -> set[str]:
"""`seed` plus every name that may alias one of them.
A view shares storage with its base (`h = x.view(-1)`), so mutating `h`
mutates `x`; tracking only `seed` would miss that. Any assignment reading a
tracked name therefore taints its targets, transitively, unless its value is
freshly allocated (`_returns_fresh`).
"""
names = set(seed)
assigns = [
(node, node.value)
for node in ast.walk(funcdef)
if isinstance(node, (ast.Assign, ast.AnnAssign))
and node.value is not None
and not _returns_fresh(node.value, module)
]
changed = True
while changed:
changed = False
for node, value in assigns:
if not _aliasing_reads(value) & names:
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
stores = {
child.id
for target in targets
for child in ast.walk(target)
if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store)
}
if stores - names:
names |= stores
changed = True
return names
def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
"""Replace the expression `old` (by identity) with `new` within `module`."""
class _Replacer(ast.NodeTransformer):
def visit(self, node: ast.AST) -> ast.AST:
if node is old:
return new
return super().generic_visit(node)
_Replacer().visit(module)
def is_complete(graph: fx.Graph) -> bool:
"""Did the trace run to completion? A partial graph has no `output` node.
Only a complete graph can be read for the *absence* of an op; in a partial
one the tail is exactly what is missing."""
return any(node.op == "output" for node in graph.nodes)
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
"""The first node in `graph` matching `predicate`, or `None`."""
return next((n for n in graph.nodes if predicate(n)), None)
def output_value(graph: fx.Graph) -> object | None:
"""The value the graph's `output` node returns, if the trace reached one."""
output = find_node(graph, lambda n: n.op == "output")
if output is None or not output.args:
return None
return output.args[0]
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
"""Nearest linear producing `node`, walking back through splits/reshapes.
Non-linear submodules are transparent too (e.g. the dropout GPT-style
attentions apply after their output projection). Never walks through a leaf
call (e.g. an attention interface): its inputs are what attention consumes,
not what produced the value."""
stack = [node]
seen: set[fx.Node] = set()
while stack:
current = stack.pop()
if not isinstance(current, fx.Node) or current in seen:
continue
seen.add(current)
if is_linear(current, module):
return current
if current.op in (
"call_function",
"call_method",
"call_module",
) and not is_leaf_call(current):
stack.extend(current.args)
return None
def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
Never walks through a leaf call (e.g. an attention interface): what crosses
it is consumed by the attention computation, not projected."""
queue = list(node.users)
seen: set[fx.Node] = set()
while queue:
current = queue.pop(0)
if current in seen:
continue
seen.add(current)
if is_linear(current, module):
return current
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
queue.extend(current.users)
return None
def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None:
"""Name of the Linear producing the graph's (first) output value."""
value = output_value(graph)
if isinstance(value, (tuple, list)) and value:
value = value[0]
linear = upstream_linear(value, module)
return None if linear is None else str(linear.target)
def is_linear(node: fx.Node, module: nn.Module) -> bool:
"""Is node `nn.Linear.__call__()`."""
return node.op == "call_module" and isinstance(
module.get_submodule(node.target), nn.Linear
)
_DTYPE_CASTS = frozenset({"to", "float", "double", "half", "bfloat16", "type_as"})
def peel(node: object) -> object:
"""Strip dtype-cast wrappers (`.to(...)`, `.float()`, `.type_as(...)`)."""
while (
isinstance(node, fx.Node)
and node.op == "call_method"
and node.target in _DTYPE_CASTS
):
node = node.args[0]
return node
def is_fn(node: object, target: object) -> TypeIs[fx.Node]:
"""Is node `<target>()`."""
return (
isinstance(node, fx.Node)
and node.op == "call_function"
and node.target is target
)
def is_method(node: object, name: str) -> TypeIs[fx.Node]:
"""Is node `.<name>()`."""
return (
isinstance(node, fx.Node) and node.op == "call_method" and node.target == name
)
def is_op(node: object, name: str) -> TypeIs[fx.Node]:
"""Is node `<mod>.<name>()` for torch, F, operator, or Tensor."""
return any(
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
) or (hasattr(torch.Tensor, name) and is_method(node, name))
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layer provider resolution for the Transformers modeling backend.
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
resolved source of every symbol is logged so it is clear which layers run
hw-agnostic and which fell back to vLLM.
"""
import importlib
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
_VLLM_PKG = "vllm.model_executor.layers"
def _resolve(module: str, name: str):
"""Return `name` from the hw-agnostic `module` when enabled and available,
else from vLLM. Logs which source was used."""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
logger.info("Using hw-agnostic layer: %s", name)
return obj
except (ImportError, AttributeError):
logger.warning(
"hw-agnostic layer %s is not available; falling back to default", name
)
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
RMSNorm = _resolve("layernorm", "RMSNorm")
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
def get_act_and_mul_fn(act_fn_name: str):
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
Resolved per call because the op is name-parameterized: an activation with
no hw-agnostic equivalent falls back to vLLM individually.
"""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
from vllm.model_executor.hw_agnostic.layers.activation import (
get_act_and_mul_fn as hw_fn,
)
fn = hw_fn(act_fn_name)
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
return fn
except (ImportError, KeyError):
logger.warning_once(
"hw-agnostic activation %s is not available; falling back to vLLM",
act_fn_name,
)
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
return vllm_fn(act_fn_name)
@@ -1,80 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixin for legacy models."""
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.models.utils import WeightsMapper
from vllm.sequence import IntermediateTensors
from .base import Base
if TYPE_CHECKING:
from vllm.config import VllmConfig
class LegacyMixin(Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.hf_to_vllm_mapper |= WeightsMapper(
# Drop unsupported/unwanted output embeddings layers.
orig_to_new_prefix={
"model.lm_head.": None,
"model.predictions.": None,
"model.qa_outputs.": None,
"model.embeddings_project.": None,
"model.discriminator_predictions.": None,
},
orig_to_new_substr={
# Some encoder models have the position_ids buffer in the checkpoint.
# vLLM always passes position_ids as an argument, so drop the buffer.
"position_ids": None,
# Some encoder models have the bias of the final classifier layer in
# the checkpoint. vLLM does not use this bias, so drop it.
"score.bias": None,
},
)
# roberta-like models an extra padding in positions.
# FIXME(Isotr0py): This is quite hacky for roberta edge case,
# we should find a better way to handle this.
self.is_roberta = "roberta" in self.text_config.model_type
self.padding_idx = self.text_config.pad_token_id
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | IntermediateTensors:
if self.is_roberta:
# RoBERTa positions start at padding_idx + 1.
# Non-in-place add to avoid mutating the persistent GPU buffer --
# in-place += would accumulate on CUDA graph padding slots.
positions = positions + self.padding_idx + 1
return super().forward(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
**kwargs,
)
@@ -1,454 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
MoERunner,
RoutedExperts,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
from .utils import log_replacement, maybe_per_layer
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class TransformersMoEState:
topk_ids: torch.Tensor | None = None
is_sequence_parallel: bool = False
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersMoERunner(MoERunner):
"""Custom MoERunner for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
super().__init__(*args, **kwargs)
self._moe_state = moe_state
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
def forward( # type: ignore[override]
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
**kwargs: Any,
) -> torch.Tensor:
"""In Transformers `experts.forward` will have this signature, which is why
it deliberately does not match `MoERunner.forward`.
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
hidden_states,
topk_ids.to(torch.int32),
topk_weights.to(torch.float32),
self.layer_name,
)
def _forward_super(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
return super().forward(hidden_states, topk_weights)
def _transformers_moe_forward(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
"""Store the `topk_ids` in the layer and call the actual forward."""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._moe_state.topk_ids = topk_ids
return self._forward_super(hidden_states, topk_weights)
def _transformers_moe_forward_fake(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
return torch.empty_like(hidden_states)
direct_register_custom_op(
op_name="transformers_moe_forward",
op_func=_transformers_moe_forward,
mutates_args=["hidden_states"],
fake_impl=_transformers_moe_forward_fake,
tags=(torch.Tag.needs_fixed_stride_order,),
)
def _hf_topk_router_precomputed(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
weights arrive as `gating_output`, ids via `moe_state`."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
def _hf_topk_router(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
gate: nn.Module,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
_, topk_weights, topk_ids = gate(hidden_states)
return topk_weights, topk_ids
class TransformersRoutedExperts(RoutedExperts):
def get_expert_mapping(
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
common_map = super().get_expert_mapping(
ckpt_gate_proj_name or "gate_proj",
ckpt_down_proj_name or "down_proj",
ckpt_up_proj_name or "up_proj",
include_fused,
)
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
if not include_fused:
return common_map + mixtral_map
common_fused, common_unfused = common_map[:3], common_map[3:]
mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:]
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
class MoEMixin(MixtureOfExperts, Base):
mlp_layers: list[nn.Module]
"""MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods."""
# Redeclared because `recursive_replace` assigns it below the first read
num_local_physical_experts: int
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip MixtureOfExperts.__init__ and call the next class in MRO
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
def update_physical_experts_metadata(
self,
num_physical_experts: int,
num_local_physical_experts: int,
):
assert self.num_local_physical_experts == num_local_physical_experts
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for moe_block in self.mlp_layers:
moe_block.n_local_physical_experts = num_local_physical_experts
moe_block.n_physical_experts = num_physical_experts
moe_block.n_redundant_experts = self.num_redundant_experts
moe_block.experts.update_expert_map()
def recursive_replace(self):
"""Initialize the MoE layers."""
experts_name = "experts"
text_config = self.text_config
# Positional arguments
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(text_config, num_experts_names, None)
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
top_k = getattr_iter(text_config, top_k_names, None)
assert top_k is not None
hidden_size = text_config.hidden_size
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
assert intermediate_size is not None
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
# Common kwargs
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
# Routed scaling factor kwargs
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
# aiter applies routed_scaling_factor internally
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
routed_scaling_factor_kwargs = dict(
routed_scaling_factor=routed_scaling_factor,
apply_routed_scale_to_output=apply_routed_scale_to_output,
)
# Dtype the router computes in, if it is not the activation dtype.
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
if config_router_dtype is not None:
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
# Grouped topk routing kwargs
num_expert_group = getattr(text_config, "n_group", None)
topk_group = getattr(text_config, "topk_group", None)
use_grouped_topk = num_expert_group is not None and topk_group is not None
grouped_topk_routing_kwargs = dict(
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
# MoE activation function
activation = "silu"
wrapped_arch = self.config.architectures[0].lower()
if "gptoss" in wrapped_arch:
activation = "swigluoai"
# Expert parallel load balancing kwargs
enable_eplb = self.parallel_config.enable_eplb
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
# MixtureOfExperts mixin settings
ep_size = get_ep_group().world_size
self.mlp_layers = []
self.moe_layers = []
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
self.num_logical_experts = num_experts
self.num_physical_experts = num_experts + num_redundant_experts
self.num_local_physical_experts = self.num_physical_experts // ep_size
self.num_routed_experts = num_experts
self.num_shared_experts = num_shared_experts
self.num_redundant_experts = num_redundant_experts
# Down projections of shared experts consumed by FusedMoE
shared_down_projs: list[tuple[nn.Module, str]] = []
# Recursively fuse MoE layers
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
qual_name = maybe_prefix(prefix, child_name)
# Naive implementations will have experts as ModuleList
is_modulelist = isinstance(child_module, nn.ModuleList)
# Packed implementations will have experts as 3D tensors of shapes like:
# gate_up_proj = (num_experts, 2 * intermediate_size, hidden_size)
# down_proj = (num_experts, intermediate_size, hidden_size)
params = list(child_module.parameters())
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
if child_name == experts_name and (is_modulelist or is_3d):
# Alias for readability
moe_block = module
experts = child_module
# Class of the fused block (parent of gate/experts/shared)
moe_block_cls = type(moe_block).__name__
experts_cls = type(experts).__name__
# Do the experts have biases
has_bias = False
for experts_param_name, _ in experts.named_parameters():
if "bias" in experts_param_name:
has_bias = True
break
# Whether the expert weights are stored in transposed format.
# Comes from Transformers `use_experts_implementation` decorator.
is_transposed = getattr(experts, "is_transposed", False)
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
for moe_block_param_name, _ in moe_block.named_parameters():
if "shared_expert" in moe_block_param_name:
self.num_shared_experts = 1
break
# Only pay for the layer index if something is per-layer;
# `extract_layer_index` raises for unnumbered module paths.
per_layer = (top_k, intermediate_size, norm_topk_prob)
layer_idx = (
extract_layer_index(qual_name)
if any(isinstance(v, list) for v in per_layer)
else 0
)
layer_top_k = maybe_per_layer(top_k, layer_idx)
layer_renormalize = maybe_per_layer(norm_topk_prob, layer_idx)
if layer_renormalize is None:
layer_renormalize = layer_top_k > 1
kwargs: dict[str, Any] = dict(
num_experts=num_experts,
top_k=layer_top_k,
hidden_size=hidden_size,
intermediate_size=maybe_per_layer(intermediate_size, layer_idx),
renormalize=layer_renormalize,
quant_config=self.quant_config,
prefix=qual_name,
activation=activation,
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
is_fused_checkpoint_transposed=is_transposed,
routed_experts_cls=TransformersRoutedExperts,
)
# _maybe_apply_routed_scale_to_output edge case. Transformers
# decoder layers do not compensate for dividing by scaling factor.
divides_shared_output = (
routed_scaling_factor != 1.0
and apply_routed_scale_to_output
and self.model_config.dtype == torch.float16
)
fuser = MoEBlockFuser.match(
moe_block, experts_name, divides_shared_output
)
if fuser is not None:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in MoERunner.
shared_experts = fuser.shared_experts(moe_block, prefix)
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared, GLUFuser)
if glu_fuser is not None:
down_name = glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
if scoring_func := fuser.scoring_func:
# Internal routing kwargs.
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=scoring_func,
renormalize=(
layer_renormalize
if fuser.renormalize is None
else fuser.renormalize
),
gate=gate,
)
if router_dtype is not None:
kwargs["router_logits_dtype"] = router_dtype
if use_grouped_topk:
kwargs |= grouped_topk_routing_kwargs
if routed_scaling_factor != 1.0:
kwargs |= routed_scaling_factor_kwargs
bias = getattr(gate, "e_score_correction_bias", None)
if bias is not None:
kwargs["e_score_correction_bias"] = bias
else:
# External routing kwargs
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
_hf_topk_router,
gate=getattr(moe_block, fuser.gate_name),
),
)
kwargs |= dict(
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
shared_experts=shared_experts,
)
fuser.rewrite_forward(moe_block)
logger.info_once(
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
" + shared experts" if fuser.shared_name else "",
moe_block_cls,
"external" if fuser.scoring_func is None else "internal",
)
else:
# MoE block forward is unmodified.
# gate/router and shared expert (if any) runs in Transformers.
# We then smuggle the topk_ids in using a custom op.
moe_state = TransformersMoEState()
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
_hf_topk_router_precomputed, moe_state=moe_state
),
runner_cls=TransformersMoERunner,
runner_args={"moe_state": moe_state},
)
logger.info_once(
"Fused: experts (%s) -> MoERunner (external routing)",
experts_cls,
)
fused_experts = FusedMoEFactory(**kwargs)
moe_block.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
self.mlp_layers.append(moe_block)
self.moe_layers.append(fused_experts)
else:
_recursive_replace(child_module, prefix=qual_name)
_recursive_replace(self.model, prefix="model")
self.num_moe_layers = len(self.moe_layers)
# Continue with the replacement of layers in Base
super().recursive_replace()
# GLUFuser likely fused shared_experts. The down projection after the GLU
# normally immediately reduces but we want FusedMoE to handle the reduction.
for hf_shared, down_name in shared_down_projs:
hf_shared.get_submodule(down_name).reduce_results = False
File diff suppressed because it is too large Load Diff
@@ -1,407 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend utilities."""
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from itertools import chain
from operator import attrgetter
from pathlib import Path
from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar
import torch
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.utils import maybe_prefix
from vllm.transformers_utils.config import iter_rope_parameters
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
# Copied from `accelerate`
@contextmanager
def init_on_device_without_buffers(device: torch.device):
"""A context manager under which models are initialized with all
parameters on the specified device. However buffers are not
initialized on specified device.
Args:
device (`torch.device`):
Device to initialize all parameters on.
"""
old_register_parameter = nn.Module.register_parameter
def register_empty_parameter(module, name, param):
old_register_parameter(module, name, param)
if param is not None:
param_cls = type(module._parameters[name])
kwargs = module._parameters[name].__dict__
kwargs["requires_grad"] = param.requires_grad
module._parameters[name] = param_cls(
module._parameters[name].to(device), **kwargs
)
tensor_constructors_to_patch: dict[str, Callable] = {}
def patch_tensor_constructor(fn):
def wrapper(*args, **kwargs):
kwargs["device"] = device
return fn(*args, **kwargs)
return wrapper
try:
nn.Module.register_parameter = register_empty_parameter
for torch_function_name in tensor_constructors_to_patch:
setattr(
torch,
torch_function_name,
patch_tensor_constructor(getattr(torch, torch_function_name)),
)
yield
finally:
nn.Module.register_parameter = old_register_parameter
for (
torch_function_name,
old_torch_function,
) in tensor_constructors_to_patch.items():
setattr(torch, torch_function_name, old_torch_function)
Style = Literal[
"colwise",
"rowwise",
"replicate",
"colwise_gather_output",
"rowwise_split_input",
]
def replace_linear_class(
linear: nn.Linear,
style: Style = "replicate",
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
"""Replace nn.Linear with one of vLLM's tensor parallel linear classes.
Args:
linear: `nn.Linear` to be replaced.
style: Tensor parallel style of the new linear, e.g. "colwise".
quant_config: Quantization config for the new linear.
prefix: Module prefix of the new linear, used for quantization lookup.
Returns:
The new linear.
"""
if not isinstance(style, str):
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
vllm_linear_cls, vllm_linear_kwargs = {
"colwise": (ColumnParallelLinear, {}),
"rowwise": (RowParallelLinear, {}),
"replicate": (ReplicatedLinear, {}),
"colwise_gather_output": (ColumnParallelLinear, {"gather_output": True}),
"rowwise_split_input": (RowParallelLinear, {"input_is_parallel": False}),
}.get(style, (ReplicatedLinear, {}))
return vllm_linear_cls(
input_size=linear.in_features,
output_size=linear.out_features,
bias=linear.bias is not None,
quant_config=quant_config,
prefix=prefix,
return_bias=False,
**vllm_linear_kwargs,
)
TorchConv: TypeAlias = nn.Conv2d | nn.Conv3d
VllmConv: TypeAlias = Conv2dLayer | Conv3dLayer
def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
"""Replace a Transformers Conv2d/Conv3d with vLLM's Conv2d/Conv3d.
Args:
conv: `nn.Conv2d` or `nn.Conv3d` to be replaced.
Returns:
The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported,
returns the original conv module.
"""
# vLLM does not handle non-zero padding modes
if conv.padding_mode != "zeros":
return conv
vllm_conv_cls = {
nn.Conv2d: Conv2dLayer,
nn.Conv3d: Conv3dLayer,
}.get(type(conv))
if vllm_conv_cls is None:
return conv
return vllm_conv_cls(
in_channels=conv.in_channels,
out_channels=conv.out_channels,
kernel_size=conv.kernel_size,
stride=conv.stride,
padding=conv.padding,
dilation=conv.dilation,
groups=conv.groups,
bias=conv.bias is not None,
padding_mode=conv.padding_mode,
params_dtype=conv.weight.dtype,
)
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
`LayerNorm`.
Args:
layernorm: `nn.LayerNorm` to be replaced.
Returns:
The new `LayerNorm`. If the layernorm is not supported (a subclass with
its own behavior, multi-dim `normalized_shape`, or not both
elementwise-affine and biased), returns the original module unchanged.
"""
if (
type(layernorm) is not nn.LayerNorm
or len(layernorm.normalized_shape) != 1
or not layernorm.elementwise_affine
or layernorm.bias is None
):
return layernorm
return LayerNorm(
layernorm.normalized_shape[0],
eps=layernorm.eps,
dtype=layernorm.weight.dtype,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
def setter(obj: object, value: object):
attr_parent = attrgetter(parent)(obj) if parent else obj
setattr(attr_parent, name, value)
return setter
class _UninitializedEmbedding(nn.Embedding):
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
def __init__(self):
pass
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
_rebased_embedding_classes: dict[type[nn.Embedding], type[VocabParallelEmbedding]] = {}
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
Args:
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
maps to the same class.
Returns:
The new class, to assign to `__class__` of an instance of `cls`.
"""
if cls not in _rebased_embedding_classes:
rebased_cls = type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
_rebased_embedding_classes[cls] = rebased_cls
return _rebased_embedding_classes[cls]
def replace_embedding_class(
embedding: nn.Embedding,
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> nn.Module:
"""Replace `nn.Embedding` with `VocabParallelEmbedding`.
Args:
embedding: The `nn.Embedding` holding a vocab table.
quant_config: Quantization config for the new embedding.
prefix: Qualname of `embedding`, used to look up its quantization method.
Returns:
The module to install in `embedding`'s place.
"""
# Tied tables are reached more than once, so don't rebase an already new embedding
if isinstance(embedding, VocabParallelEmbedding):
return embedding
kwargs = dict(
num_embeddings=embedding.num_embeddings,
embedding_dim=embedding.embedding_dim,
params_dtype=embedding.weight.dtype,
quant_config=quant_config,
prefix=prefix,
)
# If `embedding` is a bare `nn.Embedding`, simple replace
if type(embedding) is nn.Embedding:
return VocabParallelEmbedding(**kwargs)
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
VocabParallelEmbedding.__init__(embedding, **kwargs)
return embedding
def recursive_replace_linear(
model: nn.Module,
quant_config: "QuantizationConfig | None",
prefix: str = "",
):
"""Recursively replace linear modules in the model as needed."""
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
new_module = child_module
qual_name = maybe_prefix(prefix, child_name)
# Replace modules as needed
if isinstance(child_module, nn.Linear):
new_module = replace_linear_class(
child_module,
"replicate",
quant_config,
prefix=qual_name,
)
else:
_recursive_replace(child_module, prefix=qual_name)
if new_module is not child_module:
setattr(module, child_name, new_module)
_recursive_replace(model, prefix=prefix)
T = TypeVar("T")
def maybe_per_layer(value: T | list[T], layer_idx: int) -> T:
"""Pick `layer_idx`'s entry from a config field that may be sized per layer."""
return value[layer_idx] if isinstance(value, list) else value
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
logger.debug("%s: %s -> %s", name, old_module, new_module)
def get_feature_request_tip(
model: str,
trust_remote_code: bool,
) -> str:
hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new"
gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose"
url = hf_url if trust_remote_code else gh_url
prefix = f"Please open {url} to request support for this feature. "
if Path(model).exists():
prefix = ""
doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models"
tip = f"See {doc_url} for instructions on how to add support yourself."
return f"{prefix}{tip}"
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
if max_position_embeddings is None:
return False
return vllm_config.model_config.max_model_len <= max_position_embeddings
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
Defaults to `True` but is disabled in the following situations:
- The model uses dynamic rope scaling that can still rescale within
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
if "dynamic" not in rope_types:
return True
# Layer types using longrope still sync, even once dynamic rope is disabled
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
The update syncs on `torch.max(position_ids)` every forward, which breaks
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
"default" skips it while keeping the `inv_freq` computed at init.
"""
if not dynamic_rope_update_is_noop(vllm_config):
return
for module in model.modules():
rope_type = getattr(module, "rope_type", None)
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
continue
if isinstance(rope_type, dict):
module.rope_type = {
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
}
elif rope_type == "dynamic":
module.rope_type = "default"
+3 -30
View File
@@ -6,7 +6,6 @@ happen during model execution.
"""
import json
import sys
import time
from pathlib import Path
from typing import TYPE_CHECKING
@@ -134,25 +133,6 @@ def _warmup_bf16x3_router_gemm(
logger.info_once("Warmed up BF16x3 router GEMM configs: %s.", configs)
def _warmup_gemm_rs_ar() -> None:
# Model construction (Kimi-K3, DeepSeek-V4.1) imports this module only
# when GEMM-RS/AR is enabled and initializes its singleton before
# kernel_warmup runs. Avoid importing it here so other models do not
# compile the RS/AR variants.
module = sys.modules.get("vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar")
if module is None:
return
compiled = module.warmup_gemm_rs_ar()
if compiled:
logger.info_once("Warmed up %d GEMM-RS/AR variants.", compiled)
def _autotune_kimi_k3_kda_qkvg(model: torch.nn.Module) -> None:
module = sys.modules.get("vllm.models.kimi_k3.nvidia.low_latency_gemm")
if module is not None:
module.autotune_kda_qkvg(model)
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
if not worker.use_v2_model_runner:
# The KV-block zeroing kernel is driven by the scheduler's
@@ -202,8 +182,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
if current_platform.has_device_capability(90):
_warmup_ll_bf16_router_gemm(worker.get_model())
_warmup_gemm_rs_ar()
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
# to the shared JIT warmup infrastructure.
@@ -478,16 +456,11 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
)
skip_attn = skip_attn or hisparse_enabled
if hisparse_enabled:
# HiSparse hot-buffer attention is bounded by decode batch
# size, not the prefill-sized batch used for the full model.
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
autotune_hisparse_flashinfer_attention,
)
autotune_hisparse_flashinfer_attention(runner)
# HiSparse config still bounds decode-batch autotune even
# though the connector was removed from this build.
logger.info_once("HiSparse configured; skipping attention autotune.")
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
replayssm_autotune_warmup(runner)
_autotune_kimi_k3_kda_qkvg(runner.get_model())
finally:
set_autotune_process_group(None)