[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,271 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select
|
||||
from vllm.model_executor.layers.indexer_topk import (
|
||||
deep_select_topk,
|
||||
get_deep_select_stride_requirement,
|
||||
is_deep_select_supported,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _max_with_nan(a, b):
|
||||
return tl.maximum(a, b, propagate_nan=tl.PropagateNan.ALL)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["width", "nblocks"])
|
||||
def _block_scores_kernel(
|
||||
logits,
|
||||
starts,
|
||||
ends,
|
||||
scores,
|
||||
valid_blocks,
|
||||
stride_row,
|
||||
stride_col,
|
||||
stride_start,
|
||||
stride_end,
|
||||
stride_scores,
|
||||
width,
|
||||
nblocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_STARTS: tl.constexpr,
|
||||
ROW_REPEAT: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
FULL_WIDTH: tl.constexpr,
|
||||
):
|
||||
# Writes each row's valid block count; FULL_WIDTH also scores the -inf tail.
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||||
end = tl.load(ends + row // ROW_REPEAT * stride_end)
|
||||
col_end = tl.minimum(end, width)
|
||||
num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE)
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(valid_blocks + row, num_valid)
|
||||
num_blocks = nblocks if FULL_WIDTH else num_valid
|
||||
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
|
||||
num_tiles = tl.cdiv(num_blocks, TILE)
|
||||
for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)):
|
||||
blocks = tile_id * TILE + tl.arange(0, TILE)
|
||||
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
|
||||
values = tl.load(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
(offsets[None, :] < BLOCK_SIZE) & (cols < col_end),
|
||||
other=-float("inf"),
|
||||
).to(tl.float32)
|
||||
reduced = tl.reduce(values, 1, _max_with_nan)
|
||||
# Pin the newest block. DeepSelect leaves NaN rows partially written,
|
||||
# so also pin NaN blocks (torch.topk ranks NaN first too).
|
||||
pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE)
|
||||
if not FULL_WIDTH:
|
||||
pinned |= reduced != reduced
|
||||
reduced = tl.where(pinned, float("inf"), reduced)
|
||||
tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["k"])
|
||||
def _store_candidates_kernel(
|
||||
values,
|
||||
indices,
|
||||
output,
|
||||
out_stride_row,
|
||||
out_stride_col,
|
||||
k,
|
||||
OUT_K: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
|
||||
value = tl.load(values + row * k + cols, cols < k, other=-float("inf"))
|
||||
index = tl.load(indices + row * k + cols, cols < k, other=-1)
|
||||
# NaN scores can occur during warmup; only -inf denotes padding.
|
||||
tl.store(
|
||||
output + row * out_stride_row + cols * out_stride_col,
|
||||
tl.where(value != -float("inf"), index, -1),
|
||||
cols < OUT_K,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["width", "nblocks"])
|
||||
def _candidate_flags_kernel(
|
||||
candidates,
|
||||
starts,
|
||||
flags,
|
||||
stride_row,
|
||||
stride_col,
|
||||
stride_start,
|
||||
width,
|
||||
nblocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
HAS_STARTS: tl.constexpr,
|
||||
ROW_REPEAT: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
offsets = tl.arange(0, 1024)
|
||||
for tile in range(tl.cdiv(nblocks + 1, 1024)):
|
||||
slots = tile * 1024 + offsets
|
||||
tl.store(flags + row * (nblocks + 1) + slots, 0, slots <= nblocks)
|
||||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||||
cols = tl.arange(0, triton.next_power_of_2(K))
|
||||
block = tl.load(
|
||||
candidates + row * stride_row + cols * stride_col, cols < K, other=-1
|
||||
).to(tl.int64)
|
||||
# Preserve the packed-column clamp for candidates beyond the logits width.
|
||||
block = tl.where(start + block * BLOCK_SIZE >= width, nblocks, block)
|
||||
tl.debug_barrier()
|
||||
tl.store(flags + row * (nblocks + 1) + block, 1, (cols < K) & (block >= 0))
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["width", "nblocks"])
|
||||
def _mask_candidates_kernel(
|
||||
logits,
|
||||
starts,
|
||||
ends,
|
||||
flags,
|
||||
stride_row,
|
||||
stride_col,
|
||||
stride_start,
|
||||
stride_end,
|
||||
width,
|
||||
nblocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_STARTS: tl.constexpr,
|
||||
ROW_REPEAT: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
|
||||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||||
end = tl.load(ends + row // ROW_REPEAT * stride_end)
|
||||
valid = (cols >= start) & (cols < end) & (cols < width)
|
||||
block = (cols - start) // BLOCK_SIZE
|
||||
keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0)
|
||||
edge = tl.load(flags + row * (nblocks + 1) + nblocks)
|
||||
keep = (keep != 0) | ((cols == width - 1) & (edge != 0))
|
||||
tl.store(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
-float("inf"),
|
||||
(cols < width) & ~(valid & keep),
|
||||
)
|
||||
|
||||
|
||||
def select_candidate_blocks(
|
||||
logits: torch.Tensor,
|
||||
row_ks: torch.Tensor | None,
|
||||
row_ke: torch.Tensor,
|
||||
topk_blocks: int,
|
||||
block_size: int,
|
||||
out: torch.Tensor,
|
||||
row_repeat: int = 1,
|
||||
) -> None:
|
||||
"""Select local block IDs by maximum score, pinning each row's newest block.
|
||||
|
||||
Row bounds are in packed column space; absent starts mean zero.
|
||||
Decode rows share bounds in groups of ``row_repeat``. Output is -1 padded.
|
||||
"""
|
||||
assert logits.is_cuda
|
||||
rows, width = logits.shape
|
||||
if not rows:
|
||||
return
|
||||
if not width:
|
||||
out.fill_(-1)
|
||||
return
|
||||
nblocks = triton.cdiv(width, block_size)
|
||||
# Logits are max_model_len wide (CUDA-graph static) but rows are shorter:
|
||||
# DeepSelect stops at each row's valid block count. Order does not matter.
|
||||
use_deep_select = has_deep_select() and nblocks >= topk_blocks
|
||||
if use_deep_select:
|
||||
in_align, out_align = get_deep_select_stride_requirement()
|
||||
padded = round_up(nblocks, in_align // logits.element_size())
|
||||
scores = logits.new_empty((rows, padded))[:, :nblocks]
|
||||
use_deep_select = (
|
||||
out.stride(1) == 1
|
||||
and out.stride(0) * out.element_size() % out_align == 0
|
||||
and is_deep_select_supported(scores, topk_blocks)
|
||||
)
|
||||
else:
|
||||
scores = logits.new_empty((rows, nblocks))
|
||||
valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device)
|
||||
_block_scores_kernel[(rows, 32)](
|
||||
logits,
|
||||
row_ks,
|
||||
row_ke,
|
||||
scores,
|
||||
valid_blocks,
|
||||
*logits.stride(),
|
||||
row_ks.stride(0) if row_ks is not None else 0,
|
||||
row_ke.stride(0),
|
||||
scores.stride(0),
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
TILE=128,
|
||||
FULL_WIDTH=not use_deep_select,
|
||||
)
|
||||
if use_deep_select:
|
||||
deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out)
|
||||
return
|
||||
# Keep the existing top-k tie behavior.
|
||||
top = scores.topk(min(topk_blocks, nblocks), dim=-1)
|
||||
_store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))](
|
||||
top.values,
|
||||
top.indices,
|
||||
out,
|
||||
*out.stride(),
|
||||
top.values.shape[1],
|
||||
topk_blocks,
|
||||
256,
|
||||
)
|
||||
|
||||
|
||||
def apply_candidate_mask(
|
||||
logits: torch.Tensor,
|
||||
row_ks: torch.Tensor | None,
|
||||
row_ke: torch.Tensor,
|
||||
candidate_blocks: torch.Tensor,
|
||||
block_size: int,
|
||||
row_repeat: int = 1,
|
||||
) -> None:
|
||||
"""Mask packed logits outside causal bounds and request-local candidates."""
|
||||
assert logits.is_cuda
|
||||
rows, width = logits.shape
|
||||
if not rows or not width:
|
||||
return
|
||||
nblocks = triton.cdiv(width, block_size)
|
||||
flags = torch.empty((rows, nblocks + 1), device=logits.device, dtype=torch.uint8)
|
||||
start_stride = row_ks.stride(0) if row_ks is not None else 0
|
||||
_candidate_flags_kernel[(rows,)](
|
||||
candidate_blocks,
|
||||
row_ks,
|
||||
flags,
|
||||
*candidate_blocks.stride(),
|
||||
start_stride,
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
candidate_blocks.shape[1],
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
)
|
||||
_mask_candidates_kernel[(rows, triton.cdiv(width, 1024))](
|
||||
logits,
|
||||
row_ks,
|
||||
row_ke,
|
||||
flags,
|
||||
*logits.stride(),
|
||||
start_stride,
|
||||
row_ke.stride(0),
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
1024,
|
||||
)
|
||||
@@ -1,498 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepGEMM sparse MQA-logits path for the DeepSeek V4.1 two-level indexer.
|
||||
|
||||
The candidate-source indexer publishes the top candidate blocks (in units of
|
||||
``candidate_block_size`` compressed positions). Instead of scoring every KV
|
||||
position and masking the dense logits down to those blocks, this path scores
|
||||
only the candidate tokens with DeepGEMM's sparse MQA logits kernels
|
||||
(``fp8_fp4_(paged_)sparse_mqa_logits``, DeepGEMM >= 2.8, SM100 only), runs
|
||||
DeepSelect's top-k directly on the bf16 logits they produce, and remaps the
|
||||
selected columns back to request-local positions.
|
||||
|
||||
Only the MXFP4 indexer cache is supported: the kernels require UE8M0-packed
|
||||
(granularity-32) Q/KV scales, which is exactly the MXFP4 cache layout. The
|
||||
FP8 indexer cache stores one fp32 scale per quant block instead.
|
||||
|
||||
The functions here take caller-owned output buffers (``sparse_indices``,
|
||||
``end``, ``col_indices``) so the hot path allocates nothing; the indexer
|
||||
metadata builder owns them.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.indexer_topk import (
|
||||
deep_select_topk,
|
||||
get_deep_select_stride_requirement,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_paged_sparse_mqa_logits,
|
||||
fp8_fp4_sparse_mqa_logits,
|
||||
get_paged_sparse_mqa_logits_metadata,
|
||||
get_sparse_mqa_logits_metadata,
|
||||
)
|
||||
|
||||
# Sparse-block sizes (in tokens) supported by the DeepGEMM kernels.
|
||||
SPARSE_BLOCK_KV_CHOICES = (16, 8)
|
||||
|
||||
|
||||
def has_deep_select() -> bool:
|
||||
"""Whether the DeepSelect top-k extension is built and this GPU runs it."""
|
||||
return (
|
||||
importlib.util.find_spec("vllm._deepselect_C") is not None
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
|
||||
|
||||
def check_deep_select_layout(num_sparse_cols: int, topk_tokens: int) -> None:
|
||||
"""Raise unless DeepSelect accepts the sparse logits layout.
|
||||
|
||||
The sparse kernels return contiguous bf16 ``[rows, num_sparse_cols]``
|
||||
logits and the top-k writes int32 ``[rows, topk_tokens]`` indices;
|
||||
DeepSelect needs both row strides aligned (1024B / 32B on SM100).
|
||||
"""
|
||||
in_align, out_align = get_deep_select_stride_requirement()
|
||||
logits_row_bytes = num_sparse_cols * torch.bfloat16.itemsize
|
||||
if num_sparse_cols >= 2**23 or logits_row_bytes % in_align != 0:
|
||||
raise ValueError(
|
||||
f"DeepSelect needs the sparse logits row ({num_sparse_cols} bf16 "
|
||||
f"columns, {logits_row_bytes} B) to be {in_align}B-aligned."
|
||||
)
|
||||
if topk_tokens > 4096 or (topk_tokens * torch.int32.itemsize) % out_align != 0:
|
||||
raise ValueError(
|
||||
f"DeepSelect needs topk <= 4096 with a {out_align}B-aligned int32 "
|
||||
f"output row, got topk={topk_tokens}."
|
||||
)
|
||||
|
||||
|
||||
_INT32_BIG = tl.constexpr(2**31 - 1)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize_on_alignment=["end_ptr"])
|
||||
def _expand_candidates_kernel(
|
||||
cand_ptr,
|
||||
cand_stride,
|
||||
ks_ptr,
|
||||
ks_stride,
|
||||
ke_ptr,
|
||||
ke_stride,
|
||||
si_ptr,
|
||||
si_stride,
|
||||
end_ptr,
|
||||
end_stride,
|
||||
K: tl.constexpr,
|
||||
S: tl.constexpr,
|
||||
RATIO: tl.constexpr,
|
||||
SBK: tl.constexpr,
|
||||
):
|
||||
"""One program per row: expand + filter + sort candidate blocks.
|
||||
|
||||
Each row's valid sparse blocks come out sorted ascending (no dedup pass:
|
||||
production candidates are unique, and the kernel tolerates repeats),
|
||||
padded by repeating the last valid block — DeepGEMM's padding convention.
|
||||
``end`` counts valid sparse-token columns; valid columns form a prefix
|
||||
because block ids are sorted.
|
||||
"""
|
||||
row = tl.program_id(0)
|
||||
ks = tl.load(ks_ptr + row * ks_stride)
|
||||
ke = tl.load(ke_ptr + row * ke_stride)
|
||||
cols = tl.arange(0, K)
|
||||
cand = tl.load(cand_ptr + row * cand_stride + cols) # K is pow2, no mask
|
||||
if RATIO == 2:
|
||||
cand = tl.interleave(cand * RATIO, cand * RATIO + 1)
|
||||
|
||||
# Keep only blocks inside the row's KV range [0, ke - ks); the lower
|
||||
# bound is free since candidates are non-negative.
|
||||
kv_len = tl.maximum(ke - ks, 0)
|
||||
max_block = (kv_len + SBK - 1) // SBK
|
||||
valid = (cand >= 0) & (cand < max_block)
|
||||
vals = tl.where(valid, cand, _INT32_BIG)
|
||||
vals = tl.sort(vals)
|
||||
|
||||
# Anchor at the row's ks rounded down to a sparse-block boundary
|
||||
# (identity for the paged path, where ks == 0).
|
||||
base = ks // SBK
|
||||
si = tl.where(vals != _INT32_BIG, vals, 0) + base
|
||||
n_valid = tl.sum(valid.to(tl.int32))
|
||||
last_rel = tl.max(tl.where(valid, cand, -1))
|
||||
pad_val = tl.where(n_valid > 0, last_rel + base, base)
|
||||
tl.store(
|
||||
si_ptr + row * si_stride + tl.arange(0, S),
|
||||
tl.where(tl.arange(0, S) < n_valid, si, pad_val),
|
||||
)
|
||||
|
||||
# Valid sparse-token column count; only the last block can be partial.
|
||||
# Absolute start of the last block: (base + last_rel) * SBK + ks % SBK,
|
||||
# which is exactly last_rel * SBK + ks.
|
||||
last_start = last_rel * SBK + ks
|
||||
fill = tl.minimum(tl.maximum(ke - last_start, 1), SBK)
|
||||
end = tl.where(n_valid > 0, (n_valid - 1) * SBK + fill, 0)
|
||||
tl.store(end_ptr + row * end_stride, end)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_topk_remap_kernel(
|
||||
col_ptr,
|
||||
col_stride,
|
||||
si_ptr,
|
||||
si_stride,
|
||||
ks_ptr,
|
||||
ks_stride,
|
||||
out_ptr,
|
||||
out_stride,
|
||||
k,
|
||||
width,
|
||||
SBK: tl.constexpr,
|
||||
K_POW2: tl.constexpr,
|
||||
):
|
||||
"""Remap sparse top-k columns to request-local positions.
|
||||
|
||||
Column ``j * SBK + o`` scores the token at
|
||||
``(si[row, j] - ks // SBK) * SBK + o``; columns outside ``[0, width)``
|
||||
(including DeepSelect's sentinel for NaN rows) become -1.
|
||||
"""
|
||||
row = tl.program_id(0)
|
||||
ks_base = tl.load(ks_ptr + row * ks_stride) // SBK
|
||||
cols = tl.arange(0, K_POW2)
|
||||
c = tl.load(col_ptr + row * col_stride + cols, mask=cols < k, other=-1)
|
||||
valid = (c >= 0) & (c < width)
|
||||
cc = tl.where(valid, c, 0)
|
||||
blocks = tl.load(si_ptr + row * si_stride + cc // SBK)
|
||||
pos = (blocks - ks_base) * SBK + cc % SBK
|
||||
tl.store(out_ptr + row * out_stride + cols, tl.where(valid, pos, -1), mask=cols < k)
|
||||
|
||||
|
||||
def pick_sparse_block_kv(candidate_block_size: int) -> int:
|
||||
"""Largest sparse-block size (tokens) dividing the candidate block size."""
|
||||
for sbk in SPARSE_BLOCK_KV_CHOICES:
|
||||
if candidate_block_size % sbk == 0:
|
||||
return sbk
|
||||
raise ValueError(
|
||||
f"candidate_block_size={candidate_block_size} is not a multiple of "
|
||||
f"any supported sparse_block_kv {SPARSE_BLOCK_KV_CHOICES}."
|
||||
)
|
||||
|
||||
|
||||
def candidate_blocks_to_sparse_indices(
|
||||
candidate_blocks: torch.Tensor,
|
||||
row_ks: torch.Tensor,
|
||||
row_ke: torch.Tensor,
|
||||
candidate_block_size: int,
|
||||
sparse_block_kv: int,
|
||||
out: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Expand request-local candidate blocks into DeepGEMM sparse block ids.
|
||||
|
||||
Args:
|
||||
candidate_blocks: [rows, K] int32 request-local candidate block ids
|
||||
(-1 padded), in units of ``candidate_block_size`` positions. K
|
||||
must be a power of two (the in-kernel sort); rows may be strided.
|
||||
row_ks: [rows] int32 per-row K range start; bounds are in the same
|
||||
(packed-workspace) coordinates the sparse kernel iterates over.
|
||||
Pass zeros for the paged path, whose blocks are context-relative.
|
||||
row_ke: [rows] int32 per-row K range end, in the same coordinates as
|
||||
``row_ks``.
|
||||
candidate_block_size: Positions per candidate block.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
out: Optional ``(sparse_indices, end)`` buffers to write into,
|
||||
shaped [rows, K * ratio] and [rows], both int32.
|
||||
|
||||
Returns:
|
||||
sparse_indices: [rows, K * ratio] int32 block ids for DeepGEMM,
|
||||
sorted ascending per row with the valid prefix first (unique in
|
||||
production, where candidates are unique) and the remaining slots
|
||||
repeating the last valid block (the kernel's padding convention).
|
||||
With the unaligned-ks variant, block ``i`` starts at
|
||||
``i * sparse_block_kv + ks % sparse_block_kv``.
|
||||
end: [rows] int32 count of valid sparse-token columns; valid columns
|
||||
always form a prefix because the block ids are sorted.
|
||||
|
||||
"""
|
||||
rows, num_candidates = candidate_blocks.shape
|
||||
ratio = candidate_block_size // sparse_block_kv
|
||||
assert ratio * sparse_block_kv == candidate_block_size and ratio in (1, 2), (
|
||||
candidate_block_size,
|
||||
sparse_block_kv,
|
||||
)
|
||||
assert num_candidates & (num_candidates - 1) == 0, num_candidates
|
||||
num_sparse = num_candidates * ratio
|
||||
if out is None:
|
||||
device = candidate_blocks.device
|
||||
out = (
|
||||
torch.empty(rows, num_sparse, dtype=torch.int32, device=device),
|
||||
torch.empty(rows, dtype=torch.int32, device=device),
|
||||
)
|
||||
sparse_indices, end = out
|
||||
assert sparse_indices.shape == (rows, num_sparse) and end.shape == (rows,)
|
||||
assert candidate_blocks.stride(1) == 1 and sparse_indices.stride(1) == 1
|
||||
for t in (candidate_blocks, row_ks, row_ke, sparse_indices, end):
|
||||
assert t.dtype == torch.int32, t.dtype
|
||||
|
||||
_expand_candidates_kernel[(rows,)](
|
||||
candidate_blocks,
|
||||
candidate_blocks.stride(0),
|
||||
row_ks,
|
||||
row_ks.stride(0),
|
||||
row_ke,
|
||||
row_ke.stride(0),
|
||||
sparse_indices,
|
||||
sparse_indices.stride(0),
|
||||
end,
|
||||
end.stride(0),
|
||||
K=num_candidates,
|
||||
S=num_sparse,
|
||||
RATIO=ratio,
|
||||
SBK=sparse_block_kv,
|
||||
num_warps=16,
|
||||
)
|
||||
return sparse_indices, end
|
||||
|
||||
|
||||
def sparse_topk_remap(
|
||||
logits: torch.Tensor,
|
||||
sparse_indices: torch.Tensor,
|
||||
end: torch.Tensor,
|
||||
row_ks: torch.Tensor,
|
||||
sparse_block_kv: int,
|
||||
topk_tokens: int,
|
||||
topk_indices: torch.Tensor,
|
||||
*,
|
||||
col_indices: torch.Tensor,
|
||||
) -> None:
|
||||
"""Row top-k over sparse logits, remapped to request-local positions.
|
||||
|
||||
Column ``j * sparse_block_kv + o`` of ``logits`` scores the token at
|
||||
request-local position
|
||||
``(sparse_indices[row, j] - ks // sparse_block_kv) * sparse_block_kv + o``.
|
||||
|
||||
DeepSelect consumes the bf16 logits as they come out of the sparse
|
||||
kernels; the valid columns form a prefix (block ids are sorted), so the
|
||||
per-row ``end`` bounds drive it directly. Slots beyond a row's valid
|
||||
count come back as -1.
|
||||
|
||||
Args:
|
||||
logits: [rows, width] bf16 sparse logits, as produced by the sparse
|
||||
kernels.
|
||||
sparse_indices: [rows, S] int32 sparse block ids backing ``logits``.
|
||||
end: [rows] int32 per-row count of valid logit columns.
|
||||
row_ks: [rows] int32 per-row K range start.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
topk_tokens: Number of tokens to keep per row.
|
||||
topk_indices: [rows, topk_tokens] int32 output buffer.
|
||||
col_indices: [rows, topk_tokens] int32 scratch for the sparse-column
|
||||
top-k result (row stride 32B-aligned for DeepSelect).
|
||||
|
||||
"""
|
||||
rows, width = logits.shape
|
||||
assert logits.dtype == torch.bfloat16 and width >= topk_tokens
|
||||
assert end.dtype == torch.int32 and end.shape == (rows,)
|
||||
cols = col_indices[:rows]
|
||||
deep_select_topk(logits, topk_tokens, end=end, output_idx=cols)
|
||||
_sparse_topk_remap_kernel[(rows,)](
|
||||
cols,
|
||||
cols.stride(0),
|
||||
sparse_indices,
|
||||
sparse_indices.stride(0),
|
||||
row_ks,
|
||||
row_ks.stride(0),
|
||||
topk_indices,
|
||||
topk_indices.stride(0),
|
||||
topk_tokens,
|
||||
width,
|
||||
SBK=sparse_block_kv,
|
||||
K_POW2=triton.next_power_of_2(topk_tokens),
|
||||
)
|
||||
|
||||
|
||||
def sparse_mqa_logits_prefill_chunk(
|
||||
q: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
k_quant: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
candidate_blocks: torch.Tensor,
|
||||
candidate_block_size: int,
|
||||
sparse_block_kv: int,
|
||||
topk_tokens: int,
|
||||
topk_indices: torch.Tensor,
|
||||
*,
|
||||
sparse_indices: torch.Tensor,
|
||||
end: torch.Tensor,
|
||||
col_indices: torch.Tensor,
|
||||
kernel_metadata: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Sparse-MQA logits + top-k for one prefill chunk (packed KV workspace).
|
||||
|
||||
Args:
|
||||
q: [rows, H, D] packed Q values (MXFP4 viewed as int8).
|
||||
q_scale: [rows, H] int32 packed UE8M0 Q scales.
|
||||
k_quant: [total_kv, D] packed K workspace (MXFP4 viewed as int8).
|
||||
k_scale: [total_kv] int32 packed UE8M0 K scales.
|
||||
weights: [rows, H] bf16 per-head weights; the sparse kernels take
|
||||
bf16 and do not fold the Q scale in.
|
||||
cu_seqlen_ks: [rows] int32 per-token K start bounds in the packed
|
||||
workspace.
|
||||
cu_seqlen_ke: [rows] int32 per-token K end bounds in the packed
|
||||
workspace.
|
||||
candidate_blocks: [rows, K] int32 request-local candidate block ids.
|
||||
candidate_block_size: Positions per candidate block.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
topk_tokens: Number of tokens to keep per row.
|
||||
topk_indices: [rows, topk_tokens] output buffer.
|
||||
sparse_indices: Caller-owned scratch, see
|
||||
`candidate_blocks_to_sparse_indices`.
|
||||
end: Caller-owned scratch, see `candidate_blocks_to_sparse_indices`.
|
||||
col_indices: Caller-owned scratch, see `sparse_topk_remap`.
|
||||
kernel_metadata: DeepGEMM schedule from a previous call with the same
|
||||
candidates and bounds (i.e. another indexer layer in the same
|
||||
step). When given, the candidate expansion is skipped and
|
||||
``sparse_indices``/``end`` are assumed to be up to date.
|
||||
|
||||
Returns:
|
||||
The DeepGEMM schedule metadata used, for reuse by later layers.
|
||||
|
||||
"""
|
||||
assert weights.dtype == torch.bfloat16, weights.dtype
|
||||
if kernel_metadata is None:
|
||||
candidate_blocks_to_sparse_indices(
|
||||
candidate_blocks,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
candidate_block_size,
|
||||
sparse_block_kv,
|
||||
out=(sparse_indices, end),
|
||||
)
|
||||
kernel_metadata = get_sparse_mqa_logits_metadata(
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
k_quant.shape[0],
|
||||
sparse_indices,
|
||||
q.dtype,
|
||||
sparse_block_kv,
|
||||
)
|
||||
logits = fp8_fp4_sparse_mqa_logits(
|
||||
(q, q_scale),
|
||||
(k_quant, k_scale),
|
||||
weights,
|
||||
kernel_metadata,
|
||||
sparse_indices.shape[1],
|
||||
sparse_block_kv,
|
||||
)
|
||||
sparse_topk_remap(
|
||||
logits,
|
||||
sparse_indices,
|
||||
end,
|
||||
cu_seqlen_ks,
|
||||
sparse_block_kv,
|
||||
topk_tokens,
|
||||
topk_indices,
|
||||
col_indices=col_indices,
|
||||
)
|
||||
return kernel_metadata
|
||||
|
||||
|
||||
def sparse_mqa_logits_paged_decode(
|
||||
q: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
context_lens: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
row_indices: torch.Tensor,
|
||||
candidate_blocks: torch.Tensor,
|
||||
candidate_block_size: int,
|
||||
sparse_block_kv: int,
|
||||
topk_tokens: int,
|
||||
topk_indices: torch.Tensor,
|
||||
*,
|
||||
row_ks: torch.Tensor,
|
||||
sparse_indices: torch.Tensor,
|
||||
end: torch.Tensor,
|
||||
col_indices: torch.Tensor,
|
||||
kernel_metadata: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Sparse-MQA logits + top-k for paged decode rows.
|
||||
|
||||
Every query is one row (spec-decode rows are flattened by the metadata
|
||||
builder), so ``block_table`` is already per row.
|
||||
|
||||
Args:
|
||||
q: [rows, 1, H, D] packed Q values (MXFP4 viewed as int8).
|
||||
q_scale: [rows, 1, H] int32 packed UE8M0 Q scales.
|
||||
kv_cache: 4D [num_pages, page_kv, 1, head_bytes] uint8 fused cache
|
||||
view; the page stride must be 256B-aligned.
|
||||
weights: [rows, H] bf16 per-head weights.
|
||||
context_lens: [rows] int32 per-row (compressed) context lengths.
|
||||
block_table: [rows, P] int32 per-row page table, ``stride(-1) == 1``.
|
||||
row_indices: [rows] int32 row -> request map (pairing only affects
|
||||
scheduling).
|
||||
candidate_blocks: [rows, K] int32 candidate block ids.
|
||||
candidate_block_size: Positions per candidate block.
|
||||
sparse_block_kv: Positions per sparse block (8 or 16).
|
||||
topk_tokens: Number of tokens to keep per row.
|
||||
topk_indices: [rows, topk_tokens] output buffer.
|
||||
row_ks: [rows] int32 zeros (paged blocks are context-relative).
|
||||
sparse_indices: Caller-owned scratch.
|
||||
end: Caller-owned scratch.
|
||||
col_indices: Caller-owned scratch.
|
||||
kernel_metadata: See `sparse_mqa_logits_prefill_chunk`.
|
||||
|
||||
Returns:
|
||||
The DeepGEMM schedule metadata used, for reuse by later layers.
|
||||
|
||||
"""
|
||||
rows = q.shape[0]
|
||||
page_kv = kv_cache.shape[1]
|
||||
assert weights.dtype == torch.bfloat16, weights.dtype
|
||||
assert kv_cache.stride(0) % 256 == 0, (
|
||||
"paged cache page stride must be 256B-aligned: "
|
||||
f"shape={tuple(kv_cache.shape)} stride={kv_cache.stride()}"
|
||||
)
|
||||
assert page_kv % sparse_block_kv == 0, (page_kv, sparse_block_kv)
|
||||
assert block_table.shape[0] == rows and block_table.stride(-1) == 1
|
||||
assert context_lens.shape == (rows,) and context_lens.is_contiguous()
|
||||
|
||||
if kernel_metadata is None:
|
||||
candidate_blocks_to_sparse_indices(
|
||||
candidate_blocks,
|
||||
row_ks,
|
||||
context_lens,
|
||||
candidate_block_size,
|
||||
sparse_block_kv,
|
||||
out=(sparse_indices, end),
|
||||
)
|
||||
kernel_metadata = get_paged_sparse_mqa_logits_metadata(
|
||||
context_lens,
|
||||
block_table,
|
||||
row_indices,
|
||||
page_kv,
|
||||
sparse_indices,
|
||||
q.dtype,
|
||||
sparse_block_kv,
|
||||
)
|
||||
logits = fp8_fp4_paged_sparse_mqa_logits(
|
||||
(q, q_scale),
|
||||
kv_cache,
|
||||
weights,
|
||||
kernel_metadata,
|
||||
sparse_indices.shape[1],
|
||||
sparse_block_kv,
|
||||
)
|
||||
sparse_topk_remap(
|
||||
logits,
|
||||
sparse_indices,
|
||||
end,
|
||||
row_ks,
|
||||
sparse_block_kv,
|
||||
topk_tokens,
|
||||
topk_indices,
|
||||
col_indices=col_indices,
|
||||
)
|
||||
return kernel_metadata
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,231 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Replicated input embedding + its fused gather/norm kernels.
|
||||
|
||||
Groups the ``VLLM_REPLICATE_EMBED`` path in one place: the embedding factory,
|
||||
the predicate that says whether the fusions apply, and the two Triton fusions
|
||||
the full on-rank table unlocks --
|
||||
|
||||
* ``fused_embed_norm``: gather + a chained RMSNorm (e.g. the first decoder
|
||||
layer's ``input_layernorm``), and
|
||||
* ``fused_embed_eh_norm``: gather + pos-0 zeroing + enorm/hnorm + cat, the
|
||||
embed/previous-hidden input norm for a speculative (MTP/eagle) depth layer
|
||||
(the replicated-table analogue of the model-local ``fused_eh_norm``, which
|
||||
takes precomputed embeds).
|
||||
|
||||
Self-contained (no model-local imports) so it can live under ``layers/``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _rms_norm(x, w, eps, HIDDEN_SIZE: tl.constexpr):
|
||||
x = x.to(tl.float32)
|
||||
mean_sq = tl.sum(x * x, axis=0) / HIDDEN_SIZE
|
||||
rrms = tl.rsqrt(mean_sq + eps)
|
||||
w = w.to(tl.float32)
|
||||
return (x * rrms) * w
|
||||
|
||||
|
||||
def make_input_embedding(
|
||||
num_embeddings: int,
|
||||
embedding_dim: int,
|
||||
*,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
quant_config=None,
|
||||
prefix: str = "",
|
||||
tie_word_embeddings: bool = False,
|
||||
) -> VocabParallelEmbedding:
|
||||
"""Input token embedding with an optional replicated escape hatch.
|
||||
|
||||
``VLLM_REPLICATE_EMBED=1`` builds the embedding with ``disable_tp``: the full
|
||||
table lives on every rank and the lookup is a local gather with no mask and
|
||||
no all-reduce, which unlocks the fused gather+norm path. The cost is a full
|
||||
table per rank at TP>1 (no extra memory at TP=1, where vocab-parallel is
|
||||
already unsharded). A replicated, unsharded table cannot be tied to a
|
||||
vocab-parallel ``ParallelLMHead``, so tied word embeddings are rejected at
|
||||
TP>1 (at TP=1 ``disable_tp`` is a no-op and tying still works).
|
||||
"""
|
||||
disable_tp = envs.VLLM_REPLICATE_EMBED
|
||||
if disable_tp and tie_word_embeddings:
|
||||
assert get_tensor_model_parallel_world_size() == 1, (
|
||||
"VLLM_REPLICATE_EMBED is unsupported with tied word embeddings "
|
||||
"(the replicated table cannot tie to a vocab-parallel lm_head)"
|
||||
)
|
||||
return VocabParallelEmbedding(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
disable_tp=disable_tp,
|
||||
)
|
||||
|
||||
|
||||
def has_full_vocab_on_rank(embedding: torch.nn.Module) -> bool:
|
||||
"""Whether ``embedding.weight`` is the whole vocab as a plain [V, H] table.
|
||||
|
||||
The fused gather kernels index the table directly, so they need every row
|
||||
on-rank (``disable_tp``, or any TP=1 run) and an unquantized weight.
|
||||
"""
|
||||
return getattr(embedding, "tp_size", 0) == 1 and isinstance(
|
||||
getattr(embedding, "quant_method", None), UnquantizedEmbeddingMethod
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_embed_norm_kernel(
|
||||
ids_ptr, # [T] token ids
|
||||
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
|
||||
table_stride_0,
|
||||
out_ptr, # [T, H] gathered embedding (the residual stream)
|
||||
normed_ptr, # [T, H] rmsnorm(out, chain_w) (HAS_NORM only)
|
||||
chain_w_ptr, # [H] next norm weight (HAS_NORM only)
|
||||
eps,
|
||||
H: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
HAS_NORM: tl.constexpr,
|
||||
):
|
||||
tok = tl.program_id(0).to(tl.int64)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
row = tl.load(ids_ptr + tok).to(tl.int64)
|
||||
x = tl.load(table_ptr + row * table_stride_0 + off, mask=mask, other=0.0)
|
||||
tl.store(out_ptr + tok * H + off, x, mask=mask)
|
||||
if HAS_NORM:
|
||||
w = tl.load(chain_w_ptr + off, mask=mask)
|
||||
y = _rms_norm(x, w, eps, H).to(normed_ptr.dtype.element_ty)
|
||||
tl.store(normed_ptr + tok * H + off, y, mask=mask)
|
||||
|
||||
|
||||
# Base model fusion
|
||||
def fused_embed_norm(
|
||||
input_ids: torch.Tensor,
|
||||
embed_table: torch.Tensor,
|
||||
chain_weight: torch.Tensor | None = None,
|
||||
eps: float = 0.0,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused embedding row gather (``embed_table[input_ids]``).
|
||||
|
||||
Requires the full vocab on-rank (replicated embedding). When
|
||||
``chain_weight`` is given, also emits ``rmsnorm(gathered, chain_weight)``
|
||||
(the first decoder layer's ``input_layernorm``) as a second output in the
|
||||
same launch, so the returned pair is ``(residual, normed_input)``. Bit-exact
|
||||
vs a plain gather followed by an ``RMSNorm``.
|
||||
"""
|
||||
assert embed_table.ndim == 2, embed_table.shape
|
||||
ids = input_ids.view(-1)
|
||||
(t,) = ids.shape
|
||||
h = embed_table.shape[1]
|
||||
if chain_weight is not None:
|
||||
assert chain_weight.shape == (h,), (chain_weight.shape, h)
|
||||
out = torch.empty((t, h), dtype=embed_table.dtype, device=embed_table.device)
|
||||
normed = torch.empty_like(out) if chain_weight is not None else None
|
||||
if t > 0:
|
||||
block = triton.next_power_of_2(h)
|
||||
_fused_embed_norm_kernel[(t,)](
|
||||
ids,
|
||||
embed_table,
|
||||
embed_table.stride(0),
|
||||
out,
|
||||
normed if normed is not None else out,
|
||||
chain_weight if chain_weight is not None else embed_table,
|
||||
eps,
|
||||
h,
|
||||
block,
|
||||
HAS_NORM=chain_weight is not None,
|
||||
num_warps=min(32, max(4, block // 512)),
|
||||
)
|
||||
if normed is not None:
|
||||
return out, normed
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_embed_eh_norm_kernel(
|
||||
pos_ptr,
|
||||
ids_ptr, # [T] token ids
|
||||
table_ptr, # [V, H] embedding table (full vocab, replicated on-rank)
|
||||
table_stride,
|
||||
prev_ptr, # [T, H] previous-step hidden
|
||||
prev_stride,
|
||||
enorm_w_ptr,
|
||||
hnorm_w_ptr,
|
||||
eps,
|
||||
out_ptr, # [T, 2H]
|
||||
out_stride,
|
||||
H: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""MTP input fusion with a folded embedding gather: gather
|
||||
``table[ids]``, zero it at position 0, RMSNorm(embed) with enorm and
|
||||
RMSNorm(prev_hidden) with hnorm, written side-by-side into ``out`` ([N, 2H])
|
||||
ready for the eh_proj GEMM. Replaces embedding lookup + where + 2x RMSNorm +
|
||||
cat. Requires the full table on-rank (replicated embedding)."""
|
||||
tok = tl.program_id(0)
|
||||
off = tl.arange(0, BLOCK)
|
||||
mask = off < H
|
||||
|
||||
pos = tl.load(pos_ptr + tok)
|
||||
row = tl.load(ids_ptr + tok).to(tl.int64)
|
||||
e = tl.load(table_ptr + row * table_stride + off, mask=mask, other=0.0)
|
||||
e = tl.where(pos == 0, 0.0, e.to(tl.float32))
|
||||
ew = tl.load(enorm_w_ptr + off, mask=mask)
|
||||
e_normed = _rms_norm(e, ew, eps, H)
|
||||
tl.store(out_ptr + tok * out_stride + off, e_normed, mask=mask)
|
||||
|
||||
p = tl.load(prev_ptr + tok * prev_stride + off, mask=mask, other=0.0)
|
||||
hw = tl.load(hnorm_w_ptr + off, mask=mask)
|
||||
p_normed = _rms_norm(p, hw, eps, H)
|
||||
tl.store(out_ptr + tok * out_stride + H + off, p_normed, mask=mask)
|
||||
|
||||
|
||||
# MTP fusion
|
||||
def fused_embed_eh_norm(
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
embed_table: torch.Tensor,
|
||||
previous_hidden: torch.Tensor,
|
||||
enorm_w: torch.Tensor,
|
||||
hnorm_w: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``cat([enorm(masked embed_table[ids]), hnorm(prev_hidden)])`` -> [N, 2H].
|
||||
|
||||
Folds the embedding row gather into the MTP eh-norm launch; requires the full
|
||||
table on-rank (replicated embedding). Bit-exact vs gathering ``embed_table[
|
||||
input_ids]`` and passing it to the model-local ``fused_eh_norm``.
|
||||
"""
|
||||
assert previous_hidden.ndim == 2 and embed_table.ndim == 2
|
||||
n, h = previous_hidden.shape
|
||||
assert positions.shape == (n,) and input_ids.view(-1).shape == (n,)
|
||||
assert embed_table.shape[1] == h, (embed_table.shape, h)
|
||||
assert enorm_w.shape == (h,) and hnorm_w.shape == (h,)
|
||||
out = torch.empty(
|
||||
n, 2 * h, dtype=previous_hidden.dtype, device=previous_hidden.device
|
||||
)
|
||||
_fused_embed_eh_norm_kernel[(n,)](
|
||||
positions,
|
||||
input_ids,
|
||||
embed_table,
|
||||
embed_table.stride(0),
|
||||
previous_hidden,
|
||||
previous_hidden.stride(0),
|
||||
enorm_w,
|
||||
hnorm_w,
|
||||
eps,
|
||||
out,
|
||||
out.stride(0),
|
||||
h,
|
||||
triton.next_power_of_2(h),
|
||||
)
|
||||
return out
|
||||
@@ -1,391 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused MoE Triton kernels."""
|
||||
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import get_device_name_as_file_name
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Valid tiles for the aiter a16w-mix int4 kernels (aiter#4646): stage1 requires
|
||||
# tile_n in {64,128} and tile_k in {128,256} (k_wave=1); stage2 requires tile_n2==128
|
||||
# and tile_k2 in {128,256}. The pre-#4646 defaults (tile_k=512/64, tile_n2=256) are no
|
||||
# longer registered and make the port emit an empty-fragment kernel.
|
||||
_FLYDSL_MOE_DEFAULT_CONFIG = {
|
||||
1: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
2: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
4: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
8: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
16: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
24: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
32: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
48: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
64: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
128: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
256: {"tile_m": 16, "tile_n": 128, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
512: {"tile_m": 16, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
1024: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
2048: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
4096: {"tile_m": 32, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
8192: {"tile_m": 64, "tile_n": 64, "tile_k": 128, "tile_n2": 128, "tile_k2": 128},
|
||||
}
|
||||
|
||||
# aiter#4646's a16w-mix int4 kernels only register a fixed tile set. Tiles from before
|
||||
# the bump (default config or tuned JSONs) carry tile_k=512/64 and tile_n2=256, which
|
||||
# emit an empty-fragment kernel (IndexError at emit / wrong output). Snap any resolved
|
||||
# tiles to the nearest registered value so a stale tuned config can't crash the kernel.
|
||||
_FLYDSL_INT4_TILE_M = (16, 32, 64, 128)
|
||||
_FLYDSL_INT4_TILE_N1 = (64, 128)
|
||||
_FLYDSL_INT4_TILE_K = (128, 256)
|
||||
|
||||
|
||||
def _snap_flydsl_int4_tiles(tile_m, tile_n, tile_k, tile_n2, tile_k2):
|
||||
def _nearest(v, allowed):
|
||||
return min(allowed, key=lambda a: (abs(a - v), a))
|
||||
|
||||
snapped = (
|
||||
_nearest(tile_m, _FLYDSL_INT4_TILE_M),
|
||||
_nearest(tile_n, _FLYDSL_INT4_TILE_N1),
|
||||
_nearest(tile_k, _FLYDSL_INT4_TILE_K),
|
||||
128, # stage2 N tile is only registered at 128
|
||||
_nearest(tile_k2, _FLYDSL_INT4_TILE_K),
|
||||
)
|
||||
if snapped != (tile_m, tile_n, tile_k, tile_n2, tile_k2):
|
||||
logger.warning_once(
|
||||
"FlyDSL int4 MoE tiles %s are not registered post-aiter#4646; "
|
||||
"using %s. Retune the config for the a16w-mix kernel.",
|
||||
(tile_m, tile_n, tile_k, tile_n2, tile_k2),
|
||||
snapped,
|
||||
scope="local",
|
||||
)
|
||||
return snapped
|
||||
|
||||
|
||||
def moe_sorting(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
*,
|
||||
num_experts: int,
|
||||
model_dim: int,
|
||||
block_m: int,
|
||||
):
|
||||
# Imported lazily so aiter's import-time sort-backend env vars are read
|
||||
# when this is called, not whenever this module happens to be imported.
|
||||
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
|
||||
|
||||
topk_ids_i32 = topk_ids.to(torch.int32)
|
||||
topk_w_f32 = topk_weights.to(torch.float32)
|
||||
sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = (
|
||||
aiter_moe_sorting(
|
||||
topk_ids_i32,
|
||||
topk_w_f32,
|
||||
num_experts,
|
||||
model_dim,
|
||||
torch.float16,
|
||||
block_m,
|
||||
)
|
||||
)
|
||||
if num_valid_ids.numel() > 1:
|
||||
num_valid_ids = num_valid_ids[:1].contiguous()
|
||||
return sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids
|
||||
|
||||
|
||||
def build_routing_buffers(
|
||||
*,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
num_experts: int,
|
||||
model_dim: int,
|
||||
tile_m: int,
|
||||
):
|
||||
res = moe_sorting(
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
num_experts=num_experts,
|
||||
model_dim=model_dim,
|
||||
block_m=tile_m,
|
||||
)
|
||||
if res is None:
|
||||
raise RuntimeError(
|
||||
"aiter moe_sorting failed/unavailable; cannot build routing buffers."
|
||||
)
|
||||
sorted_token_ids, sorted_weights, sorted_expert_ids, num_valid_ids = res
|
||||
|
||||
sorted_token_ids = sorted_token_ids.contiguous()
|
||||
sorted_weights = sorted_weights.contiguous()
|
||||
sorted_expert_ids = sorted_expert_ids.contiguous()
|
||||
sorted_size = int(sorted_token_ids.numel())
|
||||
blocks = int(sorted_expert_ids.numel())
|
||||
return (
|
||||
sorted_token_ids,
|
||||
sorted_weights,
|
||||
sorted_expert_ids,
|
||||
num_valid_ids,
|
||||
sorted_size,
|
||||
blocks,
|
||||
)
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def try_get_optimal_config(num_experts, inter_dim):
|
||||
device_name = get_device_name_as_file_name()
|
||||
json_file_name = (
|
||||
f"E={num_experts},N={inter_dim},device_name={device_name},"
|
||||
"dtype=int4_w4a16,backend=flydsl.json"
|
||||
)
|
||||
config_file_path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name
|
||||
)
|
||||
if os.path.exists(config_file_path):
|
||||
with open(config_file_path) as f:
|
||||
logger.info_once(
|
||||
"Using tuned FlyDSL MoE config from %s",
|
||||
config_file_path,
|
||||
scope="global",
|
||||
)
|
||||
tuned_config = json.load(f)
|
||||
return {int(key): val for key, val in tuned_config.items()}
|
||||
|
||||
logger.warning_once(
|
||||
"Using default FlyDSL MoE config. Performance might be sub-optimal! "
|
||||
"Config file not found at %s",
|
||||
config_file_path,
|
||||
scope="local",
|
||||
)
|
||||
return _FLYDSL_MOE_DEFAULT_CONFIG
|
||||
|
||||
|
||||
def fused_flydsl_moe_impl(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_experts: int,
|
||||
inter_dim: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
topk: int = 8,
|
||||
group_size: int = 32,
|
||||
doweight_stage1: bool = False,
|
||||
in_dtype: str = "int4_bf16",
|
||||
out_dtype: str = "bf16",
|
||||
scale_is_bf16: bool = True,
|
||||
tile_m: int | None = None,
|
||||
tile_n: int | None = None,
|
||||
tile_k: int | None = None,
|
||||
tile_n2: int | None = None,
|
||||
tile_k2: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Imported lazily -- see the comment in moe_sorting() above.
|
||||
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
|
||||
flydsl_a16w4_gemm1,
|
||||
flydsl_a16w4_gemm2,
|
||||
)
|
||||
|
||||
device = hidden_states.device
|
||||
tokens = hidden_states.shape[0]
|
||||
model_dim = hidden_states.shape[1]
|
||||
|
||||
tuned_config = {}
|
||||
if tile_m and tile_n and tile_k and tile_n2 and tile_k2:
|
||||
tuned_config["tile_m"] = tile_m
|
||||
tuned_config["tile_n"] = tile_n
|
||||
tuned_config["tile_k"] = tile_k
|
||||
tuned_config["tile_n2"] = tile_n2
|
||||
tuned_config["tile_k2"] = tile_k2
|
||||
else:
|
||||
tuned_config = try_get_optimal_config(num_experts, inter_dim)
|
||||
tuned_config = tuned_config[
|
||||
min(tuned_config.keys(), key=lambda x: abs(x - tokens))
|
||||
]
|
||||
out_torch_dtype = torch.bfloat16 if out_dtype == "bf16" else torch.float16
|
||||
|
||||
tile_m = tuned_config["tile_m"]
|
||||
tile_n = tuned_config["tile_n"]
|
||||
tile_k = tuned_config["tile_k"]
|
||||
tile_n2 = tuned_config["tile_n2"]
|
||||
tile_k2 = tuned_config["tile_k2"]
|
||||
tile_m, tile_n, tile_k, tile_n2, tile_k2 = _snap_flydsl_int4_tiles(
|
||||
tile_m, tile_n, tile_k, tile_n2, tile_k2
|
||||
)
|
||||
|
||||
routing = build_routing_buffers(
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
num_experts=num_experts,
|
||||
model_dim=model_dim,
|
||||
tile_m=tile_m,
|
||||
)
|
||||
(
|
||||
sorted_token_ids,
|
||||
sorted_weights,
|
||||
sorted_expert_ids,
|
||||
num_valid_ids,
|
||||
sorted_size,
|
||||
blocks,
|
||||
) = routing
|
||||
|
||||
# aiter#4646 (v0.1.20) removed compile_moe_gemm{1,2} + the flyc.compile pointer ABI
|
||||
# and replaced them with the a16w-mix wrappers, which compile+launch internally
|
||||
# (functools-cached) and take torch tensors directly. w_dtype="int4" is the a16wi4
|
||||
# (W4A16) path. The stage-1 intermediate now lives in sorted
|
||||
# [sorted_size, inter_dim] layout, and topk weighting is applied in stage-2 via
|
||||
# sorted_weights (the port has no stage-1 doweight).
|
||||
sorted_weights_1d = sorted_weights.view(-1).contiguous()
|
||||
empty_scale = torch.empty((0,), device=device, dtype=torch.uint8)
|
||||
cumsum_tensor = num_valid_ids.to(torch.int32).contiguous()
|
||||
m_indices = sorted_token_ids.to(torch.int32).contiguous()
|
||||
|
||||
# stage1: bf16 A x int4 W1 + SiLU -> bf16 sorted intermediate.
|
||||
inter_sorted = torch.empty(
|
||||
(sorted_size, inter_dim), device=device, dtype=torch.bfloat16
|
||||
)
|
||||
w1_scale_u8 = (
|
||||
w1_scale.view(torch.uint8).contiguous().view(-1)
|
||||
if w1_scale is not None
|
||||
else empty_scale
|
||||
)
|
||||
flydsl_a16w4_gemm1(
|
||||
a_bf16=hidden_states.to(torch.bfloat16).contiguous(),
|
||||
w1_u8=w1.view(torch.uint8).contiguous(),
|
||||
w1_scale_u8=w1_scale_u8,
|
||||
sorted_expert_ids=sorted_expert_ids,
|
||||
cumsum_tensor=cumsum_tensor,
|
||||
m_indices=m_indices,
|
||||
inter_sorted_bf16=inter_sorted,
|
||||
n_tokens=tokens,
|
||||
NE=num_experts,
|
||||
D_HIDDEN=model_dim,
|
||||
D_INTER=inter_dim,
|
||||
topk=topk,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n,
|
||||
tile_k=tile_k,
|
||||
waves_per_eu=None,
|
||||
act="silu",
|
||||
w_dtype="int4",
|
||||
w_layout="standard",
|
||||
)
|
||||
|
||||
# stage2: bf16 inter x int4 W2 -> weighted reduce into [tokens, model_dim].
|
||||
out_stage2 = torch.zeros((tokens, model_dim), device=device, dtype=out_torch_dtype)
|
||||
w2_scale_u8 = (
|
||||
w2_scale.view(torch.uint8).contiguous().view(-1)
|
||||
if w2_scale is not None
|
||||
else empty_scale
|
||||
)
|
||||
flydsl_a16w4_gemm2(
|
||||
inter_sorted_bf16=inter_sorted,
|
||||
w2_u8=w2.view(torch.uint8).contiguous(),
|
||||
w2_scale_u8=w2_scale_u8,
|
||||
sorted_expert_ids=sorted_expert_ids,
|
||||
cumsum_tensor=cumsum_tensor,
|
||||
sorted_token_ids=sorted_token_ids,
|
||||
sorted_weights=sorted_weights_1d,
|
||||
flat_out=out_stage2.view(-1),
|
||||
M_logical=tokens,
|
||||
max_sorted=sorted_size,
|
||||
NE=num_experts,
|
||||
D_HIDDEN=model_dim,
|
||||
D_INTER=inter_dim,
|
||||
topk=topk,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n2,
|
||||
tile_k=tile_k2,
|
||||
waves_per_eu=None,
|
||||
w_dtype="int4",
|
||||
)
|
||||
return out_stage2
|
||||
|
||||
|
||||
def fused_flydsl_moe_impl_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_experts: int,
|
||||
inter_dim: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
topk: int = 8,
|
||||
group_size: int = 32,
|
||||
doweight_stage1: bool = False,
|
||||
in_dtype: str = "int4_bf16",
|
||||
out_dtype: str = "bf16",
|
||||
scale_is_bf16: bool = True,
|
||||
tile_m: int | None = None,
|
||||
tile_n: int | None = None,
|
||||
tile_k: int | None = None,
|
||||
tile_n2: int | None = None,
|
||||
tile_k2: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_flydsl_moe_impl",
|
||||
op_func=fused_flydsl_moe_impl,
|
||||
fake_impl=fused_flydsl_moe_impl_fake,
|
||||
)
|
||||
|
||||
|
||||
def fused_flydsl_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_experts: int,
|
||||
inter_dim: int,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
w1_scale: torch.Tensor | None = None,
|
||||
w2_scale: torch.Tensor | None = None,
|
||||
topk: int = 8,
|
||||
group_size: int = 32,
|
||||
doweight_stage1: bool = False,
|
||||
in_dtype: str = "int4_bf16",
|
||||
out_dtype: str = "bf16",
|
||||
scale_is_bf16: bool = True,
|
||||
config: dict | None = None,
|
||||
) -> torch.Tensor:
|
||||
tile_m = None
|
||||
tile_n = None
|
||||
tile_k = None
|
||||
tile_n2 = None
|
||||
tile_k2 = None
|
||||
if config is not None:
|
||||
tile_m = config.get("tile_m")
|
||||
tile_n = config.get("tile_n")
|
||||
tile_k = config.get("tile_k")
|
||||
tile_n2 = config.get("tile_n2")
|
||||
tile_k2 = config.get("tile_k2")
|
||||
return torch.ops.vllm.fused_flydsl_moe_impl(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
num_experts=num_experts,
|
||||
inter_dim=inter_dim,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
topk=topk,
|
||||
group_size=group_size,
|
||||
doweight_stage1=doweight_stage1,
|
||||
in_dtype=in_dtype,
|
||||
out_dtype=out_dtype,
|
||||
scale_is_bf16=scale_is_bf16,
|
||||
tile_m=tile_m,
|
||||
tile_n=tile_n,
|
||||
tile_k=tile_k,
|
||||
tile_n2=tile_n2,
|
||||
tile_k2=tile_k2,
|
||||
)
|
||||
@@ -1,682 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton input-staging kernels for DeepGEMM MegaMoE.
|
||||
|
||||
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
|
||||
global scales, or copy them as BF16, and repack the routing top-k tensors into
|
||||
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
|
||||
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
|
||||
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _quantize_fp8_groups(
|
||||
hidden,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
):
|
||||
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
|
||||
amax = tl.max(hidden_groups, axis=2)
|
||||
amax = tl.maximum(amax, 1.0e-4)
|
||||
|
||||
scale = amax / 448.0
|
||||
if USE_UE8M0:
|
||||
scale_bits = scale.to(tl.uint32, bitcast=True)
|
||||
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
|
||||
tl.uint32
|
||||
)
|
||||
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
|
||||
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
|
||||
else:
|
||||
scale_exp = scale.to(tl.uint32, bitcast=True)
|
||||
quant_scale = scale
|
||||
|
||||
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
|
||||
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
|
||||
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
|
||||
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
|
||||
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
|
||||
offsets = tl.arange(0, NUM_BYTES)
|
||||
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
|
||||
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
|
||||
m_block_id = token_id // SHARED_BLOCK_M
|
||||
m_in_block = token_id % SHARED_BLOCK_M
|
||||
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
return m_block_id * aligned_block_m + transposed_m
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
) -> None:
|
||||
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
|
||||
topk_offsets = tl.arange(0, BLOCK_TOPK)
|
||||
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
|
||||
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
|
||||
if is_padding is not None:
|
||||
token_is_padding = tl.load(
|
||||
is_padding + token_id * is_padding_stride_m,
|
||||
mask=token_mask,
|
||||
other=True,
|
||||
)
|
||||
|
||||
ids = tl.load(
|
||||
topk_ids
|
||||
+ token_id[:, None] * topk_ids_stride_m
|
||||
+ topk_offsets[None, :] * topk_ids_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ids = tl.where(token_is_padding[:, None], -1, ids)
|
||||
tl.store(
|
||||
topk_idx_out
|
||||
+ token_id[:, None] * topk_idx_stride_m
|
||||
+ topk_offsets[None, :] * topk_idx_stride_k,
|
||||
ids,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
weights = tl.load(
|
||||
topk_weights
|
||||
+ token_id[:, None] * topk_weights_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
weights = tl.where(token_is_padding[:, None], 0.0, weights)
|
||||
tl.store(
|
||||
topk_weights_out
|
||||
+ token_id[:, None] * topk_weights_out_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_out_stride_k,
|
||||
weights,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens"])
|
||||
def _prepare_megamoe_inputs_kernel(
|
||||
hidden_states,
|
||||
x_out,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_k: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
x_sf_stride_k: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
QUANTIZE: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
) -> None:
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_block_id = tl.program_id(1)
|
||||
|
||||
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
|
||||
hidden = tl.load(
|
||||
hidden_states
|
||||
+ token_id[:, None] * hidden_stride_m
|
||||
+ k_offsets[None, :] * hidden_stride_k,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
if not QUANTIZE:
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
hidden.to(x_out.dtype.element_ty),
|
||||
mask=k_mask,
|
||||
)
|
||||
else:
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
|
||||
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
|
||||
)
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
fp8,
|
||||
mask=k_mask,
|
||||
)
|
||||
|
||||
if USE_UE8M0:
|
||||
packed_scale = _pack_bytes(scale_exp, num_groups)
|
||||
tl.store(
|
||||
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
sf_offset = tl.arange(0, num_groups)
|
||||
tl.store(
|
||||
x_sf
|
||||
+ token_id[:, None] * x_sf_stride_m
|
||||
+ k_block_id * x_sf_stride_k
|
||||
+ sf_offset[None, :],
|
||||
quant_scale,
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
|
||||
# in an MN-major layout whose row permutation depends on the MegaMoE
|
||||
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
|
||||
# is already resident, avoiding another kernel and temporary tensor.
|
||||
if USE_UE8M0 and shared_x_sf is not None:
|
||||
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ shared_row * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
|
||||
if k_block_id == 0:
|
||||
_store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m,
|
||||
topk_ids_stride_k,
|
||||
topk_weights_stride_m,
|
||||
topk_weights_stride_k,
|
||||
is_padding_stride_m,
|
||||
topk_idx_stride_m,
|
||||
topk_idx_stride_k,
|
||||
topk_weights_out_stride_m,
|
||||
topk_weights_out_stride_k,
|
||||
top_k,
|
||||
BLOCK_M,
|
||||
BLOCK_TOPK,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
|
||||
def _prepare_megamoe_nvfp4_inputs_kernel(
|
||||
hidden_states,
|
||||
x_out,
|
||||
x_sf,
|
||||
x_scales,
|
||||
shared_x,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
w2_global_scale,
|
||||
l2_alpha,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
shared_x_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
l2_activation_scale,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
num_local_experts: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
BLOCK_E: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
SHARED_MXFP8: tl.constexpr,
|
||||
) -> None:
|
||||
if l2_alpha is not None:
|
||||
experts = tl.arange(0, BLOCK_E)
|
||||
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
|
||||
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
|
||||
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
|
||||
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
|
||||
|
||||
amax = tl.zeros((BLOCK_M,), tl.float32)
|
||||
for k_block_id in range(hidden_size // BLOCK_K):
|
||||
hidden = tl.load(
|
||||
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
|
||||
mask=token_mask[:, None],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
|
||||
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
|
||||
tl.store(x_scales + token_id, x_scale, mask=token_mask)
|
||||
|
||||
for k_block_id in range(hidden_size // BLOCK_K):
|
||||
k_ids = k_block_id * BLOCK_K + k_offsets
|
||||
hidden = tl.load(
|
||||
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
|
||||
mask=token_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
|
||||
values = tl.reshape(
|
||||
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
|
||||
[BLOCK_M, BLOCK_K // 16, 16],
|
||||
)
|
||||
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
|
||||
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
|
||||
block_scale = block_scale.to(tl.float8e4nv)
|
||||
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
|
||||
magnitude = tl.abs(values)
|
||||
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
|
||||
code = (
|
||||
(magnitude > 0.25).to(tl.int32)
|
||||
+ (magnitude >= 0.75).to(tl.int32)
|
||||
+ (magnitude > 1.25).to(tl.int32)
|
||||
+ (magnitude >= 1.75).to(tl.int32)
|
||||
+ (magnitude > 2.5).to(tl.int32)
|
||||
+ (magnitude >= 3.5).to(tl.int32)
|
||||
+ (magnitude > 5.0).to(tl.int32)
|
||||
)
|
||||
code = tl.where((values < 0) & (code != 0), code | 8, code)
|
||||
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
|
||||
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
|
||||
tl.store(
|
||||
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
|
||||
(low | (high << 4)).to(tl.uint8),
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
|
||||
sf_bytes = tl.reshape(
|
||||
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
|
||||
)
|
||||
tl.store(
|
||||
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
|
||||
_pack_bytes(sf_bytes, 4),
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
if shared_x is not None:
|
||||
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
|
||||
if SHARED_MXFP8:
|
||||
fp8, _, scale_exp = _quantize_fp8_groups(
|
||||
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
|
||||
)
|
||||
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
_pack_bytes(scale_exp, BLOCK_K // 32),
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
|
||||
|
||||
_store_topk(
|
||||
token_id,
|
||||
token_mask,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
topk_ids_stride_m,
|
||||
topk_ids_stride_k,
|
||||
topk_weights_stride_m,
|
||||
topk_weights_stride_k,
|
||||
is_padding_stride_m,
|
||||
topk_idx_stride_m,
|
||||
topk_idx_stride_k,
|
||||
topk_weights_out_stride_m,
|
||||
topk_weights_out_stride_k,
|
||||
top_k,
|
||||
BLOCK_M,
|
||||
BLOCK_TOPK,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
|
||||
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
|
||||
scale = hidden_quant.scale
|
||||
group_shape = scale.group_shape
|
||||
if (
|
||||
hidden_quant.dtype != FP8_DTYPE
|
||||
or not hidden_quant.symmetric
|
||||
or hidden_quant.scale2 is not None
|
||||
or scale.static
|
||||
or not group_shape.is_per_group()
|
||||
or block_k % group_shape.col != 0
|
||||
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
|
||||
):
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires symmetric dynamic fp8 "
|
||||
f"per-group quantization with a group size dividing {block_k} and "
|
||||
f"E8M0 or fp32 scales, got {hidden_quant}."
|
||||
)
|
||||
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
|
||||
|
||||
|
||||
def prepare_megamoe_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
x_sf: torch.Tensor | None,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
hidden_quant: QuantKey | None = kMxfp8Dynamic,
|
||||
) -> None:
|
||||
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
|
||||
|
||||
Args:
|
||||
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x: Output buffer for the dispatched hidden states.
|
||||
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
|
||||
``hidden_quant`` is ``None``.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x_sf: Optional output buffer for shared-expert scale factors.
|
||||
Must be given together with ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
|
||||
four per int32 into ``x_sf``; fp32 scales are stored directly.
|
||||
``None`` copies the hidden states into ``x`` unquantized.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if num_tokens == 0:
|
||||
return
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires hidden_size to be a multiple of 128."
|
||||
)
|
||||
block_k = 128
|
||||
quantize = hidden_quant is not None
|
||||
if quantize != (x_sf is not None):
|
||||
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
|
||||
group_k, use_ue8m0 = (
|
||||
(block_k, False)
|
||||
if hidden_quant is None
|
||||
else _resolve_hidden_quant(hidden_quant, block_k)
|
||||
)
|
||||
top_k = topk_ids.shape[1]
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
if (shared_x_sf is None) != (shared_block_m is None):
|
||||
raise ValueError(
|
||||
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
|
||||
)
|
||||
if shared_x_sf is not None and not use_ue8m0:
|
||||
raise ValueError(
|
||||
"MegaMoE shared input staging currently requires "
|
||||
"UE8M0-packed hidden scales."
|
||||
)
|
||||
if shared_x_sf is not None:
|
||||
assert shared_block_m is not None
|
||||
if shared_block_m <= 0:
|
||||
raise ValueError("MegaMoE shared_block_m must be positive.")
|
||||
expected_sf_k = hidden_size // 128
|
||||
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf must have shape "
|
||||
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
|
||||
)
|
||||
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
|
||||
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
|
||||
if shared_x_sf.shape[0] < required_rows:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf has insufficient rows: requires "
|
||||
f"{required_rows}, got {shared_x_sf.shape[0]}."
|
||||
)
|
||||
|
||||
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
|
||||
block_topk = triton.next_power_of_2(top_k)
|
||||
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
|
||||
_prepare_megamoe_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
x_sf.stride(0) if x_sf is not None else 0,
|
||||
x_sf.stride(1) if x_sf is not None else 0,
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
padding_stride_m,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
top_k,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
GROUP_K=group_k,
|
||||
BLOCK_TOPK=block_topk,
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
QUANTIZE=quantize,
|
||||
USE_UE8M0=use_ue8m0,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
|
||||
def prepare_megamoe_nvfp4_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
x_sf: torch.Tensor,
|
||||
x_scales: torch.Tensor,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
w2_global_scale: torch.Tensor | None = None,
|
||||
l2_alpha: torch.Tensor | None = None,
|
||||
l2_activation_scale: float = 1.0,
|
||||
) -> None:
|
||||
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
|
||||
|
||||
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
|
||||
quantized to E2M1 with one E4M3 scale per 16 values.
|
||||
|
||||
Args:
|
||||
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
|
||||
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
|
||||
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
|
||||
or a BF16 copy of the hidden states.
|
||||
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
|
||||
TMA layout for ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
w2_global_scale: Per-local-expert L2 weight global scales.
|
||||
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
|
||||
dequantization scale of each local expert. Written even without
|
||||
local tokens, as other ranks' tokens still reach these experts.
|
||||
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires hidden_size to be a multiple of 128."
|
||||
)
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
|
||||
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
|
||||
shared_block_m is None
|
||||
):
|
||||
raise ValueError(
|
||||
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
|
||||
"for an fp8 shared_x."
|
||||
)
|
||||
if (w2_global_scale is None) != (l2_alpha is None):
|
||||
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
|
||||
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
|
||||
if tensor is not None and tensor.stride(-1) != 1:
|
||||
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
|
||||
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
top_k = topk_ids.shape[1]
|
||||
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
|
||||
# One program even without tokens, to write l2_alpha.
|
||||
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
|
||||
_prepare_megamoe_nvfp4_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x.view(torch.uint8),
|
||||
x_sf,
|
||||
x_scales,
|
||||
shared_x,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
w2_global_scale,
|
||||
l2_alpha,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x.stride(0),
|
||||
x_sf.stride(0),
|
||||
shared_x.stride(0) if shared_x is not None else 0,
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
is_padding.stride(0) if is_padding is not None else 0,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
l2_activation_scale,
|
||||
hidden_size,
|
||||
top_k,
|
||||
num_local_experts,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=128,
|
||||
BLOCK_TOPK=triton.next_power_of_2(top_k),
|
||||
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
SHARED_MXFP8=shared_mxfp8,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_FP8_MIN, _FP8_MAX = get_fp8_min_max()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _relu_squared_static_fp8_quant_kernel(
|
||||
x_ptr,
|
||||
scale_ptr,
|
||||
output_ptr,
|
||||
num_elements,
|
||||
FP8_MIN: tl.constexpr,
|
||||
FP8_MAX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < num_elements
|
||||
x = tl.load(x_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
|
||||
input_is_nan = x != x
|
||||
relu = tl.maximum(x, 0.0)
|
||||
# Preserve the standalone ReLU2 kernel's BF16 output boundary.
|
||||
activated = (relu * relu).to(tl.bfloat16).to(tl.float32)
|
||||
# Match O2's Inductor-compiled QuantFP8 reciprocal-then-multiply.
|
||||
quantized = activated * (1.0 / tl.load(scale_ptr))
|
||||
quantized = tl.maximum(tl.minimum(quantized, FP8_MAX), FP8_MIN)
|
||||
# Match the O2 native chain by preserving NaN through the FP8 conversion.
|
||||
quantized = tl.where(input_is_nan, x, quantized)
|
||||
tl.store(output_ptr + offsets, quantized, mask=mask)
|
||||
|
||||
|
||||
def relu_squared_static_fp8_quant(
|
||||
x: torch.Tensor, linear: LinearBase
|
||||
) -> QuantizedActivation:
|
||||
"""BF16 ReLU2 followed by static per-tensor FP8 quantization."""
|
||||
assert x.dtype == torch.bfloat16
|
||||
assert x.is_contiguous()
|
||||
scale = linear.input_scale
|
||||
assert scale.dtype == torch.float32
|
||||
assert x.device == scale.device
|
||||
assert scale.numel() == 1
|
||||
|
||||
output = torch.empty_like(x, dtype=current_platform.fp8_dtype())
|
||||
if x.numel() != 0:
|
||||
block_size = min(triton.next_power_of_2(x.shape[-1]), 2048)
|
||||
num_warps = min(max(block_size // 256, 1), 4)
|
||||
grid = lambda meta: (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
|
||||
_relu_squared_static_fp8_quant_kernel[grid](
|
||||
x,
|
||||
scale,
|
||||
output,
|
||||
x.numel(),
|
||||
FP8_MIN=_FP8_MIN,
|
||||
FP8_MAX=_FP8_MAX,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
return QuantizedActivation(
|
||||
data=output,
|
||||
scale=scale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=x.shape,
|
||||
quant_key=kFp8StaticTensorSym,
|
||||
)
|
||||
@@ -1,405 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Top-k kernels for the DSA sparse attention indexer."""
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
|
||||
# cooperative_topk's hard row limit: one cluster wave covers at most this many
|
||||
# rows. "auto" uses cooperative_topk for every batch within the limit and falls
|
||||
# back to persistent_topk past it; explicit ``cooperative`` requests are
|
||||
# validated against the same bound.
|
||||
AUTO_COOPERATIVE_MAX_ROWS = 64
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DeepSelect (vllm._deepselect_C)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Matches the -1 fill convention used for topk_indices_buffer elsewhere.
|
||||
IDX_OOB_FILL_VALUE = -1
|
||||
|
||||
try:
|
||||
import vllm._deepselect_C # noqa: F401 (registers torch.ops.deep_select)
|
||||
|
||||
_HAS_DEEP_SELECT_C = True
|
||||
except ImportError:
|
||||
_HAS_DEEP_SELECT_C = False
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_deep_select_stride_requirement() -> tuple[int, int]:
|
||||
"""Stride alignment requirement (input, output) in bytes."""
|
||||
return torch.ops.deep_select.get_alignment_requirement()
|
||||
|
||||
|
||||
def is_deep_select_supported(input: torch.Tensor, topk: int) -> bool:
|
||||
"""Whether the kernel accepts this input (dtype/stride/topk constraints)."""
|
||||
return (
|
||||
topk <= 4096
|
||||
and input.shape[1] < 2**23
|
||||
and input.dtype in (torch.float32, torch.bfloat16)
|
||||
and input.stride(1) == 1
|
||||
and input.stride(0)
|
||||
* input.element_size()
|
||||
% get_deep_select_stride_requirement()[0]
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
def _get_empty_and_aligned_tensor(
|
||||
dim0: int, dim1: int, device: torch.device, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""Tensor with shape (dim0, dim1) whose stride(0) is 32B-aligned."""
|
||||
output_stride_requirement = get_deep_select_stride_requirement()[1] // (
|
||||
dtype.itemsize
|
||||
)
|
||||
assert output_stride_requirement > 0
|
||||
dim1_rounded = (
|
||||
(dim1 + output_stride_requirement - 1) // output_stride_requirement
|
||||
) * output_stride_requirement
|
||||
return torch.empty((dim0, dim1_rounded), device=device, dtype=dtype)[:, :dim1]
|
||||
|
||||
|
||||
def deep_select_topk(
|
||||
input: torch.Tensor,
|
||||
topk: int,
|
||||
end: torch.Tensor | None = None,
|
||||
output_idx: torch.Tensor | None = None,
|
||||
indices_dtype: torch.dtype = torch.int32,
|
||||
) -> torch.Tensor:
|
||||
"""Select the top-k indices per row of `input` with DeepSelect.
|
||||
|
||||
Args:
|
||||
input: (num_rows, vocab_size), bf16 or fp32. stride(1) must be 1 and
|
||||
stride(0) must be 1024B-aligned.
|
||||
topk: Number of elements to select per row; must be <= 4096.
|
||||
end: Optional (num_rows,) int32 tensor with the exclusive right
|
||||
boundary of each row. Rows with `end[i] < topk` get their
|
||||
remaining indices filled with -1.
|
||||
output_idx: Optional preallocated (num_rows, topk) output tensor whose
|
||||
stride(0) is 32B-aligned (e.g. a slice of a wider buffer).
|
||||
indices_dtype: Output dtype when `output_idx` is not provided.
|
||||
|
||||
Returns:
|
||||
The (num_rows, topk) indices tensor.
|
||||
|
||||
"""
|
||||
assert input.dim() == 2 and input.stride(1) == 1
|
||||
assert (
|
||||
input.stride(0) * input.element_size() % get_deep_select_stride_requirement()[0]
|
||||
== 0
|
||||
)
|
||||
|
||||
num_rows = input.shape[0]
|
||||
if output_idx is None:
|
||||
output_idx = _get_empty_and_aligned_tensor(
|
||||
num_rows, topk, input.device, indices_dtype
|
||||
)
|
||||
else:
|
||||
assert output_idx.dtype == indices_dtype
|
||||
assert output_idx.shape[0] >= num_rows and output_idx.shape[1] >= topk
|
||||
|
||||
torch.ops.deep_select.topk(
|
||||
input,
|
||||
topk,
|
||||
None, # begin is not supported
|
||||
end,
|
||||
False, # sorted_value
|
||||
False, # sorted_index
|
||||
None, # output_value
|
||||
output_idx,
|
||||
None, # output_idx_offset
|
||||
IDX_OOB_FILL_VALUE,
|
||||
float("-inf"), # value_oob_fill_value
|
||||
False, # return_value
|
||||
False, # abort_when_nan_found: dummy/capture passes can feed NaN
|
||||
# logits from uninitialized KV cache; the reference topk kernels do
|
||||
# not trap on NaN, and trapping would abort CUDA graph capture.
|
||||
)
|
||||
return output_idx
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backend selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@functools.cache
|
||||
def get_indexer_topk(backend: str) -> "SparseIndexerTopk":
|
||||
return SparseIndexerTopk(backend)
|
||||
|
||||
|
||||
class SparseIndexerTopk(torch.nn.Module):
|
||||
"""The sparse indexer's decode top-k stage.
|
||||
|
||||
Selects among the available top-k kernels (see
|
||||
kernel_config.sparse_indexer_topk_backend) and runs the chosen one.
|
||||
"""
|
||||
|
||||
def __init__(self, backend: str | None = None) -> None:
|
||||
super().__init__()
|
||||
if backend is None:
|
||||
backend = (
|
||||
get_current_vllm_config().kernel_config.sparse_indexer_topk_backend
|
||||
)
|
||||
self._backend = backend
|
||||
self._is_cuda = current_platform.is_cuda()
|
||||
self._has_deep_select = self._is_cuda and (
|
||||
current_platform.is_device_capability_family(100)
|
||||
)
|
||||
self._has_flashinfer_topk = has_flashinfer()
|
||||
self._cooperative_capable = self._is_cuda and (
|
||||
current_platform.has_device_capability(90)
|
||||
and not current_platform.is_device_capability_family(120)
|
||||
)
|
||||
self._is_rocm = current_platform.is_rocm()
|
||||
self._aiter_enabled = False
|
||||
self._aiter_capable = False
|
||||
if self._is_rocm:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
|
||||
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
|
||||
|
||||
def resolve_backend(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""Resolve the decode top-k implementation from the configured
|
||||
backend ("auto" = the pre-existing chain, or a validated explicit
|
||||
value)."""
|
||||
if self._backend == "auto":
|
||||
return self._resolve_auto(logits, topk_tokens, num_rows)
|
||||
|
||||
failures: list[str] = []
|
||||
if self._backend == "cooperative":
|
||||
failures = self._cooperative_constraints(logits, topk_tokens, num_rows)
|
||||
elif self._backend == "persistent":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
if topk_tokens not in (512, 1024, 2048):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
|
||||
)
|
||||
elif self._backend == "deep_select":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
elif not self._has_deep_select:
|
||||
failures.append("requires SM100a/SM103a (10.x device family)")
|
||||
elif not _HAS_DEEP_SELECT_C:
|
||||
failures.append("vllm._deepselect_C could not be imported")
|
||||
elif not is_deep_select_supported(logits, topk_tokens):
|
||||
failures.append(
|
||||
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
|
||||
f" stride={logits.stride()}, topk={topk_tokens}"
|
||||
)
|
||||
elif self._backend == "aiter":
|
||||
if not self._is_rocm:
|
||||
failures.append("requires a ROCm platform")
|
||||
elif not self._aiter_enabled:
|
||||
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
|
||||
elif not self._aiter_capable:
|
||||
failures.append(
|
||||
"AITER has no tuned indexer top-k kernels for this GPU arch"
|
||||
)
|
||||
if topk_tokens not in (512, 1024, 2048, 4096):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
|
||||
)
|
||||
elif self._backend == "flashinfer":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
if not self._has_flashinfer_topk:
|
||||
failures.append(
|
||||
"flashinfer.topk.top_k_ragged_transform is not importable"
|
||||
)
|
||||
if logits.dtype != torch.float32 or logits.stride(1) != 1:
|
||||
failures.append(
|
||||
f"requires fp32 logits with stride(1) == 1, got "
|
||||
f"dtype={logits.dtype}, stride={logits.stride()}"
|
||||
)
|
||||
if failures:
|
||||
raise RuntimeError(
|
||||
f"sparse_indexer_topk_backend='{self._backend}' was requested, but: "
|
||||
+ "; ".join(failures)
|
||||
)
|
||||
return self._backend
|
||||
|
||||
def _resolve_auto(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The priority chain: cooperative -> persistent -> per_row.
|
||||
aiter/deep_select/flashinfer/torch are opt-in only.
|
||||
|
||||
cooperative_topk is preferred whenever it is applicable, i.e. within
|
||||
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
|
||||
persistent_topk.
|
||||
|
||||
aiter not auto-resolved as it does not outperform the per-row on every shape,
|
||||
hence we delegate to the caller to decide when to use it.
|
||||
"""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
if self._is_cuda and topk_tokens in (512, 1024, 2048):
|
||||
return "persistent"
|
||||
return "per_row"
|
||||
|
||||
def _cooperative_constraints(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> list[str]:
|
||||
"""Unmet constraints of cooperative_topk (empty when applicable)."""
|
||||
failures = []
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
if topk_tokens not in (512, 1024, 2048):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048), got {topk_tokens}"
|
||||
)
|
||||
if num_rows > AUTO_COOPERATIVE_MAX_ROWS:
|
||||
failures.append(
|
||||
f"num_rows must be <= {AUTO_COOPERATIVE_MAX_ROWS}, got {num_rows}"
|
||||
)
|
||||
if logits.stride(0) % 4 != 0:
|
||||
failures.append(
|
||||
f"logits.stride(0) must be divisible by 4, got {logits.stride(0)}"
|
||||
)
|
||||
if self._is_cuda and not self._cooperative_capable:
|
||||
failures.append("requires SM90+ and is not supported on the SM12x family")
|
||||
return failures
|
||||
|
||||
@staticmethod
|
||||
def _row_ends(seq_lens: torch.Tensor, next_n: int, num_rows: int) -> torch.Tensor:
|
||||
"""Per-row exclusive end offsets (int32, (num_rows,)) for top-k
|
||||
kernels that take ragged lengths (DeepSelect, FlashInfer, torch
|
||||
reference).
|
||||
|
||||
seq_lens is (B, next_n) per-row effective lens for native spec decode
|
||||
and (B, 1) otherwise, in which case per-row lens are derived the same
|
||||
way as the other decode top-k kernels.
|
||||
"""
|
||||
if seq_lens.numel() == num_rows:
|
||||
row_ends = seq_lens.reshape(-1)
|
||||
else:
|
||||
next_n_offsets = torch.arange(
|
||||
next_n, dtype=torch.int32, device=seq_lens.device
|
||||
)
|
||||
row_ends = (
|
||||
(seq_lens.reshape(-1, 1) - next_n + 1 + next_n_offsets)
|
||||
.clamp_(min=0)
|
||||
.reshape(-1)
|
||||
)
|
||||
assert row_ends.dtype == torch.int32
|
||||
return row_ends
|
||||
|
||||
def forward(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
deep_select_topk(logits, topk_tokens, end=row_ends, output_idx=topk_indices)
|
||||
elif backend == "cooperative":
|
||||
(topk_workspace,) = current_workspace_manager().get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
torch.ops._C.cooperative_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
max_seq_len,
|
||||
)
|
||||
elif backend == "persistent":
|
||||
(topk_workspace,) = current_workspace_manager().get_simultaneous(
|
||||
((RADIX_TOPK_WORKSPACE_SIZE,), torch.uint8),
|
||||
)
|
||||
# persistent_topk's last argument is the host-side "max seq_len
|
||||
# across rows" bound (it gates the sampled_topk path and clamps
|
||||
# per-row lengths), not the padded row width: logits is sized to
|
||||
# max_model_len, so passing its width would incorrectly enable the
|
||||
# long-row kernels on short-context batches.
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
max_seq_len,
|
||||
)
|
||||
elif backend == "flashinfer":
|
||||
# Deferred: importing flashinfer initializes CUDA at import time.
|
||||
from flashinfer.topk import top_k_ragged_transform
|
||||
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
offsets = torch.zeros(
|
||||
logits.shape[0], dtype=torch.int32, device=logits.device
|
||||
)
|
||||
# top_k_ragged_transform selects within [0, row_ends[i]) per
|
||||
# row and -1-fills past the row length.
|
||||
indices = top_k_ragged_transform(logits, offsets, row_ends, topk_tokens)
|
||||
topk_indices.copy_(indices)
|
||||
elif backend == "torch":
|
||||
# Debug reference: mask everything past each row's end, then topk.
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
cols = torch.arange(logits.shape[1], device=logits.device)
|
||||
masked = logits.masked_fill(
|
||||
cols.unsqueeze(0) >= row_ends.unsqueeze(1), float("-inf")
|
||||
)
|
||||
indices = masked.topk(topk_tokens, dim=-1).indices
|
||||
in_range = torch.arange(topk_tokens, device=logits.device).unsqueeze(
|
||||
0
|
||||
) < row_ends.unsqueeze(1)
|
||||
indices = torch.where(in_range, indices, -1)
|
||||
topk_indices.copy_(indices)
|
||||
elif backend == "aiter":
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# AITER's decode kernel only implements the 1D/next_n == 1 form,
|
||||
# which is equivalent once the ragged ends are expanded.
|
||||
rocm_aiter_ops.indexer_top_k_decode(
|
||||
logits,
|
||||
1,
|
||||
self._row_ends(seq_lens, next_n, logits.shape[0]),
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
)
|
||||
else:
|
||||
assert backend == "per_row", f"unknown topk backend: {backend}"
|
||||
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
|
||||
|
||||
@staticmethod
|
||||
def _per_row(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
ops.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
@@ -1,741 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_diag_kernel(
|
||||
Q,
|
||||
K,
|
||||
V,
|
||||
Out,
|
||||
S,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
CBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel computes the diagonal blocks of the attention matrix
|
||||
# Each diagonal block represents attention
|
||||
# where queries attend to keys in the same block
|
||||
off = tl.program_id(0)
|
||||
off_bh = off // NUM_BLOCK # batch-head index
|
||||
off_block = off % NUM_BLOCK # block index within the sequence
|
||||
off_cblock = tl.program_id(1) # sub-block index within a block
|
||||
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
# Calculate base offsets for the current batch and head
|
||||
qk_offset = off_bh * n * d
|
||||
v_offset = off_bh * n * e
|
||||
o_offset = off_bh * n * e
|
||||
|
||||
# Calculate offsets for the current block
|
||||
block_offset = off_block * BLOCK
|
||||
qk_block_offset = block_offset * d
|
||||
v_block_offset = block_offset * e
|
||||
o_block_offset = block_offset * e
|
||||
|
||||
# Calculate offsets for the current sub-block
|
||||
cblock_offset = off_cblock * CBLOCK
|
||||
q_cblock_offset = cblock_offset * d
|
||||
o_cblock_offset = cblock_offset * e
|
||||
|
||||
# Calculate pointers to the query, key, value, and output tensors
|
||||
Q_block_ptr = (
|
||||
Q
|
||||
+ qk_offset
|
||||
+ qk_block_offset
|
||||
+ q_cblock_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * d
|
||||
+ tl.arange(0, d)[None, :]
|
||||
)
|
||||
K_trans_block_ptr = (
|
||||
K
|
||||
+ qk_offset
|
||||
+ qk_block_offset
|
||||
+ tl.arange(0, CBLOCK)[None, :] * d
|
||||
+ tl.arange(0, d)[:, None]
|
||||
)
|
||||
V_block_ptr = (
|
||||
V
|
||||
+ v_offset
|
||||
+ v_block_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, e)[None, :]
|
||||
)
|
||||
O_block_ptr = (
|
||||
Out
|
||||
+ o_offset
|
||||
+ o_block_offset
|
||||
+ o_cblock_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, e)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay rate for the current head
|
||||
S_block_ptr = S + off_h
|
||||
s = tl.load(S_block_ptr)
|
||||
|
||||
i = off_cblock
|
||||
q_index = tl.arange(0, CBLOCK) + i * CBLOCK
|
||||
|
||||
# Load query values
|
||||
q = tl.load(Q_block_ptr, mask=block_offset + q_index[:, None] < n, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
|
||||
# Initialize output accumulator
|
||||
qkv = tl.zeros([CBLOCK, e], dtype=tl.float32)
|
||||
|
||||
# Process all sub-blocks up to and
|
||||
# including the current one (causal attention)
|
||||
for j in range(i + 1):
|
||||
kv_index = tl.arange(0, CBLOCK) + j * CBLOCK
|
||||
diff = q_index[:, None] - kv_index[None, :]
|
||||
s_index = s * diff
|
||||
# Apply causal mask: only attend to positions before the current one
|
||||
s_index = tl.where(diff >= 0, -s_index, float("-inf"))
|
||||
decay = tl.exp(s_index)
|
||||
|
||||
# Load key and value
|
||||
k_trans = tl.load(
|
||||
K_trans_block_ptr,
|
||||
mask=block_offset + kv_index[None, :] < n,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
v = tl.load(
|
||||
V_block_ptr,
|
||||
mask=block_offset + kv_index[:, None] < n,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
# Compute attention scores and apply decay
|
||||
qk = tl.dot(q, k_trans) * decay
|
||||
|
||||
# Compute weighted values and accumulate
|
||||
qkv += tl.dot(qk, v)
|
||||
|
||||
# Move to the next sub-block
|
||||
K_trans_block_ptr += CBLOCK * d
|
||||
V_block_ptr += CBLOCK * e
|
||||
|
||||
# Store the result
|
||||
tl.store(
|
||||
O_block_ptr,
|
||||
qkv.to(O_block_ptr.dtype.element_ty),
|
||||
mask=block_offset + q_index[:, None] < n,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kv_parallel(
|
||||
K,
|
||||
V,
|
||||
K_decay,
|
||||
KV,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
D_FBLOCK: tl.constexpr,
|
||||
E_FBLOCK: tl.constexpr,
|
||||
NUM_FBLOCK: tl.constexpr,
|
||||
CBLOCK: tl.constexpr,
|
||||
NUM_CBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel computes the key-value outer
|
||||
# products for each block in parallel
|
||||
off_bh = tl.program_id(0) # batch-head index
|
||||
off_block = tl.program_id(1) # block index
|
||||
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
block_offset = off_block * BLOCK
|
||||
|
||||
# Calculate offsets for the current block
|
||||
k_block_offset = block_offset * d
|
||||
v_block_offset = block_offset * e
|
||||
kv_block_offset = off_block * d * e
|
||||
|
||||
# Calculate base offsets for the current batch and head
|
||||
k_offset = off_bh * n * d
|
||||
v_offset = off_bh * n * e
|
||||
kv_offset = off_bh * NUM_BLOCK * d * e
|
||||
|
||||
# Calculate pointers to the key, value, and key-value tensors
|
||||
K_trans_block_ptr = (
|
||||
K
|
||||
+ k_offset
|
||||
+ k_block_offset
|
||||
+ tl.arange(0, CBLOCK)[None, :] * d
|
||||
+ tl.arange(0, D_FBLOCK)[:, None]
|
||||
)
|
||||
V_block_ptr = (
|
||||
V
|
||||
+ v_offset
|
||||
+ v_block_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
KV_block_ptr = (
|
||||
KV
|
||||
+ kv_offset
|
||||
+ kv_block_offset
|
||||
+ tl.arange(0, D_FBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay factors for the current head and block
|
||||
k_decay_ptr = K_decay + off_h * BLOCK + tl.arange(0, CBLOCK)
|
||||
|
||||
kv_index = tl.arange(0, CBLOCK)
|
||||
|
||||
# Initialize the key-value outer product accumulator
|
||||
kv = tl.zeros([D_FBLOCK, E_FBLOCK], dtype=tl.float32)
|
||||
|
||||
# Handle the last block which might be smaller than BLOCK
|
||||
split_n = n - (NUM_BLOCK - 1) * BLOCK if off_block == NUM_BLOCK - 1 else BLOCK
|
||||
left_shift = tl.cdiv(split_n, CBLOCK) * CBLOCK - split_n
|
||||
num_blocks = min(tl.cdiv(split_n, CBLOCK), NUM_CBLOCK)
|
||||
k_decay_ptr += (NUM_CBLOCK - num_blocks) * CBLOCK
|
||||
|
||||
# Process all sub-blocks in the current block
|
||||
for j in range(num_blocks):
|
||||
left_bound = (1 - j) * left_shift
|
||||
# Load key and value, handling boundary conditions
|
||||
k_trans = tl.load(
|
||||
K_trans_block_ptr - left_shift * d,
|
||||
mask=kv_index[None, :] >= left_bound,
|
||||
other=0.0,
|
||||
)
|
||||
v = tl.load(
|
||||
V_block_ptr - left_shift * e,
|
||||
mask=kv_index[:, None] >= left_bound,
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Load decay factor and compute weighted key-value outer product
|
||||
k_decay = tl.load(k_decay_ptr)
|
||||
|
||||
# NOTE: Need to add the extra dim here due to AMD MLIR lowering error.
|
||||
# Please don't move it back until issue is resolved.
|
||||
# Issue: https://github.com/ROCm/triton/issues/907
|
||||
k_decay = k_decay[None, :]
|
||||
|
||||
kv += tl.dot(k_trans * k_decay, v)
|
||||
|
||||
# Move to the next sub-block
|
||||
K_trans_block_ptr += CBLOCK * d
|
||||
V_block_ptr += CBLOCK * e
|
||||
k_decay_ptr += CBLOCK
|
||||
|
||||
# Store the result
|
||||
tl.store(KV_block_ptr, kv.to(KV_block_ptr.dtype.element_ty))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kv_reduce(
|
||||
S,
|
||||
KV,
|
||||
KV_HISTORY,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
D_FBLOCK: tl.constexpr,
|
||||
E_FBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel reduces the key-value outer products
|
||||
# across blocks and updates the KV history
|
||||
off_bh = tl.program_id(0) # batch-head index
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
kv_offset = off_bh * NUM_BLOCK * d * e
|
||||
|
||||
# Calculate pointer to the key-value tensor
|
||||
KV_block_ptr = (
|
||||
KV
|
||||
+ kv_offset
|
||||
+ tl.arange(0, D_FBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay rate for the current head
|
||||
s_ptrs = S + off_h
|
||||
s = tl.load(s_ptrs)
|
||||
|
||||
# Calculate pointer to the key-value history tensor
|
||||
kv_history_offset = off_bh * d * e
|
||||
KV_HISTORY_block_ptr = (
|
||||
KV_HISTORY
|
||||
+ kv_history_offset
|
||||
+ tl.arange(0, D_FBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the previous key-value history
|
||||
kv_pre = tl.load(KV_HISTORY_block_ptr).to(tl.float32)
|
||||
|
||||
# Process all blocks in reverse order to compute the prefix sum
|
||||
for i in range(NUM_BLOCK):
|
||||
block_size = min(n - i * BLOCK, BLOCK)
|
||||
# Compute decay factor for the current block
|
||||
block_decay = tl.exp(-s.to(tl.float32) * block_size)
|
||||
|
||||
# Load the current key-value outer product
|
||||
kv_cur = tl.load(KV_block_ptr).to(tl.float32)
|
||||
# Store the previous key-value history to the current block
|
||||
tl.store(KV_block_ptr, kv_pre.to(KV_block_ptr.dtype.element_ty))
|
||||
|
||||
# Update the key-value history with the current block
|
||||
kv_pre = block_decay * kv_pre + kv_cur
|
||||
KV_block_ptr += d * e
|
||||
|
||||
# Store the updated key-value history
|
||||
tl.store(KV_HISTORY_block_ptr, kv_pre)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_none_diag_kernel(
|
||||
Q,
|
||||
Out,
|
||||
S,
|
||||
KV,
|
||||
b: tl.constexpr,
|
||||
h: tl.constexpr,
|
||||
n,
|
||||
d: tl.constexpr,
|
||||
e: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
NUM_BLOCK,
|
||||
E_FBLOCK: tl.constexpr,
|
||||
CBLOCK: tl.constexpr,
|
||||
NUM_CBLOCK: tl.constexpr,
|
||||
):
|
||||
# This kernel computes the non-diagonal blocks of the attention matrix
|
||||
# Each non-diagonal block represents attention
|
||||
# where queries attend to keys in different blocks
|
||||
off_bh = tl.program_id(0) # batch-head index
|
||||
off_h = off_bh % h # head index
|
||||
|
||||
off_nc = tl.program_id(1)
|
||||
off_n = off_nc // NUM_CBLOCK # block index
|
||||
off_c = off_nc % NUM_CBLOCK # sub-block index
|
||||
off_e = tl.program_id(2) # output feature block index
|
||||
|
||||
n_offset = off_n * BLOCK
|
||||
c_offset = off_c * CBLOCK
|
||||
e_offset = off_e * E_FBLOCK
|
||||
block_offset = n_offset + c_offset
|
||||
|
||||
# Calculate offsets for the current batch, head, and block
|
||||
q_offset = off_bh * n * d + (n_offset + c_offset) * d
|
||||
o_offset = off_bh * n * e + (n_offset + c_offset) * e + e_offset
|
||||
kv_offset = off_bh * NUM_BLOCK * d * e + off_n * d * e + e_offset
|
||||
|
||||
# Calculate pointers to the query, output, and key-value tensors
|
||||
Q_block_ptr = (
|
||||
Q + q_offset + tl.arange(0, CBLOCK)[:, None] * d + tl.arange(0, d)[None, :]
|
||||
)
|
||||
O_block_ptr = (
|
||||
Out
|
||||
+ o_offset
|
||||
+ tl.arange(0, CBLOCK)[:, None] * e
|
||||
+ tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
KV_block_ptr = (
|
||||
KV + kv_offset + tl.arange(0, d)[:, None] * e + tl.arange(0, E_FBLOCK)[None, :]
|
||||
)
|
||||
|
||||
# Load the decay rate for the current head
|
||||
S_block_ptr = S + off_h
|
||||
s = tl.load(S_block_ptr)
|
||||
|
||||
c_array = tl.arange(0, CBLOCK)
|
||||
|
||||
# Load the key-value outer product for the current block
|
||||
kv = tl.load(KV_block_ptr).to(tl.float32)
|
||||
q_index = block_offset + tl.arange(0, CBLOCK)
|
||||
|
||||
# Load query values
|
||||
q = tl.load(Q_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
|
||||
|
||||
# Compute decay factors for the current sub-block
|
||||
q_decay = tl.exp(-s.to(tl.float32) * (off_c * CBLOCK + c_array[:, None]))
|
||||
|
||||
# Compute non-diagonal attention output
|
||||
qkv_none_diag = tl.dot(q, kv) * q_decay
|
||||
|
||||
# Load diagonal attention output (computed by _fwd_diag_kernel)
|
||||
qkv_diag = tl.load(O_block_ptr, mask=q_index[:, None] < n, other=0.0).to(tl.float32)
|
||||
|
||||
# Combine diagonal and non-diagonal attention outputs
|
||||
qkv = qkv_diag + qkv_none_diag
|
||||
|
||||
# Store the result
|
||||
tl.store(
|
||||
O_block_ptr, qkv.to(O_block_ptr.dtype.element_ty), mask=q_index[:, None] < n
|
||||
)
|
||||
|
||||
|
||||
class _attention(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(ctx, q, k, v, s, kv_history):
|
||||
# Forward pass of the lightning attention algorithm
|
||||
q = q.contiguous()
|
||||
k = k.contiguous()
|
||||
v = v.contiguous()
|
||||
s = s.contiguous()
|
||||
|
||||
# Check CUDA compute capability (Ampere+ required for flash attention
|
||||
# path). Other accelerators (ROCm, XPU) rely on their own Triton
|
||||
# backend support and skip this check.
|
||||
if current_platform.is_cuda():
|
||||
capability = torch.cuda.get_device_capability()
|
||||
if capability[0] < 8:
|
||||
raise RuntimeError(
|
||||
"Flash attention currently only supported",
|
||||
"for compute capability >= 80",
|
||||
)
|
||||
|
||||
# Get input dimensions
|
||||
b, h, n, d = q.shape
|
||||
e = v.shape[-1]
|
||||
|
||||
# Initialize output tensor
|
||||
o = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device)
|
||||
|
||||
# Set block sizes
|
||||
BLOCK = 256
|
||||
NUM_BLOCK = triton.cdiv(n, BLOCK)
|
||||
|
||||
CBLOCK = 32
|
||||
NUM_CBLOCK = BLOCK // CBLOCK
|
||||
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
|
||||
|
||||
# Compute decay factors for keys
|
||||
array = torch.arange(0, BLOCK, device=q.device) + 1
|
||||
k_decay = torch.exp(-s * (BLOCK - array.reshape(1, -1)))
|
||||
|
||||
# Step 1: Compute diagonal blocks of attention
|
||||
grid = (b * h * NUM_BLOCK, NUM_CBLOCK)
|
||||
_fwd_diag_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
o,
|
||||
s,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
CBLOCK=CBLOCK,
|
||||
)
|
||||
|
||||
# Set feature block sizes
|
||||
NUM_FBLOCK = 1
|
||||
D_FBLOCK = d // NUM_FBLOCK
|
||||
assert d % NUM_FBLOCK == 0
|
||||
E_FBLOCK = e // NUM_FBLOCK
|
||||
assert e % NUM_FBLOCK == 0
|
||||
|
||||
CBLOCK = 64
|
||||
NUM_CBLOCK = BLOCK // CBLOCK
|
||||
assert BLOCK % CBLOCK == 0, "BLOCK must be a multiple of CBLOCK"
|
||||
|
||||
# Step 2: Compute key-value outer products for each block in parallel
|
||||
kv = torch.empty((b, h, NUM_BLOCK, d, e), dtype=torch.float32, device=q.device)
|
||||
grid = (b * h, NUM_BLOCK)
|
||||
_fwd_kv_parallel[grid](
|
||||
k,
|
||||
v,
|
||||
k_decay,
|
||||
kv,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
D_FBLOCK=D_FBLOCK,
|
||||
E_FBLOCK=E_FBLOCK,
|
||||
NUM_FBLOCK=NUM_FBLOCK,
|
||||
CBLOCK=CBLOCK,
|
||||
NUM_CBLOCK=NUM_CBLOCK,
|
||||
)
|
||||
|
||||
# Step 3: Reduce key-value outer products
|
||||
# across blocks and update KV history
|
||||
grid = (b * h, NUM_FBLOCK)
|
||||
_fwd_kv_reduce[grid](
|
||||
s,
|
||||
kv,
|
||||
kv_history,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
D_FBLOCK=D_FBLOCK,
|
||||
E_FBLOCK=E_FBLOCK,
|
||||
)
|
||||
|
||||
# Step 4: Compute non-diagonal blocks of attention
|
||||
grid = (b * h, NUM_BLOCK * NUM_CBLOCK)
|
||||
_fwd_none_diag_kernel[grid](
|
||||
q,
|
||||
o,
|
||||
s,
|
||||
kv,
|
||||
b,
|
||||
h,
|
||||
n,
|
||||
d,
|
||||
e,
|
||||
BLOCK=BLOCK,
|
||||
NUM_BLOCK=NUM_BLOCK,
|
||||
E_FBLOCK=E_FBLOCK,
|
||||
CBLOCK=CBLOCK,
|
||||
NUM_CBLOCK=NUM_CBLOCK,
|
||||
)
|
||||
|
||||
# Save tensors for backward pass
|
||||
ctx.save_for_backward(q, k, v, s, kv)
|
||||
ctx.BLOCK = BLOCK
|
||||
|
||||
return o, torch.cat([kv, kv_history.unsqueeze(2)], dim=2)
|
||||
|
||||
|
||||
# Apply the lightning attention function
|
||||
lightning_attention_ = _attention.apply
|
||||
|
||||
|
||||
def lightning_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
ed: torch.Tensor,
|
||||
block_size: int = 256,
|
||||
kv_history: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Apply lightning attention algorithm
|
||||
to compute attention efficiently.
|
||||
|
||||
Args:
|
||||
q: Query tensor of shape [batch, heads, seq_len, dim]
|
||||
k: Key tensor of shape [batch, heads, seq_len, dim]
|
||||
v: Value tensor of shape [batch, heads, seq_len, dim_v]
|
||||
ed: Decay rate tensor of shape [heads]
|
||||
block_size: Size of blocks for block-sparse attention
|
||||
kv_history: Optional key-value history from previous computations
|
||||
|
||||
Returns:
|
||||
output: Attention output
|
||||
kv: Updated key-value history
|
||||
|
||||
"""
|
||||
d = q.shape[-1]
|
||||
e = v.shape[-1]
|
||||
|
||||
if ed.dim() == 1:
|
||||
ed = ed.view(1, -1, 1, 1)
|
||||
|
||||
# Split the computation into chunks for better parallelism
|
||||
m = 128 if d >= 128 else 64
|
||||
assert d % m == 0, f"Dimension d ({d}) must be divisible by m ({m})"
|
||||
arr = [m * i for i in range(d // m + 1)]
|
||||
if arr[-1] != d:
|
||||
arr.append(d)
|
||||
n = len(arr)
|
||||
output = 0
|
||||
|
||||
# Initialize or clone key-value history
|
||||
if kv_history is None:
|
||||
kv_history = torch.zeros(
|
||||
(q.shape[0], q.shape[1], d, e), dtype=torch.float32, device=q.device
|
||||
)
|
||||
else:
|
||||
kv_history = kv_history.clone().contiguous()
|
||||
|
||||
# Process each chunk and accumulate results
|
||||
for i in range(n - 1):
|
||||
s = arr[i]
|
||||
e = arr[i + 1]
|
||||
q1 = q[..., s:e]
|
||||
k1 = k[..., s:e]
|
||||
o, kv = lightning_attention_(q1, k1, v, ed, kv_history)
|
||||
output = output + o
|
||||
return output, kv
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _linear_attn_decode_kernel(
|
||||
q_ptr,
|
||||
k_ptr,
|
||||
v_ptr,
|
||||
kv_cache_ptr,
|
||||
slope_rate,
|
||||
slot_idx,
|
||||
output_ptr,
|
||||
D: tl.constexpr,
|
||||
qkv_b_stride,
|
||||
qkv_h_stride,
|
||||
cache_b_stride,
|
||||
cache_h_stride,
|
||||
cache_d0_stride,
|
||||
cache_d1_stride,
|
||||
pad_slot_id: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Kernel for linear attention decoding with KV cache.
|
||||
|
||||
This kernel computes attention for a single token using the KV cache.
|
||||
"""
|
||||
pid_b = tl.program_id(0) # batch index
|
||||
pid_h = tl.program_id(1) # head index
|
||||
pid_d = tl.program_id(2) # dimension block index
|
||||
|
||||
# Load slot index for the current batch
|
||||
slot_id = tl.load(slot_idx + pid_b).to(tl.int64)
|
||||
|
||||
# Skip if slot_id is PAD_SLOT_ID (padding)
|
||||
if slot_id == pad_slot_id:
|
||||
return
|
||||
|
||||
batch_id = pid_b
|
||||
head_id = pid_h
|
||||
|
||||
# Load decay rate for the current head
|
||||
ratio = tl.load(slope_rate + pid_h)
|
||||
|
||||
# Calculate offsets for dimensions
|
||||
qk_d_offsets = tl.arange(0, D)
|
||||
v_d_offsets = tl.arange(0, BLOCK_SIZE) + pid_d * BLOCK_SIZE
|
||||
cache_d_offsets = (
|
||||
qk_d_offsets[:, None] * cache_d0_stride + v_d_offsets[None, :] * cache_d1_stride
|
||||
)
|
||||
|
||||
# Calculate offsets for the current batch and head
|
||||
q_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
|
||||
k_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
|
||||
v_offset = batch_id * qkv_b_stride + head_id * qkv_h_stride
|
||||
|
||||
cache_offset = slot_id * cache_b_stride + head_id * cache_h_stride
|
||||
|
||||
# Create masks for loading tensors
|
||||
qk_mask = qk_d_offsets < D
|
||||
v_mask = v_d_offsets < D
|
||||
|
||||
# Load query, key, and value tensors
|
||||
q = tl.load(q_ptr + q_offset + qk_d_offsets, mask=qk_mask, other=0.0)
|
||||
k = tl.load(k_ptr + k_offset + qk_d_offsets, mask=qk_mask, other=0.0)
|
||||
v = tl.load(v_ptr + v_offset + v_d_offsets, mask=v_mask, other=0.0)
|
||||
|
||||
# Compute key-value outer product
|
||||
kv_outer = k[:, None] * v[None, :]
|
||||
kv_mask = qk_mask[:, None] & v_mask[None, :]
|
||||
|
||||
# Apply decay to previous KV cache
|
||||
ratio = tl.exp(-ratio)
|
||||
kv_ptr = kv_cache_ptr + cache_offset + cache_d_offsets
|
||||
kv_cache_old = tl.load(kv_ptr, mask=kv_mask, other=0.0)
|
||||
kv_outer = kv_outer + ratio * kv_cache_old
|
||||
|
||||
# Compute attention output
|
||||
output = q[:, None].to(tl.float32) * kv_outer
|
||||
output = tl.sum(output, axis=0)
|
||||
|
||||
# Update KV cache and store output
|
||||
tl.store(kv_ptr, kv_outer, mask=kv_mask)
|
||||
tl.store(output_ptr + q_offset + v_d_offsets, output, mask=v_mask)
|
||||
|
||||
|
||||
def linear_decode_forward_triton(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
kv_caches: torch.Tensor,
|
||||
slope_rate: torch.Tensor,
|
||||
slot_idx: torch.Tensor,
|
||||
BLOCK_SIZE: int = 32,
|
||||
) -> torch.Tensor:
|
||||
"""Perform linear attention decoding using Triton kernels.
|
||||
|
||||
Args:
|
||||
q: Query tensor of shape [B, H, 1, D]
|
||||
k: Key tensor of shape [B, H, 1, D]
|
||||
v: Value tensor of shape [B, H, 1, D]
|
||||
kv_caches: Key-value cache tensor
|
||||
slope_rate: Decay rate tensor
|
||||
slot_idx: Slot indices for batches
|
||||
BLOCK_SIZE: Size of blocks for processing
|
||||
|
||||
Returns:
|
||||
output: Attention output tensor
|
||||
|
||||
"""
|
||||
B, H, _, D = q.shape
|
||||
assert k.shape == (B, H, 1, D)
|
||||
assert v.shape == (B, H, 1, D)
|
||||
|
||||
# Initialize output tensor
|
||||
output = torch.empty_like(q)
|
||||
|
||||
# Set grid dimensions for the kernel
|
||||
grid = (B, H, D // BLOCK_SIZE)
|
||||
|
||||
# Calculate strides for tensors
|
||||
qkv_b_stride = q.stride(0)
|
||||
qkv_h_stride = q.stride(1)
|
||||
|
||||
cache_b_stride = kv_caches.stride(0)
|
||||
cache_h_stride = kv_caches.stride(1)
|
||||
cache_d0_stride = kv_caches.stride(2)
|
||||
cache_d1_stride = kv_caches.stride(3)
|
||||
|
||||
# Launch the kernel
|
||||
_linear_attn_decode_kernel[grid](
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
kv_caches,
|
||||
slope_rate,
|
||||
slot_idx,
|
||||
output,
|
||||
D,
|
||||
qkv_b_stride,
|
||||
qkv_h_stride,
|
||||
cache_b_stride,
|
||||
cache_h_stride,
|
||||
cache_d0_stride,
|
||||
cache_d1_stride,
|
||||
pad_slot_id=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
# Reshape output and return
|
||||
output = rearrange(output, "b h n d -> b n (h d)")
|
||||
return output.squeeze(1).contiguous()
|
||||
@@ -1,382 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateDtypeCalculator,
|
||||
MambaStateShapeCalculator,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
|
||||
|
||||
|
||||
# --8<-- [start:short_conv]
|
||||
@PluggableLayer.register("short_conv")
|
||||
class ShortConv(MambaBase, PluggableLayer):
|
||||
# --8<-- [end:short_conv]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
dim: int,
|
||||
layer_idx: int,
|
||||
model_config: ModelConfig | None = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layer_idx = layer_idx
|
||||
self.conv_dim = dim
|
||||
self.L_cache = config.conv_L_cache
|
||||
self.bias = config.conv_bias
|
||||
|
||||
self.conv = ColumnParallelLinear(
|
||||
input_size=self.L_cache,
|
||||
output_size=dim,
|
||||
bias=self.bias,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
# `ColumnParallelLinear` and `set_weight_attrs`
|
||||
# doesn't allow to override it
|
||||
self.conv.weight.data = self.conv.weight.data.unsqueeze(1)
|
||||
|
||||
self.in_proj = MergedColumnParallelLinear(
|
||||
input_size=dim,
|
||||
output_sizes=[dim] * 3,
|
||||
bias=self.bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj",
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
input_size=dim,
|
||||
output_size=dim,
|
||||
bias=self.bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.num_spec = vllm_config.num_speculative_tokens
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
self.kv_cache = (torch.tensor([]),)
|
||||
|
||||
self.model_config = model_config
|
||||
self.cache_config = cache_config
|
||||
self.prefix = prefix
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
# Reference torch causal conv1d; runs on all CPU platforms. AMX kernels
|
||||
# for causal conv can be plugged in here later.
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_fn_cpu as causal_conv1d_torch,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_cpu,
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
assert attn_metadata is None or isinstance(
|
||||
attn_metadata, ShortConvAttentionMetadata
|
||||
)
|
||||
|
||||
BCx, _ = self.in_proj(hidden_states)
|
||||
B, C, x = BCx.chunk(3, dim=-1)
|
||||
|
||||
# (dim, kernel_size) — same reshape as forward_cuda
|
||||
conv_weights = self.conv.weight.view(
|
||||
self.conv.weight.size(0), self.conv.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# Profile run — output value doesn't matter
|
||||
Bx = (B * x).contiguous()
|
||||
output_tensor, _ = self.out_proj(C * Bx)
|
||||
output[: hidden_states.shape[0]] = output_tensor
|
||||
return
|
||||
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
) # (num_blocks, dim, state_len)
|
||||
|
||||
num_prefills = attn_metadata.num_prefills
|
||||
num_decodes = attn_metadata.num_decode_tokens
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
num_actual_tokens = num_decodes + num_prefill_tokens
|
||||
|
||||
B_d, B_p = torch.split(
|
||||
B[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
C_d, C_p = torch.split(
|
||||
C[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
x_d, x_p = torch.split(
|
||||
x[:num_actual_tokens], [num_decodes, num_prefill_tokens], dim=0
|
||||
)
|
||||
|
||||
conv_output_list = []
|
||||
|
||||
if has_prefill:
|
||||
assert attn_metadata.state_indices_tensor_p is not None
|
||||
Bx_p = (B_p * x_p).transpose(0, 1) # (dim, num_prefill_tokens)
|
||||
out_p = causal_conv1d_torch(
|
||||
Bx_p,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
conv_state,
|
||||
attn_metadata.query_start_loc_p,
|
||||
attn_metadata.state_indices_tensor_p.flatten(),
|
||||
attn_metadata.has_initial_states_p,
|
||||
activation=None,
|
||||
).transpose(0, 1)[:num_prefill_tokens] # (num_prefill_tokens, dim)
|
||||
conv_output_list.append(C_p * out_p)
|
||||
|
||||
if has_decode:
|
||||
assert attn_metadata.state_indices_tensor_d is not None
|
||||
state_indices_d = attn_metadata.state_indices_tensor_d.flatten()
|
||||
Bx_d = B_d * x_d # (num_decodes, dim)
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
conv_state_view = conv_state[state_indices_d].contiguous()
|
||||
out_d = causal_conv1d_update_torch(
|
||||
Bx_d.unsqueeze(-1),
|
||||
conv_state_view,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
).squeeze(-1)
|
||||
conv_state[state_indices_d] = conv_state_view
|
||||
else:
|
||||
out_d = causal_conv1d_update_cpu(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_d,
|
||||
)
|
||||
conv_output_list.insert(0, C_d * out_d)
|
||||
|
||||
hidden_states_out = torch.vstack(conv_output_list)
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden_states_out)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
torch.ops.vllm.short_conv(
|
||||
hidden_states,
|
||||
output,
|
||||
self.prefix,
|
||||
)
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
):
|
||||
forward_context = get_forward_context()
|
||||
# ShortConvAttentionMetadata contains metadata necessary for the
|
||||
# short_conv triton kernels to operate in continuous batching and in
|
||||
# chunked prefill modes; they are computed at top-level model forward
|
||||
# since they stay the same and reused for all mamba layers in the same
|
||||
# iteration.
|
||||
attn_metadata_raw = forward_context.attn_metadata
|
||||
attn_metadata: AttentionMetadata | None = None
|
||||
if attn_metadata_raw is not None:
|
||||
assert isinstance(attn_metadata_raw, dict)
|
||||
attn_metadata = attn_metadata_raw.get(self.prefix)
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, ShortConvAttentionMetadata)
|
||||
conv_state = (
|
||||
self.kv_cache[0]
|
||||
if is_conv_state_dim_first()
|
||||
else self.kv_cache[0].transpose(-1, -2)
|
||||
)
|
||||
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
|
||||
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
|
||||
has_initial_states_p = attn_metadata.has_initial_states_p
|
||||
query_start_loc_p = attn_metadata.query_start_loc_p
|
||||
num_accepted_tokens = attn_metadata.num_accepted_tokens
|
||||
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||
|
||||
BCx, _ = self.in_proj(hidden_states)
|
||||
|
||||
B, C, x = BCx.chunk(3, dim=-1)
|
||||
|
||||
conv_weights = self.conv.weight.view(
|
||||
self.conv.weight.size(0), self.conv.weight.size(2)
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run
|
||||
Bx = (B * x).contiguous()
|
||||
hidden_states = C * Bx
|
||||
contextualized_states, _ = self.out_proj(hidden_states)
|
||||
return contextualized_states
|
||||
|
||||
num_prefills = attn_metadata.num_prefills # request count
|
||||
num_decodes = attn_metadata.num_decode_tokens # token count (=request)
|
||||
num_prefill_tokens = attn_metadata.num_prefill_tokens # token count
|
||||
has_prefill = num_prefills > 0
|
||||
has_decode = num_decodes > 0
|
||||
num_actual_tokens = num_decodes + num_prefill_tokens
|
||||
|
||||
# NOTE: V1 puts decode before prefill
|
||||
# Separate prefill and decode by splitting varlen input
|
||||
# Split along token dimension
|
||||
B_d, B_p = torch.split(
|
||||
B[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
C_d, C_p = torch.split(
|
||||
C[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
x_d, x_p = torch.split(
|
||||
x[:num_actual_tokens],
|
||||
[num_decodes, num_prefill_tokens],
|
||||
dim=0,
|
||||
)
|
||||
conv_output_list = []
|
||||
|
||||
if has_prefill:
|
||||
Bx_p = (B_p * x_p).transpose(0, 1)
|
||||
Bx = causal_conv1d_fn(
|
||||
Bx_p,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_states=conv_state,
|
||||
has_initial_state=has_initial_states_p,
|
||||
cache_indices=state_indices_tensor_p,
|
||||
metadata=attn_metadata,
|
||||
query_start_loc=query_start_loc_p,
|
||||
).transpose(0, 1)[:num_prefill_tokens]
|
||||
|
||||
y = C_p * Bx
|
||||
conv_output_list.append(y)
|
||||
|
||||
if has_decode:
|
||||
Bx_d = (B_d * x_d).contiguous()
|
||||
if num_accepted_tokens is not None:
|
||||
# Speculative decode: the verify step feeds >1 query token per
|
||||
# decode request, so use the spec-aware conv update path
|
||||
# (mirrors mamba_mixer2). state_indices_tensor_d is
|
||||
# (num_decodes, 1 + num_spec_tokens) here.
|
||||
assert state_indices_tensor_d is not None
|
||||
Bx = causal_conv1d_update(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc_d,
|
||||
max_query_len=state_indices_tensor_d.size(-1),
|
||||
)
|
||||
else:
|
||||
# Non-spec decode
|
||||
Bx = causal_conv1d_update(
|
||||
Bx_d,
|
||||
conv_state,
|
||||
conv_weights,
|
||||
self.conv.bias,
|
||||
activation=None,
|
||||
conv_state_indices=state_indices_tensor_d,
|
||||
)
|
||||
y = C_d * Bx
|
||||
conv_output_list.insert(0, y)
|
||||
|
||||
# Merge prefill and decode outputs before passing to gated MLP
|
||||
hidden_states = torch.vstack(conv_output_list)
|
||||
|
||||
# Final linear projection
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden_states)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
return MambaStateDtypeCalculator.short_conv_state_dtype(
|
||||
self.model_config.dtype,
|
||||
self.cache_config.mamba_cache_dtype,
|
||||
)
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...]]:
|
||||
return MambaStateShapeCalculator.short_conv_state_shape(
|
||||
tp_world_size=get_tensor_model_parallel_world_size(),
|
||||
intermediate_size=self.conv_dim,
|
||||
conv_kernel=self.L_cache,
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def short_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
if not current_platform.is_cpu():
|
||||
self.forward_cuda(hidden_states=hidden_states, output=output)
|
||||
else:
|
||||
self.forward_native(hidden_states=hidden_states, output=output)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="short_conv",
|
||||
op_func=short_conv,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -1,10 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.minimax_rms_norm.rms_norm_tp import (
|
||||
MiniMaxText01RMSNormTP,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MiniMaxText01RMSNormTP",
|
||||
]
|
||||
@@ -1,316 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import array
|
||||
import contextlib
|
||||
import ctypes
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import torch
|
||||
|
||||
try:
|
||||
from cuda.bindings import runtime as cudart
|
||||
except ImportError:
|
||||
from cuda import cudart
|
||||
|
||||
_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment
|
||||
_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CUDA helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _check(error):
|
||||
"""Raise on CUDA runtime error."""
|
||||
success = getattr(cudart.cudaError_t, "cudaSuccess", None) or cudart.cudaError_t(0)
|
||||
if error != success:
|
||||
raise RuntimeError(f"CUDA runtime error: {error}")
|
||||
|
||||
|
||||
# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so
|
||||
# copy the raw handle bytes through getPtr() instead. This is safe because the
|
||||
# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]).
|
||||
def _ipc_handle_to_bytes(handle) -> bytes:
|
||||
return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE)
|
||||
|
||||
|
||||
def _ipc_handle_from_bytes(raw: bytes):
|
||||
assert len(raw) == _IPC_HANDLE_SIZE, len(raw)
|
||||
handle = cudart.cudaIpcMemHandle_t()
|
||||
ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE)
|
||||
return handle
|
||||
|
||||
|
||||
def _cuda_malloc(size: int):
|
||||
aligned = ((size + _ALIGN - 1) >> 21) << 21
|
||||
err, ptr = cudart.cudaMalloc(aligned)
|
||||
_check(err)
|
||||
return ptr, aligned
|
||||
|
||||
|
||||
def _cuda_free(ptr: int):
|
||||
if ptr:
|
||||
_check(cudart.cudaFree(ptr)[0])
|
||||
|
||||
|
||||
def _cuda_memset_zero(ptr: int, size: int):
|
||||
_check(cudart.cudaMemset(ptr, 0, size)[0])
|
||||
|
||||
|
||||
def _cuda_memcpy_d2d(dst: int, src: int, size: int):
|
||||
_check(
|
||||
cudart.cudaMemcpy(
|
||||
dst, src, size, cudart.cudaMemcpyKind.cudaMemcpyDeviceToDevice
|
||||
)[0]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IPC buffer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class IpcBuffer:
|
||||
"""Allocates CUDA device memory and exchanges IPC handles with all ranks
|
||||
so that every rank holds a valid device pointer to every other rank's buffer.
|
||||
"""
|
||||
|
||||
def __init__(self, rank: int, world_size: int, size: int, process_group=None):
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.peer_ptrs: list[int] = [0] * world_size
|
||||
self.local_ptr: int = 0
|
||||
self._alive = False
|
||||
|
||||
if size <= 0:
|
||||
return
|
||||
|
||||
self.local_ptr, _ = _cuda_malloc(size)
|
||||
_cuda_memset_zero(self.local_ptr, size)
|
||||
self._alive = True
|
||||
|
||||
# --- exchange IPC handles via torch.distributed ---
|
||||
err, local_handle = cudart.cudaIpcGetMemHandle(self.local_ptr)
|
||||
_check(err)
|
||||
|
||||
all_handles: list[bytes | None] = [None] * world_size
|
||||
torch.distributed.all_gather_object(
|
||||
all_handles, _ipc_handle_to_bytes(local_handle), group=process_group
|
||||
)
|
||||
|
||||
for r in range(world_size):
|
||||
if r == rank:
|
||||
self.peer_ptrs[r] = self.local_ptr
|
||||
else:
|
||||
raw = all_handles[r]
|
||||
assert raw is not None
|
||||
handle = _ipc_handle_from_bytes(raw)
|
||||
err, ptr = cudart.cudaIpcOpenMemHandle(
|
||||
handle, cudart.cudaIpcMemLazyEnablePeerAccess
|
||||
)
|
||||
_check(err)
|
||||
self.peer_ptrs[r] = ptr
|
||||
|
||||
def serialize(self) -> list[int]:
|
||||
"""Return peer pointers as a list of int64 values (one per rank)."""
|
||||
raw = b""
|
||||
for ptr in self.peer_ptrs:
|
||||
raw += struct.pack("P", ptr)
|
||||
return array.array("Q", raw).tolist()
|
||||
|
||||
def cleanup(self):
|
||||
if not self._alive:
|
||||
return
|
||||
self._alive = False
|
||||
for r in range(self.world_size):
|
||||
if self.peer_ptrs[r] == 0:
|
||||
continue
|
||||
if r == self.rank:
|
||||
_cuda_free(self.peer_ptrs[r])
|
||||
else:
|
||||
with contextlib.suppress(RuntimeError):
|
||||
_check(cudart.cudaIpcCloseMemHandle(self.peer_ptrs[r])[0])
|
||||
self.peer_ptrs[r] = 0
|
||||
self.local_ptr = 0
|
||||
|
||||
def __del__(self):
|
||||
if not sys.is_finalizing():
|
||||
self.cleanup()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lamport negative-zero initialization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _lamport_fill_neg_zero(device_ptr: int, size_bytes: int):
|
||||
"""Fill device memory with IEEE-754 negative zero (-0.0f = 0x80000000).
|
||||
This is the "slot empty" sentinel for the Lamport protocol: the kernel
|
||||
spin-waits until a value is *not* negative zero.
|
||||
"""
|
||||
if size_bytes == 0 or device_ptr == 0:
|
||||
return
|
||||
n_floats = size_bytes // 4
|
||||
# torch preserves -0.0 in IEEE-754
|
||||
fill = torch.full((n_floats,), -0.0, dtype=torch.float32, device="cuda")
|
||||
_cuda_memcpy_d2d(device_ptr, fill.data_ptr(), size_bytes)
|
||||
del fill
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LamportWorkspace — the main class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LamportWorkspace:
|
||||
"""Self-contained workspace for Lamport-based cross-GPU AllReduce.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
rank : int
|
||||
Local rank (0-based).
|
||||
world_size : int
|
||||
Total number of ranks in the TP group.
|
||||
comm_size : int
|
||||
Size in bytes of *one* Lamport buffer slot. The total IPC allocation
|
||||
per rank is ``3 * comm_size`` (triple-buffering). Must be large enough
|
||||
to hold the per-slot data written by the kernel. Use
|
||||
``compute_comm_size_for_minimax()`` for a safe default.
|
||||
process_group : optional
|
||||
``torch.distributed`` process group for IPC handle exchange.
|
||||
``None`` uses the default group.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, rank: int, world_size: int, comm_size: int, process_group=None):
|
||||
assert world_size >= 2, "Lamport workspace requires at least 2 ranks"
|
||||
assert comm_size > 0, "comm_size must be positive"
|
||||
|
||||
self.rank = rank
|
||||
self.world_size = world_size
|
||||
self.comm_size = comm_size
|
||||
|
||||
# 1) Lamport triple-buffer (the only IPC memory the kernel reads/writes)
|
||||
lamport_total = 3 * comm_size
|
||||
self._lamport = IpcBuffer(rank, world_size, lamport_total, process_group)
|
||||
_lamport_fill_neg_zero(self._lamport.local_ptr, lamport_total)
|
||||
|
||||
# 2) flag_buffer on device: int32[3] = {counter, unused, lamport_flag}
|
||||
# counter — used for block-level sync inside the kernel
|
||||
# unused — reserved (index 1)
|
||||
# lamport_flag — triple-buffer rotation index (0 → 1 → 2 → 0 …)
|
||||
self._flag_buf = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
|
||||
# 3) layout_buffer on device: int64[2] = {clear_size, comm_size}
|
||||
# clear_size — bytes to clear from *previous* slot (set by kernel)
|
||||
# comm_size — size of one triple-buffer slot
|
||||
self._layout_buf = torch.tensor(
|
||||
[0, comm_size], dtype=torch.int64, device="cuda"
|
||||
)
|
||||
|
||||
# 4) Assemble device-side void* pointer array
|
||||
N = world_size
|
||||
ptrs: list[int] = []
|
||||
ptrs += [0] * N # [0 .. N-1] ipc_buffers (placeholder)
|
||||
ptrs += [0] * N # [N .. 2N-1] ipc_barriers (placeholder)
|
||||
ptrs += self._lamport.serialize() # [2N .. 3N-1] lamport peer ptrs
|
||||
ptrs.append(self._flag_buf.data_ptr()) # [3N] flag_buffer
|
||||
ptrs.append(self._layout_buf.data_ptr()) # [3N+1] layout_buffer
|
||||
|
||||
self._workspace = torch.tensor(ptrs, dtype=torch.int64, device="cuda")
|
||||
|
||||
@property
|
||||
def workspace(self) -> torch.Tensor:
|
||||
"""Device tensor (int64) that can be passed to the kernel
|
||||
as ``void** workspace``."""
|
||||
return self._workspace
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def compute_comm_size_for_minimax(
|
||||
max_tokens: int,
|
||||
world_size: int,
|
||||
fused_qk: bool = True,
|
||||
) -> int:
|
||||
"""Return a safe ``comm_size`` (in bytes) for MiniMaxReduceRMSKernel.
|
||||
|
||||
The kernel stores per-token variance scalars in the Lamport buffer:
|
||||
- single-matrix path: ``world_size × max_tokens × 4`` bytes per slot
|
||||
- fused Q+K path: ``world_size × 2 × ceil(max_tokens/4) × 16`` bytes per slot
|
||||
|
||||
The returned value is rounded up to 2 MiB alignment.
|
||||
"""
|
||||
if fused_qk:
|
||||
groups = (max_tokens + 3) // 4
|
||||
slot_bytes = world_size * 2 * groups * 16 # 16 = sizeof(float4)
|
||||
else:
|
||||
slot_bytes = world_size * max_tokens * 4 # 4 = sizeof(float)
|
||||
return ((slot_bytes + _ALIGN - 1) >> 21) << 21
|
||||
|
||||
def cleanup(self):
|
||||
if hasattr(self, "_lamport"):
|
||||
self._lamport.cleanup()
|
||||
|
||||
def __del__(self):
|
||||
if not sys.is_finalizing():
|
||||
self.cleanup()
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"LamportWorkspace(rank={self.rank}, world_size={self.world_size}, "
|
||||
f"comm_size={self.comm_size})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cached convenience function (mirrors TRT-LLM's get_allreduce_workspace)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_cache_lock = threading.Lock()
|
||||
_workspace_cache: dict = {}
|
||||
|
||||
|
||||
def get_allreduce_workspace(
|
||||
rank: int,
|
||||
world_size: int,
|
||||
comm_size: int | None = None,
|
||||
max_tokens: int = 16384,
|
||||
process_group=None,
|
||||
) -> torch.Tensor:
|
||||
"""Return a cached workspace tensor for the given (rank, world_size) pair.
|
||||
|
||||
On first call the workspace is allocated and IPC handles are exchanged;
|
||||
subsequent calls with the same arguments return the cached tensor.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
rank, world_size : int
|
||||
TP rank and TP size.
|
||||
comm_size : int, optional
|
||||
Explicit slot size in bytes. If ``None``, computed automatically
|
||||
from ``max_tokens`` and ``world_size`` (fused Q+K path).
|
||||
max_tokens : int
|
||||
Maximum number of tokens per batch (used when ``comm_size is None``).
|
||||
process_group : optional
|
||||
``torch.distributed`` process group.
|
||||
|
||||
"""
|
||||
if comm_size is None:
|
||||
comm_size = LamportWorkspace.compute_comm_size_for_minimax(
|
||||
max_tokens, world_size, fused_qk=True
|
||||
)
|
||||
pg_id = id(process_group) if process_group is not None else 0
|
||||
key = (rank, world_size, comm_size, pg_id)
|
||||
with _cache_lock:
|
||||
if key not in _workspace_cache:
|
||||
ws = LamportWorkspace(rank, world_size, comm_size, process_group)
|
||||
_workspace_cache[key] = ws
|
||||
return _workspace_cache[key].workspace
|
||||
@@ -1,421 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Max number of tokens supported by the Lamport fused allreduce+RMSNorm kernel.
|
||||
# Larger batches fall back to the eager allreduce + RMSNorm path.
|
||||
MINIMAX_QK_NORM_MAX_TOKEN_NUM = 2048
|
||||
|
||||
_MINIMAX_FUSED_AR_RMS_QK = getattr(torch.ops._C, "minimax_allreduce_rms_qk", None)
|
||||
|
||||
|
||||
def _all_reduce_variance(var: torch.Tensor) -> torch.Tensor:
|
||||
"""All-reduce a per-token variance tensor across the TP group.
|
||||
|
||||
Variance is accumulated in fp32 for numerical stability. The FlashInfer
|
||||
fused all-reduce caches a single global workspace keyed to the model's
|
||||
16-bit activation dtype (``use_fp32_lamport=False``); routing an fp32
|
||||
reduction through it would read against a mismatched workspace and corrupt
|
||||
the result. FlashInfer's fast-path only triggers for 2D inputs, so reducing
|
||||
a flattened (1D) view keeps these fp32 reductions on custom all-reduce /
|
||||
pynccl, both of which handle fp32 correctly.
|
||||
"""
|
||||
return tensor_model_parallel_all_reduce(var.flatten()).view_as(var)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _minimax_qk_var_kernel(
|
||||
qkv_ptr, # [num_tokens, hidden], 16-bit activations
|
||||
var_ptr, # [num_tokens, 2], fp32
|
||||
row_stride, # element stride between tokens in qkv
|
||||
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
|
||||
kv_size: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""TP-pre stage: per-token mean-of-squares for the q and k segments.
|
||||
|
||||
Accumulates in fp32 while reading the 16-bit qkv in place, so no fp32
|
||||
copy of q/k is materialized. ``var[:, 0]`` is the q variance and
|
||||
``var[:, 1]`` the k variance; both are the local-shard means, ready for
|
||||
the all-reduce that follows.
|
||||
"""
|
||||
token = tl.program_id(0)
|
||||
base = qkv_ptr + token * row_stride
|
||||
|
||||
q_acc = 0.0
|
||||
for off in range(0, q_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < q_size
|
||||
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
q_acc += tl.sum(x * x, axis=0)
|
||||
|
||||
k_acc = 0.0
|
||||
for off in range(0, kv_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < kv_size
|
||||
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
k_acc += tl.sum(x * x, axis=0)
|
||||
|
||||
tl.store(var_ptr + token * 2 + 0, q_acc / q_size)
|
||||
tl.store(var_ptr + token * 2 + 1, k_acc / kv_size)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _minimax_rms_apply_kernel(
|
||||
qkv_ptr, # [num_tokens, hidden]
|
||||
var_ptr, # [num_tokens, 2], fp32, all-reduced sum of per-shard means
|
||||
q_w_ptr, # [q_size], q per-channel weight
|
||||
k_w_ptr, # [kv_size], k per-channel weight
|
||||
q_out_ptr, # [num_tokens, q_size], contiguous
|
||||
k_out_ptr, # [num_tokens, kv_size], contiguous
|
||||
row_stride, # element stride between tokens in qkv
|
||||
q_size: tl.constexpr, # constant per deployment -> loops unroll, mask elides
|
||||
kv_size: tl.constexpr,
|
||||
tp_world: tl.constexpr, # folds the post-all-reduce /tp_world into rsqrt
|
||||
eps: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""TP-post stage: ``x * rsqrt(var / tp_world + eps) * weight``.
|
||||
|
||||
A single program normalizes both the q and k segments of one token, so q
|
||||
and k share one launch instead of two. The all-reduce yields the sum of
|
||||
per-shard means, so the ``/ tp_world`` that recovers the global
|
||||
mean-of-squares is folded into the ``rsqrt`` here rather than run as a
|
||||
separate elementwise pass over the ``[num_tokens, 2]`` variance tensor.
|
||||
"""
|
||||
token = tl.program_id(0)
|
||||
base = qkv_ptr + token * row_stride
|
||||
|
||||
q_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 0) / tp_world + eps)
|
||||
q_out_row = q_out_ptr + token * q_size
|
||||
for off in range(0, q_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < q_size
|
||||
x = tl.load(base + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
w = tl.load(q_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * q_inv * w
|
||||
tl.store(q_out_row + idx, y.to(q_out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
k_inv = tl.rsqrt(tl.load(var_ptr + token * 2 + 1) / tp_world + eps)
|
||||
k_out_row = k_out_ptr + token * kv_size
|
||||
for off in range(0, kv_size, BLOCK):
|
||||
idx = off + tl.arange(0, BLOCK)
|
||||
mask = idx < kv_size
|
||||
x = tl.load(base + q_size + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
w = tl.load(k_w_ptr + idx, mask=mask, other=0.0).to(tl.float32)
|
||||
y = x * k_inv * w
|
||||
tl.store(k_out_row + idx, y.to(k_out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
def _minimax_qk_norm_tp_eager(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pure-torch reference path used when Triton is unavailable."""
|
||||
q, k, _ = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
orig_dtype = q.dtype
|
||||
q = q.to(torch.float32)
|
||||
k = k.to(torch.float32)
|
||||
q_var = q.pow(2).mean(dim=-1, keepdim=True)
|
||||
k_var = k.pow(2).mean(dim=-1, keepdim=True)
|
||||
|
||||
qk_var = torch.cat([q_var, k_var], dim=-1)
|
||||
qk_var = _all_reduce_variance(qk_var) / tp_world
|
||||
q_var, k_var = qk_var.chunk(2, dim=-1)
|
||||
q = q * torch.rsqrt(q_var + eps) * q_weight
|
||||
k = k * torch.rsqrt(k_var + eps) * k_weight
|
||||
return q.to(orig_dtype), k.to(orig_dtype)
|
||||
|
||||
|
||||
def _minimax_qk_norm_tp_fallback(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_rank: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce + QK RMSNorm without the Lamport fused kernel.
|
||||
|
||||
The all-reduce is a TP communication barrier and cannot live inside a
|
||||
single kernel, so the eager-torch path is split into two Triton kernels
|
||||
around it: a variance reduction before the all-reduce and a normalize
|
||||
after. Compared to the ``torch.compile`` path this avoids materializing
|
||||
fp32 copies of q/k and the ``cat``/``chunk`` temporaries.
|
||||
"""
|
||||
if not HAS_TRITON:
|
||||
return _minimax_qk_norm_tp_eager(
|
||||
qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps
|
||||
)
|
||||
|
||||
num_tokens = qkv.shape[0]
|
||||
row_stride = qkv.stride(0)
|
||||
BLOCK = 1024
|
||||
grid = (num_tokens,)
|
||||
|
||||
qk_var = torch.empty(num_tokens, 2, dtype=torch.float32, device=qkv.device)
|
||||
_minimax_qk_var_kernel[grid](
|
||||
qkv, qk_var, row_stride, q_size=q_size, kv_size=kv_size, BLOCK=BLOCK
|
||||
)
|
||||
|
||||
# All-reduce sums the per-shard means; the /tp_world that turns this back
|
||||
# into the global mean is folded into the apply kernel's rsqrt below.
|
||||
qk_var = _all_reduce_variance(qk_var)
|
||||
|
||||
q_out = torch.empty(num_tokens, q_size, dtype=qkv.dtype, device=qkv.device)
|
||||
k_out = torch.empty(num_tokens, kv_size, dtype=qkv.dtype, device=qkv.device)
|
||||
_minimax_rms_apply_kernel[grid](
|
||||
qkv,
|
||||
qk_var,
|
||||
q_weight,
|
||||
k_weight,
|
||||
q_out,
|
||||
k_out,
|
||||
row_stride,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
tp_world=tp_world,
|
||||
eps=eps,
|
||||
BLOCK=BLOCK,
|
||||
)
|
||||
return q_out, k_out
|
||||
|
||||
|
||||
def _minimax_qk_norm_fusion(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_rank: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
workspace: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert qkv.ndim == 2
|
||||
num_tokens = qkv.shape[0]
|
||||
if (
|
||||
workspace is not None
|
||||
and tp_world > 1
|
||||
and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM
|
||||
and _MINIMAX_FUSED_AR_RMS_QK is not None
|
||||
):
|
||||
return _MINIMAX_FUSED_AR_RMS_QK(
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
workspace,
|
||||
q_size,
|
||||
kv_size,
|
||||
tp_rank,
|
||||
tp_world,
|
||||
eps,
|
||||
)
|
||||
# ROCm AITER fused QK-norm + AllReduce; skips unaligned shapes.
|
||||
if tp_world > 1 and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM:
|
||||
pack_size = 16 // qkv.element_size()
|
||||
warp_work_size = 32 * pack_size
|
||||
if q_size % warp_work_size == 0 and kv_size % warp_work_size == 0:
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if aiter_ar is not None:
|
||||
q_out, k_out, _ = aiter_ar.aiter_ca.custom_fused_qknorm_ar(
|
||||
qkv, q_weight, k_weight, eps
|
||||
)
|
||||
return q_out, k_out
|
||||
return _minimax_qk_norm_tp_fallback(
|
||||
qkv, q_weight, k_weight, q_size, kv_size, tp_rank, tp_world, eps
|
||||
)
|
||||
|
||||
|
||||
def _minimax_qk_norm_fusion_fake(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
tp_rank: int,
|
||||
tp_world: int,
|
||||
eps: float,
|
||||
workspace: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert qkv.ndim == 2
|
||||
num_tokens = qkv.shape[0]
|
||||
return (
|
||||
torch.empty([num_tokens, q_size], dtype=qkv.dtype, device=qkv.device),
|
||||
torch.empty([num_tokens, kv_size], dtype=qkv.dtype, device=qkv.device),
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="minimax_qk_norm_fusion",
|
||||
op_func=_minimax_qk_norm_fusion,
|
||||
fake_impl=_minimax_qk_norm_fusion_fake,
|
||||
mutates_args=[],
|
||||
)
|
||||
|
||||
|
||||
@CustomOp.register("minimax_text01_rmsnorm_tp")
|
||||
class MiniMaxText01RMSNormTP(CustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-6,
|
||||
*,
|
||||
weight_shard_world_size: int | None = None,
|
||||
weight_shard_rank: int | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_world = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.weight_shard_world = weight_shard_world_size or self.tp_world
|
||||
self.weight_shard_rank = (
|
||||
self.tp_rank if weight_shard_rank is None else weight_shard_rank
|
||||
)
|
||||
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size // self.weight_shard_world))
|
||||
self.weight.weight_loader = partial(
|
||||
self.weight_loader,
|
||||
shard_world_size=self.weight_shard_world,
|
||||
shard_rank=self.weight_shard_rank,
|
||||
)
|
||||
self.variance_epsilon = eps
|
||||
|
||||
self.workspace = None
|
||||
if _MINIMAX_FUSED_AR_RMS_QK is not None and self.tp_world > 1:
|
||||
from .lamport_workspace import (
|
||||
get_allreduce_workspace,
|
||||
)
|
||||
|
||||
# The Lamport workspace exchanges CUDA IPC handles and enables peer
|
||||
# access between GPUs. This requires P2P (IPC peer access) to be
|
||||
# available; on topologies where it is not (e.g. consumer PCIe cards
|
||||
# with P2P disabled in the driver), allocation raises. Fall back to
|
||||
# the eager allreduce + RMSNorm path instead of failing model load.
|
||||
try:
|
||||
self.workspace = get_allreduce_workspace(
|
||||
rank=self.tp_rank,
|
||||
world_size=self.tp_world,
|
||||
max_tokens=MINIMAX_QK_NORM_MAX_TOKEN_NUM,
|
||||
process_group=get_tp_group().cpu_group,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning_once(
|
||||
"Failed to initialize MiniMax fused allreduce+RMSNorm "
|
||||
"Lamport workspace: %s. This is expected on GPUs without "
|
||||
"P2P (IPC peer access) support. Falling back to the eager "
|
||||
"allreduce + RMSNorm path.",
|
||||
e,
|
||||
)
|
||||
self.workspace = None
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_world_size: int | None = None,
|
||||
shard_rank: int | None = None,
|
||||
) -> None:
|
||||
if shard_world_size is None:
|
||||
shard_world_size = get_tensor_model_parallel_world_size()
|
||||
if shard_rank is None:
|
||||
shard_rank = get_tensor_model_parallel_rank()
|
||||
|
||||
shard_size = loaded_weight.shape[0] // shard_world_size
|
||||
shard = slice(shard_rank * shard_size, (shard_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard])
|
||||
|
||||
def _forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
orig_dtype = x.dtype
|
||||
x = x.to(torch.float32)
|
||||
variance = x.pow(2).mean(dim=-1, keepdim=True, dtype=torch.float32)
|
||||
if self.tp_world > 1:
|
||||
variance = _all_reduce_variance(variance) / self.tp_world
|
||||
x = x * torch.rsqrt(variance + self.variance_epsilon)
|
||||
x = (x * self.weight).to(orig_dtype)
|
||||
return x
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
assert residual is None, "RMSNorm does not support residual connection."
|
||||
return self._forward(x)
|
||||
|
||||
@staticmethod
|
||||
def forward_qk(
|
||||
q_norm: "MiniMaxText01RMSNormTP",
|
||||
k_norm: "MiniMaxText01RMSNormTP",
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
orig_dtype = q.dtype
|
||||
q = q.to(torch.float32)
|
||||
k = k.to(torch.float32)
|
||||
q_var = q.pow(2).mean(dim=-1, keepdim=True)
|
||||
k_var = k.pow(2).mean(dim=-1, keepdim=True)
|
||||
|
||||
q = q * torch.rsqrt(q_var + q_norm.variance_epsilon) * q_norm.weight
|
||||
k = k * torch.rsqrt(k_var + k_norm.variance_epsilon) * k_norm.weight
|
||||
q = q.to(orig_dtype)
|
||||
k = k.to(orig_dtype)
|
||||
return q, k
|
||||
|
||||
@staticmethod
|
||||
def forward_qkv(
|
||||
q_norm: "MiniMaxText01RMSNormTP",
|
||||
k_norm: "MiniMaxText01RMSNormTP",
|
||||
qkv: torch.Tensor,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
assert qkv.ndim == 2
|
||||
|
||||
assert q_norm.variance_epsilon == k_norm.variance_epsilon
|
||||
# Case 0: tp_size=1
|
||||
if get_tensor_model_parallel_world_size() == 1:
|
||||
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
q, k = MiniMaxText01RMSNormTP.forward_qk(q_norm, k_norm, q, k)
|
||||
return q, k, v
|
||||
# Case : tp_size>1
|
||||
q, k = torch.ops.vllm.minimax_qk_norm_fusion(
|
||||
qkv,
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
q_size,
|
||||
kv_size,
|
||||
q_norm.tp_rank,
|
||||
q_norm.tp_world,
|
||||
q_norm.variance_epsilon,
|
||||
q_norm.workspace,
|
||||
)
|
||||
_, _, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
return q, k, v
|
||||
@@ -115,77 +115,9 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
}
|
||||
|
||||
_TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
# Text generation models
|
||||
"ArceeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"CwmForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"PhiForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
# Multimodal models
|
||||
"Emu3ForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"HunYuanVLForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
}
|
||||
_TRANSFORMERS_SUPPORTED_MODELS = {}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
# Text generation models
|
||||
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
# Multimodal models
|
||||
"TransformersMultiModalForCausalLM": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"TransformersMultiModalMoEForCausalLM": (
|
||||
"transformers",
|
||||
"TransformersMultiModalMoEForCausalLM",
|
||||
),
|
||||
# Embedding models
|
||||
"TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
|
||||
"TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
|
||||
"TransformersMultiModalEmbeddingModel": (
|
||||
"transformers",
|
||||
"TransformersMultiModalEmbeddingModel",
|
||||
),
|
||||
# Sequence classification models
|
||||
"TransformersForSequenceClassification": (
|
||||
"transformers",
|
||||
"TransformersForSequenceClassification",
|
||||
),
|
||||
"TransformersMoEForSequenceClassification": (
|
||||
"transformers",
|
||||
"TransformersMoEForSequenceClassification",
|
||||
),
|
||||
"TransformersMultiModalForSequenceClassification": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForSequenceClassification",
|
||||
),
|
||||
}
|
||||
_TRANSFORMERS_BACKEND_MODELS = {}
|
||||
|
||||
_VLLM_MODELS = {
|
||||
**_TEXT_GENERATION_MODELS,
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Wrapper around `transformers` models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch.nn.functional as F
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR
|
||||
from vllm.model_executor.models.transformers.causal import CausalMixin
|
||||
from vllm.model_executor.models.transformers.moe import MoEMixin
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
MultiModalDummyInputsBuilder,
|
||||
MultiModalMixin,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
|
||||
|
||||
def check_sinks(
|
||||
module: "torch.nn.Module",
|
||||
self_attn: "Attention | MLAAttention",
|
||||
s_aux: "torch.Tensor | None",
|
||||
):
|
||||
"""Fail loudly if the model applies a sink the attention layer will not.
|
||||
|
||||
Only the attention impl can fold a sink into the softmax denominator, so a sink
|
||||
that never reached `Attention` is dropped and every softmax is subtly wrong.
|
||||
"""
|
||||
if s_aux is None or getattr(self_attn, "has_sink", False):
|
||||
return
|
||||
raise ValueError(
|
||||
f"{type(module).__name__} applies attention sinks, but they were not passed "
|
||||
f"to {type(self_attn).__name__}, so the output would be wrong. Either the "
|
||||
"Transformers modeling backend could not find the parameter holding them, or "
|
||||
"vLLM does not support sinks for this kind of attention. Please open an issue "
|
||||
"at https://github.com/vllm-project/vllm/issues/new"
|
||||
)
|
||||
|
||||
|
||||
def vllm_attention_forward(
|
||||
# Transformers args
|
||||
module: "torch.nn.Module",
|
||||
query: "torch.Tensor",
|
||||
key: "torch.Tensor",
|
||||
value: "torch.Tensor",
|
||||
attention_mask: "torch.Tensor",
|
||||
**kwargs,
|
||||
):
|
||||
self_attn = getattr(module, VLLM_ATTN_ATTR)
|
||||
check_sinks(module, self_attn, kwargs.get("s_aux"))
|
||||
hidden = query.shape[-2]
|
||||
head_dim_qk = query.shape[-1]
|
||||
head_dim_v = value.shape[-1]
|
||||
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
|
||||
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
|
||||
# Pad `value` up to the query/key head size when it is smaller (expanded
|
||||
# MLA). A larger last dim just means `value` isn't split per head, e.g.
|
||||
# packed grouped/multi-query projections, and needs no padding.
|
||||
pad_value = head_dim_v < head_dim_qk
|
||||
if pad_value:
|
||||
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
|
||||
value = value.reshape(hidden, -1)
|
||||
attn_output = self_attn.forward(query, key, value)
|
||||
if pad_value:
|
||||
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
|
||||
attn_output = attn_output.reshape(hidden, -1)
|
||||
return attn_output, None
|
||||
|
||||
|
||||
def vllm_mla_attention_forward(
|
||||
# Transformers args
|
||||
module: "torch.nn.Module",
|
||||
query: "torch.Tensor",
|
||||
kv_c_normed: "torch.Tensor",
|
||||
k_pe: "torch.Tensor",
|
||||
attention_mask: "torch.Tensor",
|
||||
**kwargs,
|
||||
):
|
||||
self_attn = getattr(module, VLLM_ATTN_ATTR)
|
||||
check_sinks(module, self_attn, kwargs.get("s_aux"))
|
||||
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
|
||||
query = query.transpose(1, 2).flatten(0, 1)
|
||||
num_tokens, num_heads = query.shape[:2]
|
||||
# [batch=1, num_tokens, kv_lora_rank] -> [num_tokens, kv_lora_rank]
|
||||
kv_c_normed = kv_c_normed.reshape(-1, kv_c_normed.shape[-1])
|
||||
# [batch=1, heads=1, num_tokens, qk_rope] -> [num_tokens, 1, qk_rope]
|
||||
k_pe = k_pe.reshape(-1, 1, k_pe.shape[-1])
|
||||
attn_output = self_attn.forward(
|
||||
query,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(num_tokens, num_heads * self_attn.v_head_dim),
|
||||
)
|
||||
return attn_output, None
|
||||
|
||||
|
||||
ALL_ATTENTION_FUNCTIONS.register("vllm", vllm_attention_forward)
|
||||
ALL_ATTENTION_FUNCTIONS.register("vllm_mla", vllm_mla_attention_forward)
|
||||
|
||||
|
||||
# Text only models
|
||||
class TransformersForCausalLM(CausalMixin): ...
|
||||
|
||||
|
||||
class TransformersMoEForCausalLM(MoEMixin, CausalMixin): ...
|
||||
|
||||
|
||||
# Multimodal models
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MultiModalProcessor,
|
||||
info=MultiModalProcessingInfo,
|
||||
dummy_inputs=MultiModalDummyInputsBuilder,
|
||||
)
|
||||
class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ...
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MultiModalProcessor,
|
||||
info=MultiModalProcessingInfo,
|
||||
dummy_inputs=MultiModalDummyInputsBuilder,
|
||||
)
|
||||
class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ...
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Handle imports of non-existent classes with a helpful error message."""
|
||||
if name not in globals():
|
||||
raise AttributeError(
|
||||
"The Transformers modeling backend does not currently have a class to "
|
||||
f"handle the requested model type: {name}. Please open an issue at "
|
||||
"https://github.com/vllm-project/vllm/issues/new"
|
||||
)
|
||||
return globals()[name]
|
||||
@@ -1,860 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend base class."""
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import contextmanager
|
||||
from functools import cached_property
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from torch import nn
|
||||
from transformers import AutoModel, PreTrainedConfig
|
||||
from transformers.conversion_mapping import (
|
||||
WeightRenaming,
|
||||
get_checkpoint_conversion_mapping,
|
||||
get_model_conversion_mapping,
|
||||
)
|
||||
|
||||
from vllm.compilation.decorators import DynamicArgDims, support_torch_compile
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_pp_group, get_tp_group
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention, EncoderOnlyAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
|
||||
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers.attention import (
|
||||
VLLM_ATTN_IMPL,
|
||||
VLLM_MLA_ATTN_IMPL,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
named_state,
|
||||
replace_conv_class,
|
||||
replace_embedding_class,
|
||||
replace_layernorm_class,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
ShardId,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
try:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
except ImportError: # MLA support removed from this build
|
||||
MLAAttention = type("AbsentMLAAttention", (), {})
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
VLLM_ATTN_ATTR = "attn"
|
||||
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
|
||||
attention module. `"attn"` is chosen deliberately to match the mappings generated by
|
||||
`maybe_remap_kv_scale_name`."""
|
||||
|
||||
|
||||
class PreTrainedModelClasses(NamedTuple):
|
||||
decoder: type["PreTrainedModel"]
|
||||
encoders: dict[str, type["PreTrainedModel"]]
|
||||
"""Modality -> encoder class, for each modality that has one."""
|
||||
|
||||
|
||||
class Base(
|
||||
nn.Module,
|
||||
VllmModel,
|
||||
SupportsQuant,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
):
|
||||
hf_to_vllm_mapper: WeightsMapper
|
||||
|
||||
# TODO transformers will have a util to get these
|
||||
embedding_modules = {"embed_tokens": "input_embeddings"}
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
logger.info("Using Transformers modeling backend.")
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.text_config = self.config.get_text_config()
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.device_config = vllm_config.device_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.pp_group = get_pp_group()
|
||||
self.tp_group = get_tp_group()
|
||||
|
||||
# Attrs for weight loading (see self.load_weights)
|
||||
self.ignore_unexpected_prefixes: list[str] = []
|
||||
"""Ignore unexpected weights whose qualname starts with these prefixes."""
|
||||
self.ignore_unexpected_suffixes: list[str] = []
|
||||
"""Ignore unexpected weights whose qualname ends with these suffixes."""
|
||||
self.packed_modules_mapping: dict[str, list[str]] = {}
|
||||
"""Fused module -> constituent projections, populated by `recursive_replace`
|
||||
for the quantization machinery and loaders (e.g. bitsandbytes)."""
|
||||
self.fusers: dict[str, list[BaseFuser]] = {}
|
||||
"""Module qualname -> the fusers applied to it, populated
|
||||
by `recursive_replace` for `_create_attention_instances`."""
|
||||
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
|
||||
"""`layer_idx` -> the qualname and fuser of the module computing that
|
||||
layer's attention, populated by `recursive_replace` for
|
||||
`_create_attention_instances`."""
|
||||
|
||||
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
|
||||
self._target_class: type[nn.Module] = nn.Module
|
||||
"""Target class for Eagle3 aux hidden state recording."""
|
||||
self._layer_names: dict[int, str] = {}
|
||||
"""Mapping from layer index to layer name for Eagle3."""
|
||||
self._output_aux_hidden_states_kwargs: dict[str, bool] = {}
|
||||
"""Kwargs to pass to model forward for Eagle3 aux hidden states."""
|
||||
|
||||
if self.quant_config:
|
||||
quant_method_name = self.quant_config.get_name()
|
||||
# Check for unsupported quantization methods.
|
||||
if quant_method_name in ("mxfp4", "gpt_oss_mxfp4"):
|
||||
raise NotImplementedError(
|
||||
"Transformers modeling backend does "
|
||||
"not support MXFP4 quantization yet."
|
||||
)
|
||||
|
||||
self._patch_config()
|
||||
self._decorate_for_torch_compile()
|
||||
# Init on "meta" to delay allocating GPU tensors
|
||||
with (
|
||||
self._mark_model_components(vllm_config),
|
||||
init_on_device_without_buffers("meta"),
|
||||
):
|
||||
from_config_kwargs = self._from_config_kwargs
|
||||
self.model: PreTrainedModel = AutoModel.from_config(**from_config_kwargs)
|
||||
|
||||
# Create weight name to module qualname mapper
|
||||
self._create_hf_to_vllm_mapper()
|
||||
# Remove layers not on this pipeline parallel rank
|
||||
self.pipeline_parallel()
|
||||
# Skip the device sync of dynamic rope where it can never rescale
|
||||
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
self._create_attention_instances(prefix)
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
|
||||
# Upcast weights Transformers always keeps in fp32
|
||||
self.keep_in_fp32(self.model)
|
||||
|
||||
# Pipeline parallel intermediate tensors
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.text_config.hidden_size
|
||||
)
|
||||
|
||||
def _patch_config(self):
|
||||
"""Patch the config to ensure that the model is created correctly:
|
||||
|
||||
- Sets the attention implementation to "vllm" so the attention instances from
|
||||
`_create_attention_instances` are used
|
||||
- Sets the dtype to the default torch dtype set by vLLM because Transformers
|
||||
uses the config dtype when creating the model
|
||||
"""
|
||||
self.text_config._attn_implementation = VLLM_ATTN_IMPL
|
||||
self.config.dtype = torch.get_default_dtype()
|
||||
|
||||
@contextmanager
|
||||
def _mark_model_components(self, vllm_config: "VllmConfig"):
|
||||
"""Mark language model and tower submodules as `self.model` is created.
|
||||
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override."""
|
||||
yield
|
||||
|
||||
@cached_property
|
||||
def _from_config_kwargs(self) -> dict[str, Any]:
|
||||
"""The kwargs used to create `self.model`."""
|
||||
return dict(
|
||||
config=self.config,
|
||||
dtype=self.model_config.dtype,
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
def _find_encoder_classes(
|
||||
self, model: "PreTrainedModel"
|
||||
) -> dict[str, type["PreTrainedModel"]]:
|
||||
"""Find the encoder class of each modality `model` has one for.
|
||||
|
||||
Text models have none. Multi-modal ones override this.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@cached_property
|
||||
def _pre_trained_model_classes(self) -> PreTrainedModelClasses:
|
||||
"""The decoder class and the encoder class of each modality that has one.
|
||||
|
||||
Both come from a single throwaway model, since building one is not cheap.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
model: PreTrainedModel = AutoModel.from_config(**self._from_config_kwargs)
|
||||
model_classes = PreTrainedModelClasses(
|
||||
decoder=type(model.get_decoder()),
|
||||
encoders=self._find_encoder_classes(model),
|
||||
)
|
||||
del model
|
||||
|
||||
logger.debug("Identified model classes as: %s", model_classes)
|
||||
return model_classes
|
||||
|
||||
def _decorate_cls_for_torch_compile(
|
||||
self,
|
||||
cls: type["PreTrainedModel"],
|
||||
dynamic_arg_dims: DynamicArgDims | None,
|
||||
enable_if: Callable[["VllmConfig"], bool],
|
||||
is_encoder: bool,
|
||||
):
|
||||
"""Decorate `cls` to indicate to vLLM that it supports torch compile.
|
||||
|
||||
Args:
|
||||
cls: The PreTrainedModel class to decorate.
|
||||
dynamic_arg_dims: A mapping from argument name to the dynamic dimensions
|
||||
of the argument. If None, default dynamic arg dims will be used. See
|
||||
[`support_torch_compile`][vllm.compilation.decorators.support_torch_compile]
|
||||
for more details.
|
||||
enable_if: A function which takes in the vLLM config and returns whether
|
||||
torch compile should be enabled for this class.
|
||||
is_encoder: Whether the class being decorated is an encoder.
|
||||
|
||||
"""
|
||||
logger.debug(
|
||||
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
|
||||
cls.__name__,
|
||||
"encoder" if is_encoder else "decoder",
|
||||
dynamic_arg_dims,
|
||||
)
|
||||
|
||||
support_torch_compile(
|
||||
dynamic_arg_dims=dynamic_arg_dims,
|
||||
enable_if=enable_if,
|
||||
is_encoder=is_encoder,
|
||||
use_aot_compile=False,
|
||||
)(cls)
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
"""Decorate the model's decoder class to indicate to vLLM that it
|
||||
supports torch compile if `can_enable_torch_compile` is True."""
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._pre_trained_model_classes.decoder,
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
dynamic_arg_dims=DynamicArgDims(
|
||||
input_ids=1, # shape: [1, seq_len]
|
||||
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
|
||||
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
|
||||
),
|
||||
enable_if=can_enable_torch_compile,
|
||||
is_encoder=False,
|
||||
)
|
||||
|
||||
def _create_hf_to_vllm_mapper(self):
|
||||
"""Create a WeightsMapper to map checkpoint weight names to module qualnames.
|
||||
|
||||
This handles:
|
||||
|
||||
- Transformers weight renaming from `WeightRenaming`
|
||||
- Checkpoints saved with a base model prefix that is not `model`
|
||||
- Checkpoints saved with no base model prefix
|
||||
- Any quantization config specific mappings
|
||||
"""
|
||||
orig_to_new_renaming: list[WeightRenaming] = []
|
||||
orig_to_new_regex: dict[re.Pattern, str | None] = {}
|
||||
|
||||
mappings = get_model_conversion_mapping(self.model)
|
||||
# Ensure layers that are part of the model, but not AutoModel, get their mapping
|
||||
instantiated = {type(m).__name__ for m in self.model.modules()}
|
||||
for arch in self.config.architectures or []:
|
||||
if arch not in instantiated:
|
||||
mappings.extend(get_checkpoint_conversion_mapping(arch) or [])
|
||||
|
||||
for mapping in mappings:
|
||||
# Handle weights which have been renamed in Transformers
|
||||
if isinstance(mapping, WeightRenaming):
|
||||
orig_to_new_renaming.append(mapping)
|
||||
# TODO: Handle WeightConverter to enable layer merging
|
||||
|
||||
# Handle unexpected weights which should be ignored
|
||||
if self.model._keys_to_ignore_on_load_unexpected is not None:
|
||||
for key in self.model._keys_to_ignore_on_load_unexpected:
|
||||
orig_to_new_regex[re.compile(key)] = None
|
||||
|
||||
# Standardise base model prefix
|
||||
bmp = self.model.base_model_prefix
|
||||
expected_bmp = r"model.\1"
|
||||
# Handle checkpoints saved with different base model prefix
|
||||
if bmp and bmp != "model":
|
||||
different_bmp_pattern = re.compile(rf"^{bmp}\.(.+)")
|
||||
orig_to_new_regex[different_bmp_pattern] = expected_bmp
|
||||
# Handle direct children of self.model which were saved without the model prefix
|
||||
direct_children = chain(
|
||||
self.model.named_children(),
|
||||
self.model.named_parameters(recurse=False),
|
||||
self.model.named_buffers(recurse=False),
|
||||
)
|
||||
model_children = "|".join(name for name, _ in direct_children)
|
||||
missing_bmp_pattern = re.compile(rf"^(?!model\.)(({model_children}).*)")
|
||||
orig_to_new_regex[missing_bmp_pattern] = expected_bmp
|
||||
# Handle weights saved as direct children of self.model which no longer are
|
||||
unexpected_bmp_pattern = re.compile(rf"^(model\.)((?!{model_children}).+)")
|
||||
orig_to_new_regex[unexpected_bmp_pattern] = r"\2"
|
||||
# Handle lm_head which was saved inside the base model
|
||||
nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)")
|
||||
orig_to_new_regex[nested_lm_head_pattern] = r"\2"
|
||||
|
||||
self.hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_renaming=orig_to_new_renaming,
|
||||
orig_to_new_regex=orig_to_new_regex,
|
||||
)
|
||||
|
||||
# Apply mapping to quantization config if needed
|
||||
self._maybe_apply_model_mapping()
|
||||
|
||||
def _get_tie_word_embeddings(self):
|
||||
"""Check if the model has tied word embeddings."""
|
||||
# Models created with Transformers v4 and v5 will store this in different places
|
||||
tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False)
|
||||
tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False)
|
||||
return tie_word_embeddings_v4 or tie_word_embeddings_v5
|
||||
|
||||
def pipeline_parallel(self):
|
||||
"""Apply the model's pipeline parallelization plan."""
|
||||
if self.pp_group.world_size <= 1:
|
||||
return
|
||||
|
||||
if self.model.supports_pp_plan:
|
||||
module = self.model
|
||||
names = list(module._pp_plan.keys())
|
||||
else:
|
||||
module = self.model.get_decoder()
|
||||
has_parameters = lambda m: next(m.parameters(), None) is not None
|
||||
names = [n for n, c in module.named_children() if has_parameters(c)]
|
||||
tip = get_feature_request_tip(
|
||||
self.model_config.model, self.model_config.trust_remote_code
|
||||
)
|
||||
logger.warning_once(
|
||||
"%s does not define a pipeline parallel plan. The Transformers "
|
||||
"modeling backend will infer the split from the layers of %s in order "
|
||||
"of declaration and keep parameter-free modules on every rank. This "
|
||||
"may fail if the model's structure is non-standard. %s",
|
||||
type(self.model).__name__,
|
||||
type(module).__name__,
|
||||
tip,
|
||||
)
|
||||
|
||||
module_lists = []
|
||||
module_list_idx = None
|
||||
for i, name in enumerate(names):
|
||||
# attrgetter in case the module is nested (e.g. "text_model.layers")
|
||||
if isinstance(attrgetter(name)(module), nn.ModuleList):
|
||||
module_lists.append(name)
|
||||
module_list_idx = i
|
||||
|
||||
if len(module_lists) > 1:
|
||||
raise ValueError(
|
||||
"Pipeline parallel of models with multiple `ModuleList`s "
|
||||
"in the base model are not supported yet!"
|
||||
)
|
||||
if module_list_idx is None:
|
||||
raise ValueError(f"Could not find `ModuleList` in {type(module)}")
|
||||
|
||||
# Layers before module list
|
||||
for name in names[:module_list_idx]:
|
||||
if self.pp_group.is_first_rank or (
|
||||
self._get_tie_word_embeddings() and self.pp_group.is_last_rank
|
||||
):
|
||||
continue
|
||||
# attrsetter in case the module is nested (e.g. "text_model.embed_tokens")
|
||||
attrsetter(name)(module, PPMissingLayer())
|
||||
|
||||
# Module list
|
||||
start_layer, end_layer = get_pp_indices(
|
||||
self.text_config.num_hidden_layers,
|
||||
self.pp_group.rank_in_group,
|
||||
self.pp_group.world_size,
|
||||
)
|
||||
layers_name = names[module_list_idx]
|
||||
# attrgetter in case the module is nested (e.g. "text_model.layers")
|
||||
layers = attrgetter(layers_name)(module)
|
||||
for i in range(len(layers)):
|
||||
if start_layer <= i and i < end_layer:
|
||||
continue
|
||||
layers[i] = PPMissingLayer()
|
||||
|
||||
# Layers after module list
|
||||
for name in names[module_list_idx + 1 :]:
|
||||
# Modules that should be on last rank
|
||||
if not self.pp_group.is_last_rank:
|
||||
# attrsetter in case the module is nested (e.g. "text_model.norm")
|
||||
attrsetter(name)(module, PPMissingLayer())
|
||||
|
||||
def _vocab_embeddings(self) -> set[nn.Embedding]:
|
||||
"""The `nn.Embedding`s in `self.model` which hold a vocab table."""
|
||||
|
||||
def vocab_sizes(config: PreTrainedConfig):
|
||||
for key, value in vars(config).items():
|
||||
if isinstance(value, PreTrainedConfig):
|
||||
yield from vocab_sizes(value)
|
||||
elif "vocab_size" in key and isinstance(value, int):
|
||||
yield value
|
||||
|
||||
sizes = set(vocab_sizes(self.config))
|
||||
# `get_input_embeddings` may return a module which composes the `nn.Embedding`,
|
||||
# or a `PPMissingLayer` if the embeddings are not on this pipeline stage
|
||||
embeddings = {
|
||||
module
|
||||
for module in self.model.get_input_embeddings().modules()
|
||||
if isinstance(module, nn.Embedding)
|
||||
}
|
||||
embeddings.update(
|
||||
module
|
||||
for module in self.model.modules()
|
||||
if isinstance(module, nn.Embedding) and module.num_embeddings in sizes
|
||||
)
|
||||
return embeddings
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Recursively replace modules in the model as needed.
|
||||
|
||||
Currently, this replaces:
|
||||
|
||||
- GLUs with a fused `MergedColumnParallelLinear` + `...AndMul`
|
||||
- Attention QKV projections with a fused `QKVParallelLinear` + split
|
||||
- `nn.Linear` with vLLM's tensor parallel linear classes
|
||||
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
|
||||
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
|
||||
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
|
||||
- `nn.LayerNorm` with vLLM's `LayerNorm`
|
||||
"""
|
||||
tp_plan = self.model.tp_plan or {}
|
||||
|
||||
if not tp_plan and self.tp_group.world_size > 1:
|
||||
tip = get_feature_request_tip(
|
||||
self.model_config.model, self.model_config.trust_remote_code
|
||||
)
|
||||
logger.warning_once(
|
||||
"%s does not define a tensor parallel plan. The Transformers modeling "
|
||||
"backend will shard the model the best it can during graph fusion and "
|
||||
"replicate the rest. This may be suboptimal or fail if the model does "
|
||||
"not fuse cleanly. %s",
|
||||
type(self.model).__name__,
|
||||
tip,
|
||||
)
|
||||
|
||||
# Prefix the patterns because we always start from `self.model`
|
||||
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
|
||||
# Detect fusable patterns once per module class (cached, so this is cheap)
|
||||
fusers = Fusers(self.model, self.vllm_config)
|
||||
|
||||
vocab_embeddings = self._vocab_embeddings()
|
||||
|
||||
orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {}
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
self.fusers.setdefault(prefix, []).append(fuser)
|
||||
|
||||
if (
|
||||
isinstance(fuser, AttentionFuser)
|
||||
and (index := fuser.layer_index(module)) is not None
|
||||
):
|
||||
if index in self.attention_fusers:
|
||||
other, _ = self.attention_fusers[index]
|
||||
raise ValueError(
|
||||
f"Layer {index} dispatches attention from more than one "
|
||||
f"module ({other} and {prefix}), which one vLLM attention "
|
||||
"layer cannot serve."
|
||||
)
|
||||
self.attention_fusers[index] = (prefix, fuser)
|
||||
|
||||
orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix))
|
||||
|
||||
packed_modules_mapping = fuser.packed_modules_mapping
|
||||
self.packed_modules_mapping.update(packed_modules_mapping)
|
||||
if self.quant_config is not None:
|
||||
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
|
||||
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
new_module = child_module
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
if (
|
||||
isinstance(module, nn.ModuleList)
|
||||
and len(module) == self.text_config.num_hidden_layers
|
||||
):
|
||||
# Populate Eagle3 attrs
|
||||
self._target_class = type(child_module)
|
||||
layer_name = qual_name.removeprefix("model.")
|
||||
self._layer_names[int(child_name)] = layer_name
|
||||
# MTP weights should not be loaded into the base model
|
||||
num_hidden_layers = self.text_config.num_hidden_layers
|
||||
names = (
|
||||
"n_predict", # Override from SpeculativeConfig
|
||||
"num_nextn_predict_layers", # Most models
|
||||
"mtp_num_hidden_layers", # Qwen 3.5
|
||||
)
|
||||
n_predict = getattr_iter(self.text_config, names, 0)
|
||||
for i in range(num_hidden_layers, num_hidden_layers + n_predict):
|
||||
mtp_prefix = f"{prefix}.{i}."
|
||||
if mtp_prefix not in self.ignore_unexpected_prefixes:
|
||||
self.ignore_unexpected_prefixes.append(mtp_prefix)
|
||||
# Replace modules as needed
|
||||
if isinstance(child_module, nn.Linear):
|
||||
generator = (p for p in tp_plan if re.match(p, qual_name))
|
||||
pattern = next(generator, None)
|
||||
# Some weight loaders expect all linear layers to inherit
|
||||
# LinearBase, so we set a default style which causes any
|
||||
# unspecified layers to be replaced with ReplicatedLinear
|
||||
style = tp_plan.get(pattern, "replicate")
|
||||
new_module = replace_linear_class(
|
||||
child_module, style, self.quant_config, prefix=qual_name
|
||||
)
|
||||
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
|
||||
new_module = replace_conv_class(child_module)
|
||||
elif child_module in vocab_embeddings:
|
||||
new_module = replace_embedding_class(
|
||||
child_module, self.quant_config, prefix=qual_name
|
||||
)
|
||||
elif isinstance(child_module, nn.LayerNorm):
|
||||
new_module = replace_layernorm_class(child_module)
|
||||
elif child_module_fusers := fusers[child_module]:
|
||||
for fuser in child_module_fusers:
|
||||
register_fusion(fuser, qual_name, child_module)
|
||||
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
|
||||
logger.info_once(fuser.info(child_name))
|
||||
_recursive_replace(new_module, prefix=qual_name)
|
||||
elif not isinstance(child_module, MoERunner):
|
||||
# MoERunner can contain aliases of shared experts and gates,
|
||||
# so we don't want to recurse into it and break weight loading.
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
if new_module is not child_module:
|
||||
setattr(module, child_name, new_module)
|
||||
log_replacement(qual_name, child_module, new_module)
|
||||
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
||||
|
||||
def _create_attention_instances(self, prefix: str = ""):
|
||||
"""Create `Attention` instances to inform KV cache allocation."""
|
||||
text_config = self.text_config
|
||||
attn_cls = self._get_attn_cls()
|
||||
|
||||
# kv_lora_rank indicates that this is an MLA model
|
||||
if getattr(text_config, "kv_lora_rank", None) is not None:
|
||||
if attn_cls is MLAAttention:
|
||||
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
|
||||
else:
|
||||
# MLA model not using MLAAttention: recompute head_size for full attn
|
||||
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
|
||||
qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0)
|
||||
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
layer_types = getattr(text_config, "layer_types", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
pp_size = self.pp_group.world_size
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
if i not in self.attention_fusers:
|
||||
layer = (
|
||||
f"{i} ({layer_types[i]})"
|
||||
if layer_types and i < len(layer_types)
|
||||
else str(i)
|
||||
)
|
||||
raise ValueError(
|
||||
f"Layer {layer} does not dispatch through the Transformers "
|
||||
"attention interface and vLLM has no other way to handle it."
|
||||
)
|
||||
attn_prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(attn_prefix)
|
||||
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
arch_config = self.model_config.model_arch_config[i]
|
||||
num_heads = self.model_config.get_num_attention_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
head_size = arch_config.head_size
|
||||
if (scale := attn_fuser.scale(attn_module)) is None:
|
||||
# Default to Llama scale if AttentionFuser couldn't identify it
|
||||
scale = head_size**-0.5
|
||||
|
||||
kwargs = dict(
|
||||
num_heads=num_heads,
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
|
||||
)
|
||||
|
||||
if attn_cls is MLAAttention:
|
||||
mla_fuser = next(
|
||||
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
|
||||
None,
|
||||
)
|
||||
if mla_fuser is None:
|
||||
raise ValueError(
|
||||
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
|
||||
"of the model did, so its latent KV cache cannot be allocated."
|
||||
)
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
get_mla_dims,
|
||||
)
|
||||
|
||||
dims = get_mla_dims(self.model_config)
|
||||
kwargs.update(
|
||||
qk_nope_head_dim=dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=dims.qk_rope_head_dim,
|
||||
v_head_dim=dims.v_head_dim,
|
||||
q_lora_rank=dims.q_lora_rank,
|
||||
kv_lora_rank=dims.kv_lora_rank,
|
||||
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
|
||||
)
|
||||
else:
|
||||
kwargs.update(
|
||||
head_size=head_size,
|
||||
num_kv_heads=self.model_config.get_num_kv_heads(
|
||||
self.parallel_config, arch_config
|
||||
),
|
||||
logits_soft_cap=logits_soft_cap,
|
||||
)
|
||||
|
||||
# Handle interleaved sliding window attention
|
||||
if layer_types and layer_types[i] == "sliding_attention":
|
||||
kwargs["per_layer_sliding_window"] = text_config.sliding_window
|
||||
# Handle attention sinks
|
||||
if (sinks := attn_fuser.sinks(attn_module)) is not None:
|
||||
kwargs["sinks"] = sinks
|
||||
|
||||
attn_instance = attn_cls(**kwargs)
|
||||
|
||||
# Attach the instance to the module it serves, which is the only way it
|
||||
# is in the module tree at all: a plain dict is invisible to
|
||||
# `nn.Module.__setattr__`. Being in the tree runs its
|
||||
# `process_weights_after_loading` and moves its KV-scale buffers with
|
||||
# `model.to(device)`, and lets the attention interface read it off the
|
||||
# module rather than index by `module.layer_idx`, an `int` that bakes
|
||||
# layer identity into the traced graph and so costs one compiled
|
||||
# artifact per layer.
|
||||
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
|
||||
|
||||
def _get_attn_cls(self) -> type[AttentionLayerBase]:
|
||||
"""Return the `Attention` class to use for this model's layers."""
|
||||
# In encoder models, the attention layers will have `is_causal=False`
|
||||
is_encoder = lambda module: not getattr(module, "is_causal", True)
|
||||
has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
|
||||
is_multimodal = lambda config: config != config.get_text_config()
|
||||
# vLLM does not support encoder-decoder models, so if any encoder layer is
|
||||
# found in a text only model, we assume the whole model is an encoder model
|
||||
if has_encoder(self.model) and not is_multimodal(self.config):
|
||||
return EncoderOnlyAttention
|
||||
if self.model_config.use_mla:
|
||||
if any(
|
||||
isinstance(fuser, MLAFuser)
|
||||
for fusers in self.fusers.values()
|
||||
for fuser in fusers
|
||||
):
|
||||
return MLAAttention
|
||||
logger.warning_once(
|
||||
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
|
||||
"MLA attention layers. Falling back to full attention with a padded "
|
||||
"`value` head dimension."
|
||||
)
|
||||
os.environ["VLLM_MLA_DISABLE"] = "1"
|
||||
return Attention
|
||||
|
||||
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
|
||||
"""If a `parameter` is on the `meta` device, then its parent
|
||||
`module` is the original module created by:
|
||||
|
||||
```python
|
||||
with torch.device("meta"):
|
||||
self.model: "PreTrainedModel" = AutoModel.from_config(...)
|
||||
```
|
||||
"""
|
||||
dtype = dtype or self.model_config.dtype
|
||||
device = self.device_config.device
|
||||
|
||||
def _init_parameters(module: nn.Module):
|
||||
for name, param in module.named_parameters(recurse=False):
|
||||
# Already on device, nothing to do
|
||||
if param.device != torch.device("meta"):
|
||||
continue
|
||||
# Already a vLLM parameter, nothing to do
|
||||
if hasattr(param, "weight_loader"):
|
||||
continue
|
||||
data = torch.empty_like(param.data, dtype=dtype, device=device)
|
||||
setattr(module, name, nn.Parameter(data=data))
|
||||
for child in module.children():
|
||||
_init_parameters(child)
|
||||
|
||||
_init_parameters(module)
|
||||
|
||||
def keep_in_fp32(self, module: nn.Module):
|
||||
"""Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would."""
|
||||
if self.model_config.dtype not in (torch.float16, torch.bfloat16):
|
||||
return
|
||||
fragments = getattr(module, "_keep_in_fp32_modules_strict", None)
|
||||
if not fragments:
|
||||
return
|
||||
pattern = re.compile("|".join(re.escape(f) for f in fragments))
|
||||
for name, tensor in named_state(module):
|
||||
if not hasattr(tensor, "weight_loader") and pattern.search(name):
|
||||
tensor.data = tensor.data.to(torch.float32)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings()(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if not self.pp_group.is_first_rank:
|
||||
assert intermediate_tensors is not None
|
||||
input_ids = None
|
||||
inputs_embeds = intermediate_tensors["hidden_states"]
|
||||
|
||||
# Add batch dimension before entering Transformers model
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
# [seq_len] -> [1, seq_len]
|
||||
input_ids = input_ids[None, ...]
|
||||
if inputs_embeds is not None and inputs_embeds.ndim == 2:
|
||||
# [seq_len, hidden_size] -> [1, seq_len, hidden_size]
|
||||
inputs_embeds = inputs_embeds[None, ...]
|
||||
if positions.ndim == 1:
|
||||
# [seq_len] -> [1, seq_len]
|
||||
positions = positions[None, ...]
|
||||
|
||||
# Transformers models expect either input_ids or inputs_embeds, but not both
|
||||
if input_ids is not None and inputs_embeds is not None:
|
||||
input_ids = None
|
||||
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=False,
|
||||
position_ids=positions,
|
||||
return_dict=False,
|
||||
**self._output_aux_hidden_states_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Remove batch dimension after exiting Transformers model
|
||||
hidden_states = outputs[0][0, ...]
|
||||
if self._output_aux_hidden_states_kwargs:
|
||||
aux_hidden_states = [x[0][0, ...] for x in outputs[1:]]
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
if self._output_aux_hidden_states_kwargs and len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
|
||||
ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@staticmethod
|
||||
def check_version(min_version: str, feature: str):
|
||||
installed = Version(transformers.__version__)
|
||||
required = Version(min_version)
|
||||
if installed < required:
|
||||
raise ImportError(
|
||||
f"Transformers modeling backend requires transformers>={required} "
|
||||
f"for {feature}, but got {installed}"
|
||||
)
|
||||
|
||||
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
from transformers.utils.output_capturing import (
|
||||
OutputRecorder,
|
||||
maybe_install_capturing_hooks,
|
||||
)
|
||||
|
||||
# The default value in PreTrainedModel is None
|
||||
if self.model._can_record_outputs is None:
|
||||
self.model._can_record_outputs = {}
|
||||
|
||||
target_class = self._target_class
|
||||
for layer in layers:
|
||||
# layer - 1 because we want the input to the layer
|
||||
layer_name = self._layer_names[layer - 1]
|
||||
layer_key = f"aux_hidden_state_{layer}"
|
||||
aux_hidden_state_i = OutputRecorder(target_class, layer_name=layer_name)
|
||||
self.model._can_record_outputs[layer_key] = aux_hidden_state_i
|
||||
self._output_aux_hidden_states_kwargs[f"output_{layer_key}"] = True
|
||||
|
||||
# Ensure that the capture hooks are installed before dynamo traces the model
|
||||
maybe_install_capturing_hooks(self.model)
|
||||
|
||||
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
|
||||
num_layers = self.text_config.num_hidden_layers
|
||||
return (2, num_layers // 2, num_layers - 3)
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for causal language models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
|
||||
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
|
||||
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class CausalMixin(VllmModelForTextGeneration, Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip VllmModelForTextGeneration.__init__ and call the next class in MRO
|
||||
super(VllmModelForTextGeneration, self).__init__(
|
||||
vllm_config=vllm_config, prefix=prefix
|
||||
)
|
||||
|
||||
tie_word_embeddings = self._get_tie_word_embeddings()
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.text_config.vocab_size,
|
||||
self.text_config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if tie_word_embeddings:
|
||||
for module in self.model.get_input_embeddings().modules():
|
||||
if isinstance(module, VocabParallelEmbedding):
|
||||
self.lm_head = self.lm_head.tie_weights(module)
|
||||
break
|
||||
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.text_config.vocab_size,
|
||||
scale=getattr(self.text_config, "logit_scale", 1.0),
|
||||
soft_cap=getattr(self.text_config, "final_logit_softcapping", None),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
|
||||
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
|
||||
lm_head_bias = set()
|
||||
load_device = self.vllm_config.load_config.device or self.device_config.device
|
||||
|
||||
def auto_load_lm_head_bias(weights):
|
||||
for name, weight in weights:
|
||||
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
|
||||
# load_weights is called outside vLLM's device context,
|
||||
# so we must enter it again when registering the bias.
|
||||
with torch.device(load_device):
|
||||
self.lm_head._register_bias()
|
||||
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
|
||||
lm_head_bias.add(name)
|
||||
else:
|
||||
yield name, weight
|
||||
|
||||
return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias
|
||||
|
||||
def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
|
||||
return logits
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fuser detection for the Transformers modeling backend."""
|
||||
|
||||
from collections import UserDict
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from cachetools import cached
|
||||
from torch import nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers import (
|
||||
AttentionFuser,
|
||||
BaseFuser,
|
||||
GLUFuser,
|
||||
MergedColumnParallelFuser,
|
||||
MLAFuser,
|
||||
PackedQKVFuser,
|
||||
QKVFuser,
|
||||
RewriteFuser,
|
||||
RMSNormFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
F = TypeVar("F", bound=BaseFuser)
|
||||
|
||||
|
||||
def key(module: nn.Module) -> tuple:
|
||||
"""Cache key for `get_fusers`. Considers module type and its immediate children."""
|
||||
return (type(module), tuple(name for name, _ in module.named_children()))
|
||||
|
||||
|
||||
FUSERS: tuple[type[BaseFuser], ...] = (
|
||||
# Order these by priority, e.g. QKV would be preferred over MergedColumnParallel
|
||||
MLAFuser,
|
||||
GLUFuser,
|
||||
QKVFuser,
|
||||
PackedQKVFuser,
|
||||
MergedColumnParallelFuser,
|
||||
RMSNormFuser,
|
||||
# Put fusers that don't redefine forward after this comment
|
||||
AttentionFuser,
|
||||
)
|
||||
"""Every fuser, in priority order: those that redefine the forward first, then those
|
||||
that leave it alone. A new fuser is added here."""
|
||||
|
||||
|
||||
@cached(cache={}, key=key)
|
||||
def get_fusers(module: nn.Module) -> list[BaseFuser]:
|
||||
"""Every fuser that could apply to `module`'s class (cached), in `FUSERS` order."""
|
||||
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
|
||||
# leaf module (raw tensor math, no submodules). Nothing else can match, and
|
||||
# tracing is skipped for it.
|
||||
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
|
||||
is_leaf = next(module.children(), None) is None
|
||||
graph = trace(module) if n_linear >= 2 or is_leaf else None
|
||||
|
||||
# Every redefining candidate that matches is cached, not just the first:
|
||||
# instances of this class can be shaped heterogeneously, so which one is
|
||||
# usable is decided per instance in Fusers.__getitem__, not here. A forward
|
||||
# can only be rewritten from a trace of it, so each starts fresh regardless.
|
||||
fusers: list[BaseFuser] = []
|
||||
for fuser_cls in FUSERS:
|
||||
if fuser_cls.redefines_forward and graph is None:
|
||||
continue
|
||||
if (fuser := fuser_cls.match(graph, module)) is None:
|
||||
continue
|
||||
if isinstance(fuser, RewriteFuser):
|
||||
try:
|
||||
fuser.update_forward(module)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Attempted to fuse %s using %s but failed "
|
||||
"to update its forward method: %s",
|
||||
type(module),
|
||||
fuser_cls.__name__,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
fusers.append(fuser)
|
||||
|
||||
# A norm we could not match structurally is left unfused; flag likely misses.
|
||||
if (
|
||||
module.__class__.__name__.endswith("RMSNorm")
|
||||
and graph is not None
|
||||
and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
|
||||
):
|
||||
logger.warning_once(
|
||||
"%s looks like an RMSNorm but its computation did not match the "
|
||||
"expected pattern, so it was left unfused.",
|
||||
module.__class__.__name__,
|
||||
)
|
||||
return fusers
|
||||
|
||||
|
||||
def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
|
||||
"""The first `fuser_cls` that applies to `module`, if one does."""
|
||||
return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
|
||||
|
||||
|
||||
class Fusers(UserDict):
|
||||
"""Mapping from module class and shape to fusers, for all fusable modules."""
|
||||
|
||||
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
|
||||
self.vllm_config = vllm_config
|
||||
super().__init__({key(m): get_fusers(m) for m in model.modules()})
|
||||
|
||||
def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
|
||||
"""The fusers this instance can take, in the order to apply them."""
|
||||
chosen: list[BaseFuser] = []
|
||||
redefined = False
|
||||
for fuser in self.data.get(key(m), ()):
|
||||
# Only one candidate may rewrite m's forward; take the first
|
||||
# (highest-priority) one that validates for this instance.
|
||||
if fuser.redefines_forward and redefined:
|
||||
continue
|
||||
if not fuser.validate(m, self.vllm_config):
|
||||
continue
|
||||
redefined = redefined or fuser.redefines_forward
|
||||
chosen.append(fuser)
|
||||
return chosen
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Concrete fusers for the Transformers modeling backend."""
|
||||
|
||||
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
BaseFuser,
|
||||
RewriteFuser,
|
||||
StackedFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.merged_column import (
|
||||
MergedColumnParallelFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.mla import MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
|
||||
|
||||
__all__ = [
|
||||
"AttentionFuser",
|
||||
"BaseFuser",
|
||||
"RewriteFuser",
|
||||
"StackedFuser",
|
||||
"GLUFuser",
|
||||
"MergedColumnParallelFuser",
|
||||
"MLAFuser",
|
||||
"MoEBlockFuser",
|
||||
"PackedQKVFuser",
|
||||
"QKVFuser",
|
||||
"RMSNormFuser",
|
||||
]
|
||||
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Attention fuser: the module that dispatches to the attention interface."""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
VLLM_ATTN_IMPL = "vllm"
|
||||
VLLM_MLA_ATTN_IMPL = "vllm_mla"
|
||||
|
||||
|
||||
def _is_interface_lookup(node: ast.expr | None) -> bool:
|
||||
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
|
||||
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
|
||||
if isinstance(node, ast.Call):
|
||||
node = node.func
|
||||
if not isinstance(node, (ast.Attribute, ast.Subscript)):
|
||||
return False
|
||||
return (
|
||||
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def interface_call(forward: Callable) -> ast.Call | None:
|
||||
"""The attention interface call in `forward`, if it makes exactly one."""
|
||||
try:
|
||||
source = inspect.getsource(inspect.unwrap(forward))
|
||||
tree = ast.parse(textwrap.dedent(source))
|
||||
except (AttributeError, OSError, SyntaxError, TypeError):
|
||||
return None
|
||||
|
||||
# Find the names of the local variables that read from the interface lookup
|
||||
names: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
|
||||
continue
|
||||
if not _is_interface_lookup(node.value):
|
||||
continue
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
names.update(t.id for t in targets if isinstance(t, ast.Name))
|
||||
# Find the calls to those names, and return the one if there is exactly one
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id in names
|
||||
]
|
||||
return calls[0] if len(calls) == 1 else None
|
||||
|
||||
|
||||
def _resolve(node: ast.expr, module: nn.Module) -> object:
|
||||
"""The value of `node` on `module`, for literals and `self.<attr>`."""
|
||||
if isinstance(node, ast.Constant):
|
||||
return node.value
|
||||
if (
|
||||
isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
):
|
||||
return getattr(module, node.attr, None)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttentionFuser(BaseFuser):
|
||||
"""A module that dispatches through the Transformers attention interface."""
|
||||
|
||||
redefines_forward: ClassVar[bool] = False
|
||||
|
||||
source_cls: str
|
||||
"""Class of the HF module that dispatches (for logging)."""
|
||||
scale_expr: ast.expr | None = None
|
||||
"""Source of the `scaling=` the module hands the interface, if it hands one."""
|
||||
s_aux_expr: ast.expr | None = None
|
||||
"""Source of the `s_aux=` the module hands the interface, if it hands one."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
return f"Found: {name} ({self.source_cls}) -> attention interface"
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls, graph: fx.Graph | None, module: nn.Module
|
||||
) -> "AttentionFuser | None":
|
||||
if (call := interface_call(type(module).forward)) is None:
|
||||
return None
|
||||
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
|
||||
scale_expr = scaling[0] if len(scaling) == 1 else None
|
||||
s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"]
|
||||
s_aux_expr = s_aux[0] if len(s_aux) == 1 else None
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
scale_expr=scale_expr,
|
||||
s_aux_expr=s_aux_expr,
|
||||
)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether `module` will actually dispatch to vLLM."""
|
||||
config = getattr(module, "config", None)
|
||||
# Only patched in the text config, this excludes attention based mm encoders
|
||||
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
|
||||
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
if (sinks := self.sinks(module)) is not None:
|
||||
assert isinstance(self.s_aux_expr, ast.Attribute)
|
||||
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
|
||||
device = vllm_config.device_config.device
|
||||
data = torch.empty(size, dtype=sinks.dtype, device=device)
|
||||
sinks_param = nn.Parameter(data, requires_grad=False)
|
||||
set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)})
|
||||
setattr(module, self.s_aux_expr.attr, sinks_param)
|
||||
return module
|
||||
|
||||
def layer_index(self, module: nn.Module) -> int | None:
|
||||
"""The layer `module` computes attention for, if it declares one."""
|
||||
layer_idx = getattr(module, "layer_idx", None)
|
||||
return layer_idx if isinstance(layer_idx, int) else None
|
||||
|
||||
def scale(self, module: nn.Module) -> float | None:
|
||||
"""The softmax scale `module` passes to the interface, or `None`."""
|
||||
if self.scale_expr is None:
|
||||
return None
|
||||
scale = _resolve(self.scale_expr, module)
|
||||
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
|
||||
expression = ast.unparse(self.scale_expr)
|
||||
raise ValueError(
|
||||
f"Cannot resolve attention scaling expression {expression!r} in "
|
||||
f"{type(module).__name__}."
|
||||
)
|
||||
return float(scale)
|
||||
|
||||
def sinks(self, module: nn.Module) -> nn.Parameter | None:
|
||||
"""The per-head sink tensor `module` passes to the interface, or `None`."""
|
||||
if self.s_aux_expr is None:
|
||||
return None
|
||||
s_aux = _resolve(self.s_aux_expr, module)
|
||||
if not isinstance(s_aux, (nn.Parameter, type(None))):
|
||||
expression = ast.unparse(self.s_aux_expr)
|
||||
raise ValueError(
|
||||
f"Cannot resolve attention s_aux expression {expression!r} in "
|
||||
f"{type(module).__name__}."
|
||||
)
|
||||
return s_aux
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base classes for the Transformers backend fusers."""
|
||||
|
||||
import ast
|
||||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
aliasing_names,
|
||||
bypass_existence_guard,
|
||||
replace_expr,
|
||||
self_call_and_refs,
|
||||
written_names,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaseFuser(ABC):
|
||||
"""A detected fusion and how to apply it.
|
||||
|
||||
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
|
||||
then applies the fusion to an instance in `recursive_replace`, returning the
|
||||
module to install in its place.
|
||||
"""
|
||||
|
||||
redefines_forward: ClassVar[bool] = True
|
||||
"""Whether `fuse` gives the module a different forward,
|
||||
by rewriting its source or by returning a different module in its place."""
|
||||
|
||||
@abstractmethod
|
||||
def info(self, name: str) -> str:
|
||||
"""A human-readable description of the fusion at `name`, for logging."""
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "BaseFuser | None":
|
||||
"""Match the pattern in `graph`, returning a fuser if found."""
|
||||
|
||||
@abstractmethod
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether this fuser can be applied to this `module` instance."""
|
||||
|
||||
@abstractmethod
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
|
||||
(none unless it stacks weights)."""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`packed_modules_mapping` entries this fuser contributes (none unless
|
||||
it stacks weights)."""
|
||||
return {}
|
||||
|
||||
|
||||
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
|
||||
"""The head size of `module`, which head counts are derived from.
|
||||
|
||||
Prefer the module's own `head_dim`, which Transformers sets per instance:
|
||||
the model-wide value is the largest head size across layers, so on a
|
||||
heterogeneous checkpoint it would divide out the wrong number of heads, and
|
||||
it is the text head size, so it does not describe a vision tower at all.
|
||||
"""
|
||||
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
|
||||
|
||||
|
||||
def local_output_sizes(merged_name: str) -> str:
|
||||
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
|
||||
merged = f"self.{merged_name}"
|
||||
return f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RewriteFuser(BaseFuser):
|
||||
"""A fuser that rewrites the module's forward and rebinds it.
|
||||
|
||||
`match` and `update_forward` analyse the class once; `fuse` swaps the
|
||||
submodules and binds the compiled forward on an instance in place, so it
|
||||
keeps its class and any attribute the fusion does not consume.
|
||||
"""
|
||||
|
||||
source_cls: str
|
||||
"""Class of the HF module the fused projections belonged to (for logging)."""
|
||||
|
||||
fused_forward: Callable = field(init=False, repr=False)
|
||||
"""The compiled rewritten forward, set by `update_forward`."""
|
||||
|
||||
@abstractmethod
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite and compile `type(module)`'s forward source.
|
||||
|
||||
Raises if the source does not admit the rewrite (fusion is then skipped).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace `module`'s submodules with their vLLM equivalents."""
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it."""
|
||||
|
||||
merged_cls_name: ClassVar[str]
|
||||
"""Name of the vLLM class the merged projection becomes (for logging)."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def merged_name(self) -> str:
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
sources = " + ".join(shard for shard, _ in self.shards)
|
||||
return (
|
||||
f"Fused: {sources} ({name}: {self.source_cls}) -> "
|
||||
f"{self.merged_name} ({self.merged_cls_name})"
|
||||
)
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""Each projection's original name and its shard id in the merged module.
|
||||
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
|
||||
|
||||
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
|
||||
so only this exact layer is remapped, never a same-named projection
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
|
||||
merged = maybe_prefix(prefix, self.merged_name)
|
||||
return {
|
||||
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
|
||||
}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _unguarded_calls(
|
||||
self, funcdef: ast.FunctionDef, names: Iterable[str]
|
||||
) -> list[ast.Call]:
|
||||
"""One `self.<name>(arg)` call per projection, existence guards folded.
|
||||
|
||||
`update_attrs` deletes the projections it merges, so any reference to one beyond
|
||||
its call site must be a guard on its existence; folding those to their constant
|
||||
value keeps the rewritten forward off a name that no longer exists. A reference
|
||||
that is not such a guard raises, so fusion is skipped."""
|
||||
calls = []
|
||||
for name in names:
|
||||
call, refs = self_call_and_refs(funcdef, name)
|
||||
for ref in refs:
|
||||
bypass_existence_guard(funcdef, ref, name)
|
||||
calls.append(call)
|
||||
return calls
|
||||
|
||||
def _check_input_stable(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
module: nn.Module,
|
||||
calls: list[ast.Call],
|
||||
block: list[ast.stmt],
|
||||
indices: list[int],
|
||||
) -> None:
|
||||
"""Raise unless hoisting the merged GEMM preserves what it reads.
|
||||
|
||||
Fusing moves every projection to one call at `min(indices)`, so the
|
||||
merged GEMM reads the input once, up front, where the last of `calls`
|
||||
would have read it later. That holds only if nothing in between changes
|
||||
the input, and a change need not name it: writing any view that shares
|
||||
its storage changes it too. Both halves of the check over-approximate,
|
||||
since a false hit costs a fusion while a miss returns wrong numbers.
|
||||
"""
|
||||
arg_names = {
|
||||
node.id
|
||||
for node in ast.walk(calls[0].args[0])
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
|
||||
}
|
||||
# Protect the names the input reads and anything that may alias them.
|
||||
tracked = aliasing_names(funcdef, arg_names, module)
|
||||
# Any of them rebound, written through, or passed to a call that writes
|
||||
# its argument would leave the hoisted GEMM reading a different value.
|
||||
region = block[min(indices) : max(indices) + 1]
|
||||
if tracked & written_names(region):
|
||||
raise ValueError("projection input is rebound or mutated before all calls")
|
||||
|
||||
def _splice_merged_split(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
calls: list[ast.Call],
|
||||
block: list[ast.stmt],
|
||||
index: int,
|
||||
) -> None:
|
||||
"""Insert `temps = self.<merged_name>(arg).split(sizes, -1)` at
|
||||
`block[index]` and replace each of `calls` with its temp name.
|
||||
|
||||
`calls` must share one input argument; `block[index]` must be where
|
||||
they are (or would be) evaluated. Raises if a generated temporary
|
||||
would shadow an existing name in `funcdef`.
|
||||
"""
|
||||
temps = [f"_vllm_merged_{i}" for i in range(len(calls))]
|
||||
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
|
||||
if names & set(temps):
|
||||
raise ValueError("fused temporaries would shadow existing names")
|
||||
targets = ", ".join(temps)
|
||||
sections = local_output_sizes(self.merged_name)
|
||||
source = f"{targets} = self.{self.merged_name}(__arg__).split({sections}, -1)"
|
||||
assign = ast.parse(source).body[0]
|
||||
arg = next(
|
||||
node
|
||||
for node in ast.walk(assign)
|
||||
if isinstance(node, ast.Name) and node.id == "__arg__"
|
||||
)
|
||||
replace_expr(assign, arg, calls[0].args[0])
|
||||
ast.copy_location(assign, block[index])
|
||||
block.insert(index, assign)
|
||||
for call, temp in zip(calls, temps):
|
||||
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
|
||||
@@ -1,191 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLU projection fuser: `act(gate(x)) * up(x)` -> a fused gate/up linear."""
|
||||
|
||||
import ast
|
||||
import operator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
from transformers.activations import ACT2CLS
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
|
||||
from vllm.model_executor.models.transformers.fusers.merged_column import (
|
||||
MergedColumnParallelFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
find_node,
|
||||
is_linear,
|
||||
peel,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
CLS2ACT: dict[type, list[str]] = {}
|
||||
for _act_name, _act_cls in ACT2CLS.items():
|
||||
if isinstance(_act_cls, tuple):
|
||||
_act_cls = _act_cls[0]
|
||||
CLS2ACT.setdefault(_act_cls, []).append(_act_name)
|
||||
|
||||
ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys())
|
||||
|
||||
|
||||
@dataclass
|
||||
class GLUFuser(MergedColumnParallelFuser):
|
||||
"""Fuser for the GLU pattern `act(gate(x)) * up(x)`."""
|
||||
|
||||
act_name: str
|
||||
down_name: str | None
|
||||
merged_name: ClassVar[str] = "gate_up_proj"
|
||||
|
||||
@property
|
||||
def gate_name(self) -> str:
|
||||
return self.linear_names[0]
|
||||
|
||||
@property
|
||||
def up_name(self) -> str:
|
||||
return self.linear_names[1]
|
||||
|
||||
@classmethod
|
||||
def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool:
|
||||
"""Is node `act(gate(x))` where `gate` is linear and `act` is not linear."""
|
||||
return (
|
||||
node.op == "call_module"
|
||||
and not is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
and is_linear(node.args[0], module)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_glu_nodes(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Search graph for the GLU pattern `act(gate(x)) * up(x)`."""
|
||||
for mul in graph.nodes:
|
||||
if (
|
||||
mul.op == "call_function"
|
||||
and mul.target == operator.mul
|
||||
and len(mul.args) == 2
|
||||
and all(isinstance(arg, fx.Node) for arg in mul.args)
|
||||
):
|
||||
a, b = mul.args
|
||||
if cls._is_act_of_gate(a, module) and is_linear(b, module):
|
||||
act, gate, up = a, a.args[0], b
|
||||
elif cls._is_act_of_gate(b, module) and is_linear(a, module):
|
||||
act, gate, up = b, b.args[0], a
|
||||
else:
|
||||
continue
|
||||
if (
|
||||
all(len(args) == 1 for args in (gate.args, up.args))
|
||||
and isinstance(x := gate.args[0], fx.Node)
|
||||
and x is up.args[0]
|
||||
):
|
||||
return act, gate, up, mul
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_act_and_mul_name(act: nn.Module) -> str | None:
|
||||
"""Get the name of `act` if it has an `...AndMul` equivalent."""
|
||||
for name in CLS2ACT.get(type(act), []):
|
||||
if name in ACT_AND_MUL_NAMES:
|
||||
return name
|
||||
# nn.GELU is not in ACT2CLS, but could be in model code
|
||||
if type(act) is nn.GELU:
|
||||
return "gelu_pytorch_tanh" if act.approximate == "tanh" else "gelu"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _get_act_and_mul(cls, act: nn.Module) -> nn.Module:
|
||||
"""Get the `...AndMul` equivalent of a Transformers activation module."""
|
||||
if name := cls._get_act_and_mul_name(act):
|
||||
return get_act_and_mul_fn(name)
|
||||
raise ValueError(f"No AndMul equivalent for {type(act)}")
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "GLUFuser | None":
|
||||
if (glu_nodes := cls._get_glu_nodes(graph, module)) is None:
|
||||
return None
|
||||
act_node, gate_node, up_node, mul_node = glu_nodes
|
||||
|
||||
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
|
||||
down_node = find_node(graph, predicate)
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
act_name=act_node.target,
|
||||
linear_names=(gate_node.target, up_node.target),
|
||||
down_name=down_node.target if down_node is not None else None,
|
||||
)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
act_call = single_self_call(funcdef, self.act_name)
|
||||
calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name))
|
||||
gate_call, up_call = calls
|
||||
if act_call.args[0] is not gate_call:
|
||||
raise ValueError("activation does not directly wrap the gate")
|
||||
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
|
||||
raise ValueError("gate and up inputs are written differently")
|
||||
muls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.BinOp)
|
||||
and isinstance(node.op, ast.Mult)
|
||||
and {id(node.left), id(node.right)} == {id(act_call), id(up_call)}
|
||||
]
|
||||
if len(muls) != 1:
|
||||
raise ValueError("no multiply of the activation and up projection")
|
||||
|
||||
# act(gate(x)) * up(x) -> act(gate_up(x))
|
||||
assert isinstance(gate_call.func, ast.Attribute)
|
||||
gate_call.func.attr = self.merged_name
|
||||
replace_expr(funcdef, muls[0], act_call)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
if not super().validate(module, vllm_config):
|
||||
return False
|
||||
# An AndMul kernel splits its input in half, so gate and up must match.
|
||||
gate = module.get_submodule(self.gate_name)
|
||||
up = module.get_submodule(self.up_name)
|
||||
if gate.out_features != up.out_features:
|
||||
logger.debug("%s and %s differ in size; skipping fusion", gate, up)
|
||||
return False
|
||||
act = module.get_submodule(self.act_name)
|
||||
if self._get_act_and_mul_name(act) is None:
|
||||
logger.debug("%s has no AndMul equivalent; skipping fusion", act)
|
||||
return False
|
||||
return True
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
super().update_attrs(module, prefix, vllm_config)
|
||||
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
|
||||
setattr(module, self.act_name, act_fn)
|
||||
# If there is a down projection, we know it must be rowwise.
|
||||
if self.down_name is not None:
|
||||
down_prefix = maybe_prefix(prefix, self.down_name)
|
||||
down = module.get_submodule(self.down_name)
|
||||
new_down = replace_linear_class(
|
||||
down, "rowwise", vllm_config.quant_config, prefix=down_prefix
|
||||
)
|
||||
setattr(module, self.down_name, new_down)
|
||||
log_replacement(down_prefix, down, new_down)
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fuser for parallel linear projections."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
block_chain,
|
||||
compile_forward,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MergedColumnParallelFuser(StackedFuser):
|
||||
"""Fuser for merging column-parallel linear projections."""
|
||||
|
||||
linear_names: tuple[str, ...]
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
@property
|
||||
def merged_name(self) -> str:
|
||||
"""Programmatic name for the merged projection, based on the original names."""
|
||||
# len is used to disambiguate names like `a_b` + `c` vs `a` + `b_c`
|
||||
parts = "_".join(f"{len(name)}_{name}" for name in self.linear_names)
|
||||
return f"merged_proj_{parts}"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
return [(name, index) for index, name in enumerate(self.linear_names)]
|
||||
|
||||
@staticmethod
|
||||
def _names(group: list[fx.Node]) -> tuple[str, ...] | None:
|
||||
names = tuple(str(node.target) for node in group)
|
||||
if len(set(names)) != len(names) or any("." in name for name in names):
|
||||
return None
|
||||
return names
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> "MergedColumnParallelFuser | None":
|
||||
"""Fuse the module's sibling linears when there is only one such group."""
|
||||
by_input: dict[fx.Node, list[fx.Node]] = {}
|
||||
for node in graph.nodes:
|
||||
if (
|
||||
is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
# Like QKVFuser/PackedQKVFuser: this fuser has no head or
|
||||
# TP-replication awareness, so it must not absorb a QKV-shaped
|
||||
# pattern (e.g. behind a norm) a more specific fuser declined.
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
by_input.setdefault(node.args[0], []).append(node)
|
||||
# A group whose members are not distinct direct children is dropped:
|
||||
# the source rewrite addresses each projection as `self.<name>` once.
|
||||
groups = [
|
||||
(group, names)
|
||||
for group in by_input.values()
|
||||
if len(group) >= 2 and (names := cls._names(group)) is not None
|
||||
]
|
||||
if len(groups) > 1:
|
||||
logger.debug(
|
||||
"%s has %d fusable sibling-linear groups; skipping fusion "
|
||||
"since which one to merge is ambiguous",
|
||||
type(module).__name__,
|
||||
len(groups),
|
||||
)
|
||||
if len(groups) != 1:
|
||||
return None
|
||||
_, names = groups[0]
|
||||
fuser = cls(source_cls=type(module).__name__, linear_names=names)
|
||||
return None if hasattr(module, fuser.merged_name) else fuser
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace the parallel calls with one merged call and split."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
calls = self._unguarded_calls(funcdef, self.linear_names)
|
||||
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
|
||||
raise ValueError("parallel linears read different inputs")
|
||||
chains = [block_chain(funcdef.body, call) for call in calls]
|
||||
if any(not chain for chain in chains):
|
||||
raise ValueError("parallel linear calls not found in the function body")
|
||||
blocks = [chain[-1] for chain in chains]
|
||||
if len(set(id(block) for block, _ in blocks)) != 1:
|
||||
raise ValueError("parallel linear calls are in different blocks")
|
||||
|
||||
block = blocks[0][0]
|
||||
indices = [index for _, index in blocks]
|
||||
index = min(indices)
|
||||
self._check_input_stable(funcdef, module, calls, block, indices)
|
||||
|
||||
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
|
||||
self._splice_merged_split(funcdef, calls, block, index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Check that the parallel linears are compatible for merging."""
|
||||
linear_layers = [module.get_submodule(name) for name in self.linear_names]
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
return (
|
||||
len(linear_layers) >= 2
|
||||
and len(set(linear.in_features for linear in linear_layers)) == 1
|
||||
and len(set(linear.bias is None for linear in linear_layers)) == 1
|
||||
and all(linear.out_features % tp_size == 0 for linear in linear_layers)
|
||||
)
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace the module's parallel linears with one merged projection."""
|
||||
linear_modules = [module.get_submodule(name) for name in self.linear_names]
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=linear_modules[0].in_features,
|
||||
output_sizes=[linear.out_features for linear in linear_modules],
|
||||
bias=linear_modules[0].bias is not None,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
for name in self.linear_names:
|
||||
delattr(module, name)
|
||||
logger.debug(
|
||||
"%s -> %s: %s",
|
||||
", ".join(f"{n}: {m}" for n, m in zip(self.linear_names, linear_modules)),
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
@@ -1,303 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MLA fuser: adapt a Transformers MLA attention module for vLLM's MLA layer."""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
downstream_linear,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
Style,
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _consumes_placeholder(node: fx.Node) -> bool:
|
||||
"""Whether `node` is a linear applied directly to a `forward` input."""
|
||||
return (
|
||||
len(node.args) == 1
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
and node.args[0].op == "placeholder"
|
||||
)
|
||||
|
||||
|
||||
def _norm_size(norm: nn.Module) -> int:
|
||||
weight = getattr(norm, "weight", None)
|
||||
return weight.numel() if weight is not None else -1
|
||||
|
||||
|
||||
def _is_rms_norm(module: nn.Module) -> bool:
|
||||
"""Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher."""
|
||||
# Go via `get_fuser` because it caches
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
|
||||
return get_fuser(module, RMSNormFuser) is not None
|
||||
|
||||
|
||||
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
|
||||
"""Index in `funcdef.body` of the top-level statement containing `node`."""
|
||||
for index, stmt in enumerate(funcdef.body):
|
||||
if any(child is node for child in ast.walk(stmt)):
|
||||
return index
|
||||
raise ValueError("node is not in the function body")
|
||||
|
||||
|
||||
def _single_expand_call(
|
||||
funcdef: ast.FunctionDef, module: nn.Module, kv_b_proj_name: str
|
||||
) -> ast.Call:
|
||||
"""The KV expansion call `self.<method>(kv_c_normed, k_pe)`: the unique
|
||||
two-argument call to a method of `module` with the expansion's signature.
|
||||
|
||||
The expansion is `kv_c_normed, k_pe -> key, value`: two inputs, every
|
||||
`return` a pair, and it applies `kv_b_proj` -- the projection `MLAAttention`
|
||||
owns after the bypass, absorbed into the attention computation.
|
||||
"""
|
||||
|
||||
def is_expansion_method(name: str) -> bool:
|
||||
method = getattr(type(module), name, None)
|
||||
if method is None:
|
||||
return False
|
||||
code = getattr(inspect.unwrap(method), "__code__", None)
|
||||
if (
|
||||
code is None
|
||||
or code.co_argcount != 3 # self, kv_c_normed, k_pe
|
||||
or kv_b_proj_name not in code.co_names
|
||||
):
|
||||
return False
|
||||
try:
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(method)))
|
||||
except (OSError, TypeError):
|
||||
return False
|
||||
returns = [node for node in ast.walk(tree) if isinstance(node, ast.Return)]
|
||||
return bool(returns) and all(
|
||||
isinstance(ret.value, ast.Tuple) and len(ret.value.elts) == 2
|
||||
for ret in returns
|
||||
)
|
||||
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and isinstance(node.func.value, ast.Name)
|
||||
and node.func.value.id == "self"
|
||||
and len(node.args) == 2
|
||||
and not node.keywords
|
||||
and is_expansion_method(node.func.attr)
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError("expected exactly one KV expansion method call")
|
||||
return calls[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLAFuser(StackedFuser):
|
||||
"""Fuser for the MLA attention pattern."""
|
||||
|
||||
q_proj_name: str | None
|
||||
q_a_proj_name: str | None
|
||||
q_a_layernorm_name: str | None
|
||||
q_b_proj_name: str | None
|
||||
kv_a_proj_name: str
|
||||
kv_a_layernorm_name: str
|
||||
kv_b_proj_name: str
|
||||
o_proj_name: str | None
|
||||
merged_name: ClassVar[str] = "fused_qkv_a_proj"
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
info_str = (
|
||||
f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)"
|
||||
)
|
||||
if self.q_a_proj_name is not None:
|
||||
info_str += "; " + super().info(name).removeprefix("Fused: ")
|
||||
return info_str
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection."""
|
||||
if self.q_a_proj_name is None:
|
||||
return []
|
||||
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
if self.q_a_proj_name is None:
|
||||
return {}
|
||||
return super().packed_modules_mapping
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None":
|
||||
# Find all `rms_norm(linear(placeholder))` chains.
|
||||
chains = []
|
||||
for node in graph.nodes:
|
||||
if node.op != "call_module" or is_linear(node, module) or not node.args:
|
||||
continue
|
||||
source = upstream_linear(node.args[0], module)
|
||||
if source is None or not _consumes_placeholder(source):
|
||||
continue
|
||||
if not _is_rms_norm(module.get_submodule(node.target)):
|
||||
continue
|
||||
chains.append((source, node))
|
||||
|
||||
# Tell the chains apart by width.
|
||||
def is_kv_a(chain) -> bool:
|
||||
source, rms_norm = chain
|
||||
source_mod = module.get_submodule(source.target)
|
||||
rms_norm_mod = module.get_submodule(rms_norm.target)
|
||||
return source_mod.out_features != _norm_size(rms_norm_mod)
|
||||
|
||||
kv_a_chains = [chain for chain in chains if is_kv_a(chain)]
|
||||
q_a_chains = [chain for chain in chains if not is_kv_a(chain)]
|
||||
# Exactly one KV chain is MLA's signature. The query chain is optional.
|
||||
if len(kv_a_chains) != 1 or len(q_a_chains) > 1:
|
||||
return None
|
||||
kv_a_proj, kv_a_layernorm = kv_a_chains[0]
|
||||
|
||||
# Linear children claimed for a role so far; the rest resolve by elimination.
|
||||
claimed_linears = {kv_a_proj.target}
|
||||
q_proj_name = q_a_proj = q_a_layernorm = q_b_proj = None
|
||||
if q_a_chains:
|
||||
# Find `q_b_proj(q_a_layernorm(q_a_proj(placeholder)))`.
|
||||
q_a_proj, q_a_layernorm = q_a_chains[0]
|
||||
q_b_proj = downstream_linear(q_a_layernorm, module)
|
||||
if q_b_proj is None:
|
||||
return None
|
||||
claimed_linears |= {q_a_proj.target, q_b_proj.target}
|
||||
else:
|
||||
# Find `q_proj(placeholder)`.
|
||||
placeholder_linears = {
|
||||
node.target
|
||||
for node in graph.nodes
|
||||
if is_linear(node, module) and _consumes_placeholder(node)
|
||||
}
|
||||
if len(q_proj_candidates := placeholder_linears - claimed_linears) != 1:
|
||||
return None
|
||||
q_proj_name = next(iter(q_proj_candidates))
|
||||
claimed_linears.add(q_proj_name)
|
||||
|
||||
# Find `kv_b_proj(kv_a_layernorm(...))`.
|
||||
kv_b_proj = downstream_linear(kv_a_layernorm, module)
|
||||
if kv_b_proj is None or kv_b_proj.target in claimed_linears:
|
||||
return None
|
||||
claimed_linears.add(kv_b_proj.target)
|
||||
|
||||
# Find `o_proj` if it is returned by the forward graph.
|
||||
o_proj_name = returned_linear(graph, module)
|
||||
if o_proj_name in claimed_linears:
|
||||
o_proj_name = None
|
||||
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
q_proj_name=q_proj_name,
|
||||
q_a_proj_name=q_a_proj.target if q_a_proj else None,
|
||||
q_a_layernorm_name=q_a_layernorm.target if q_a_layernorm else None,
|
||||
q_b_proj_name=q_b_proj.target if q_b_proj else None,
|
||||
kv_a_proj_name=kv_a_proj.target,
|
||||
kv_a_layernorm_name=kv_a_layernorm.target,
|
||||
kv_b_proj_name=kv_b_proj.target,
|
||||
o_proj_name=o_proj_name,
|
||||
)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
return vllm_config.model_config.use_mla
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
|
||||
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
|
||||
attention interface unexpanded."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
if (q_a_proj_name := self.q_a_proj_name) is not None:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
# The fused call is inserted at the top-level statement preceding both.
|
||||
names = [q_a_proj_name, self.kv_a_proj_name]
|
||||
calls = self._unguarded_calls(funcdef, names)
|
||||
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
|
||||
raise ValueError("down-projections read different inputs")
|
||||
indices = [_top_level_index(funcdef, call) for call in calls]
|
||||
self._check_input_stable(funcdef, module, calls, funcdef.body, indices)
|
||||
self._splice_merged_split(funcdef, calls, funcdef.body, min(indices))
|
||||
|
||||
# Transformers expands the latent into full key/value in a dedicated method.
|
||||
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
|
||||
# so replace the expansion call with its own arguments so `kv_c_normed, k_pe`
|
||||
# flow to the interface in place of `key, value`.
|
||||
expand_call = _single_expand_call(funcdef, module, self.kv_b_proj_name)
|
||||
replace_expr(
|
||||
funcdef, expand_call, ast.Tuple(elts=list(expand_call.args), ctx=ast.Load())
|
||||
)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
def replace_linear_by_name(name: str, style: Style):
|
||||
linear = module.get_submodule(name)
|
||||
replacement = replace_linear_class(
|
||||
linear, style, quant_config, prefix=maybe_prefix(prefix, name)
|
||||
)
|
||||
setattr(module, name, replacement)
|
||||
log_replacement(maybe_prefix(prefix, name), linear, replacement)
|
||||
|
||||
if (q_a_proj_name := self.q_a_proj_name) is not None:
|
||||
# `match` sets the q-LoRA names together, or none of them
|
||||
assert self.q_b_proj_name is not None
|
||||
q_a = module.get_submodule(q_a_proj_name)
|
||||
kv_a = module.get_submodule(self.kv_a_proj_name)
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=q_a.in_features,
|
||||
output_sizes=[q_a.out_features, kv_a.out_features],
|
||||
bias=q_a.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
logger.debug(
|
||||
"%s: %s, %s: %s -> %s: %s",
|
||||
q_a_proj_name,
|
||||
q_a,
|
||||
self.kv_a_proj_name,
|
||||
kv_a,
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
# The rewritten forward calls the merged projection instead.
|
||||
delattr(module, q_a_proj_name)
|
||||
delattr(module, self.kv_a_proj_name)
|
||||
replace_linear_by_name(self.q_b_proj_name, "colwise")
|
||||
else:
|
||||
assert self.q_proj_name is not None
|
||||
replace_linear_by_name(self.kv_a_proj_name, "replicate")
|
||||
replace_linear_by_name(self.q_proj_name, "colwise")
|
||||
|
||||
replace_linear_by_name(self.kv_b_proj_name, "colwise")
|
||||
# MLAAttention calls kv_b_proj and expects vLLM's default return_bias=True
|
||||
module.get_submodule(self.kv_b_proj_name).return_bias = True
|
||||
if self.o_proj_name is not None:
|
||||
replace_linear_by_name(self.o_proj_name, "rowwise")
|
||||
@@ -1,439 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
|
||||
|
||||
import ast
|
||||
import functools
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
is_complete,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
stack = list(ast.iter_child_nodes(node))
|
||||
while stack:
|
||||
child = stack.pop()
|
||||
if isinstance(child, ast.Return):
|
||||
yield child
|
||||
elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
||||
stack.extend(ast.iter_child_nodes(child))
|
||||
|
||||
|
||||
def _returns_tuple(cls: type[nn.Module]) -> bool:
|
||||
"""Does `cls.forward()` return a tuple?"""
|
||||
try:
|
||||
source = textwrap.dedent(inspect.getsource(inspect.unwrap(cls.forward)))
|
||||
forward = ast.parse(source).body[0]
|
||||
except (OSError, SyntaxError, TypeError, IndexError):
|
||||
return True
|
||||
# Names bound to a tuple literal, e.g. `out = hidden, logits` then `return out`.
|
||||
tuple_names = {
|
||||
target.id
|
||||
for node in ast.walk(forward)
|
||||
if isinstance(node, ast.Assign) and isinstance(node.value, ast.Tuple)
|
||||
for target in node.targets
|
||||
if isinstance(target, ast.Name)
|
||||
}
|
||||
|
||||
def yields_tuple(value: ast.expr | None) -> bool:
|
||||
if isinstance(value, ast.Tuple):
|
||||
return True
|
||||
if isinstance(value, ast.Name):
|
||||
return value.id in tuple_names
|
||||
if isinstance(value, ast.IfExp):
|
||||
return yields_tuple(value.body) or yields_tuple(value.orelse)
|
||||
return False
|
||||
|
||||
return any(yields_tuple(ret.value) for ret in _own_returns(forward))
|
||||
|
||||
|
||||
def _is_scalar_gate(module: nn.Module) -> bool:
|
||||
"""A linear projecting to a single logit (the shared-expert sigmoid gate)."""
|
||||
weight = getattr(module, "weight", None)
|
||||
return (
|
||||
isinstance(module, nn.Linear)
|
||||
and weight is not None
|
||||
and weight.ndim == 2
|
||||
and weight.shape[0] == 1
|
||||
)
|
||||
|
||||
|
||||
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
"""The floating dtype `nodes` cast to, if any.
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it."""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
name = str(node.target).rsplit(".", 1)[-1]
|
||||
if name == "float":
|
||||
return torch.float32
|
||||
if name in ("to", "type"):
|
||||
for arg in (*node.args[1:], *node.kwargs.values()):
|
||||
if isinstance(arg, torch.dtype) and arg.is_floating_point:
|
||||
return arg
|
||||
return None
|
||||
|
||||
|
||||
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
|
||||
"""Returns the set of nodes reachable from `node` by following `key` edges."""
|
||||
seen: set[fx.Node] = set()
|
||||
stack = [node]
|
||||
while stack:
|
||||
n = stack.pop()
|
||||
if n in seen:
|
||||
continue
|
||||
seen.add(n)
|
||||
stack.extend(getattr(n, key))
|
||||
return seen
|
||||
|
||||
|
||||
def _renormalizes(topk: fx.Node) -> bool:
|
||||
"""Are the top-k weights divided by their own sum?
|
||||
|
||||
Every HF router spells renormalization as a division by a sum over the
|
||||
selection, optionally offset by an epsilon."""
|
||||
downstream = _reaches(topk, "users")
|
||||
for node in downstream:
|
||||
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
|
||||
continue
|
||||
denominator = node.args[1]
|
||||
if not isinstance(denominator, fx.Node):
|
||||
continue
|
||||
cone = _reaches(denominator, "all_input_nodes")
|
||||
if any(is_op(n, "sum") and n in downstream for n in cone):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _unwrap(node: object, moe_block: nn.Module) -> object:
|
||||
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
|
||||
while True:
|
||||
node = peel(node)
|
||||
if (
|
||||
is_op(node, "reshape")
|
||||
or is_op(node, "view")
|
||||
or (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
|
||||
)
|
||||
):
|
||||
node = node.args[0]
|
||||
else:
|
||||
return node
|
||||
|
||||
|
||||
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
|
||||
"""Is `node` a child module of `moe_block` called on just `x`?"""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and _unwrap(node.args[0], moe_block) is x
|
||||
)
|
||||
|
||||
|
||||
class SharedExpertMLP(nn.Module):
|
||||
"""Wraps an HF shared expert, applying the output gating it is paired with."""
|
||||
|
||||
def __init__(self, shared_experts: nn.Module, gate: nn.Module | None = None):
|
||||
super().__init__()
|
||||
self.shared_experts = shared_experts
|
||||
self.gate = gate
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
out = self.shared_experts(hidden_states)
|
||||
if self.gate is not None:
|
||||
out = torch.sigmoid(self.gate(hidden_states)[0]) * out
|
||||
return out
|
||||
|
||||
|
||||
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Standard MoE block forward.
|
||||
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_sequence_parallel = self.experts.moe_config.is_sequence_parallel
|
||||
if is_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
out = self.experts(hidden_states, router_logits=hidden_states)
|
||||
if is_sequence_parallel:
|
||||
out = tensor_model_parallel_all_gather(out, 0)[:num_tokens]
|
||||
return out.reshape(orig_shape)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoEBlockFuser:
|
||||
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
|
||||
|
||||
gate_name: str
|
||||
scoring_func: str | None
|
||||
"""`None` if routing stays in the Transformers gate."""
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
renormalize: bool | None = None
|
||||
|
||||
@staticmethod
|
||||
def _match_router(
|
||||
gate: nn.Module,
|
||||
) -> tuple[str, torch.dtype | None, bool | None] | None:
|
||||
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
|
||||
`softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function, the dtype the router computes in, and
|
||||
whether the top-k weights are renormalized (`None` if the trace was cut
|
||||
short before it could say)."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
graph = trace(gate)
|
||||
if graph is None:
|
||||
return None
|
||||
# The routing top-k is the last one; any earlier one scores expert groups.
|
||||
topks = [node for node in graph.nodes if is_op(node, "topk")]
|
||||
if not topks:
|
||||
return None
|
||||
topk = topks[-1]
|
||||
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
|
||||
post_topk = False
|
||||
# Find all scoring functions that reach the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
|
||||
if not scorers:
|
||||
# Find all scoring functions that are users of the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
|
||||
post_topk = True
|
||||
if len(scorers) != 1:
|
||||
return None
|
||||
scorer = scorers[0]
|
||||
logits_cone = _reaches(scorer, "all_input_nodes")
|
||||
if not any(is_op(n, "linear") for n in logits_cone):
|
||||
return None
|
||||
is_softmax = is_op(scorer, "softmax")
|
||||
# vLLM always scores every expert, so matching a router that softmaxes
|
||||
# the selection alone takes renormalizing. Otherwise look for the
|
||||
# division, reading its absence as "no" only from a complete trace.
|
||||
renormalize: bool | None
|
||||
if (post_topk and is_softmax) or _renormalizes(topk):
|
||||
renormalize = True
|
||||
else:
|
||||
renormalize = False if is_complete(graph) else None
|
||||
scoring_func = "softmax" if is_softmax else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone), renormalize
|
||||
|
||||
@staticmethod
|
||||
def _match_experts(
|
||||
graph: fx.Graph, experts_name: str, moe_block: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
|
||||
being the block's input.
|
||||
|
||||
Returns the experts call, `x` and the gate call."""
|
||||
experts = find_node(
|
||||
graph, lambda n: n.op == "call_module" and n.target == experts_name
|
||||
)
|
||||
if experts is None:
|
||||
return None
|
||||
# Bind by signature so keyword calls match too. Fail closed on any other
|
||||
# argument: the rewritten forward would drop it.
|
||||
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
|
||||
try:
|
||||
bound = forward.bind(*experts.args, **experts.kwargs)
|
||||
except TypeError:
|
||||
return None
|
||||
if len(bound.arguments) != 3:
|
||||
return None
|
||||
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
|
||||
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
|
||||
return None
|
||||
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
|
||||
return None
|
||||
gate = ids.args[0]
|
||||
if not (
|
||||
_reads_input(gate, x, moe_block)
|
||||
and ids.args == (gate, 2)
|
||||
and weights.args == (gate, 1)
|
||||
):
|
||||
return None
|
||||
return experts, x, gate
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
node: object, x: fx.Node, moe_block: nn.Module
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
|
||||
|
||||
Returns the shared expert's name and its optional gate's name."""
|
||||
if _reads_input(node, x, moe_block):
|
||||
return node.target, None
|
||||
if not (is_op(node, "mul") and len(node.args) == 2):
|
||||
return None
|
||||
for gated, shared in (node.args, node.args[::-1]):
|
||||
gated, shared = peel(gated), peel(shared)
|
||||
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
|
||||
continue
|
||||
gate = peel(gated.args[0])
|
||||
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
|
||||
moe_block.get_submodule(gate.target)
|
||||
):
|
||||
return shared.target, gate.target
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls,
|
||||
moe_block: nn.Module,
|
||||
experts_name: str,
|
||||
divides_shared_output: bool = False,
|
||||
) -> "MoEBlockFuser | None":
|
||||
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
|
||||
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
|
||||
reshapes, casts and `nn.Identity`.
|
||||
|
||||
Args:
|
||||
moe_block: The MoE block.
|
||||
experts_name: The name of the experts in `moe_block`.
|
||||
divides_shared_output: Whether vLLM routing would divide the shared
|
||||
expert's output by `routed_scaling_factor` (fp16 overflow
|
||||
protection), which Transformers decoder layers do not undo.
|
||||
|
||||
Returns:
|
||||
A fuser if more than the experts can run in vLLM, else `None`.
|
||||
|
||||
"""
|
||||
# Standard MoE block returns a single tensor.
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
if (graph := trace(moe_block)) is None:
|
||||
return None
|
||||
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
|
||||
return None
|
||||
experts, x, gate = matched
|
||||
unwrap = functools.partial(_unwrap, moe_block=moe_block)
|
||||
gate_name = gate.target
|
||||
router = cls._match_router(getattr(moe_block, gate_name))
|
||||
scoring_func, router_dtype, renormalize = router or (None, None, None)
|
||||
|
||||
def is_routed(n: object) -> bool:
|
||||
return n is experts or (
|
||||
is_op(n, "mul")
|
||||
and not isinstance(n.args[1], fx.Node)
|
||||
and unwrap(n.args[0]) is experts
|
||||
)
|
||||
|
||||
# Output: the experts' output, plus the shared expert's if any.
|
||||
routed = out = unwrap(output_value(graph))
|
||||
shared = None
|
||||
if not is_routed(out):
|
||||
if not (is_op(out, "add") and len(out.args) == 2):
|
||||
return None
|
||||
routed, shared = map(unwrap, out.args)
|
||||
if is_routed(shared):
|
||||
routed, shared = shared, routed
|
||||
if not is_routed(routed):
|
||||
return None
|
||||
shared_name = shared_gate_name = None
|
||||
if shared is not None:
|
||||
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = match
|
||||
# Fail closed: `rewrite_forward` runs only the experts and the detected
|
||||
# shared expert, so any other stateful child would be dropped.
|
||||
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
if scoring_func is not None and shared_name and divides_shared_output:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert output, "
|
||||
"which only fp16 overflow protection expects the decoder layer to "
|
||||
"compensate for. Falling back to routing in Transformers; run in "
|
||||
"bfloat16 to fuse it.",
|
||||
type(moe_block).__name__,
|
||||
)
|
||||
scoring_func = router_dtype = renormalize = None
|
||||
# Without vLLM routing, only a shared expert can move to vLLM, and only
|
||||
# if the block does not scale the experts' output (vLLM routing does).
|
||||
if scoring_func is None and (shared_name is None or routed is not experts):
|
||||
return None
|
||||
return cls(
|
||||
gate_name,
|
||||
scoring_func,
|
||||
shared_name,
|
||||
shared_gate_name,
|
||||
router_dtype,
|
||||
renormalize,
|
||||
)
|
||||
|
||||
def gate(
|
||||
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
|
||||
) -> GateLinear:
|
||||
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
|
||||
hf_gate = getattr(moe_block, self.gate_name)
|
||||
num_experts, hidden_size = hf_gate.weight.shape
|
||||
gate = GateLinear(
|
||||
hidden_size,
|
||||
num_experts,
|
||||
bias=False,
|
||||
out_dtype=out_dtype or self.router_dtype,
|
||||
prefix=maybe_prefix(prefix, self.gate_name),
|
||||
)
|
||||
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
|
||||
gate.register_buffer("e_score_correction_bias", bias)
|
||||
setattr(moe_block, self.gate_name, gate)
|
||||
return gate
|
||||
|
||||
def shared_experts(
|
||||
self, moe_block: nn.Module, prefix: str
|
||||
) -> SharedExpertMLP | None:
|
||||
"""Build the HF shared expert (and its optional gate)
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE."""
|
||||
if self.shared_name is None:
|
||||
return None
|
||||
shared_experts = getattr(moe_block, self.shared_name)
|
||||
gate = None
|
||||
if self.shared_gate_name is not None:
|
||||
hf_gate = getattr(moe_block, self.shared_gate_name)
|
||||
gate = ReplicatedLinear(
|
||||
hf_gate.in_features,
|
||||
hf_gate.out_features,
|
||||
bias=hf_gate.bias is not None,
|
||||
prefix=maybe_prefix(prefix, self.shared_gate_name),
|
||||
)
|
||||
setattr(moe_block, self.shared_gate_name, gate)
|
||||
return SharedExpertMLP(shared_experts, gate)
|
||||
|
||||
def rewrite_forward(self, moe_block: nn.Module) -> None:
|
||||
"""Rewrite `moe_block.forward` to route through vLLM's fused MoE."""
|
||||
moe_block.forward = types.MethodType(_moe_block_forward, moe_block)
|
||||
@@ -1,164 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
RewriteFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
is_method,
|
||||
recover_forward,
|
||||
returned_linear,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedQKVFuser(RewriteFuser):
|
||||
"""Fuser for attention with q, k and v packed into one projection."""
|
||||
|
||||
qkv_name: str
|
||||
o_name: str | None
|
||||
q_size: int
|
||||
kv_size: int
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
return (
|
||||
f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _packed_sizes(node: fx.Node) -> tuple[int, int] | None:
|
||||
"""`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one."""
|
||||
if not is_method(node, "split") or len(node.args) < 2:
|
||||
return None
|
||||
sizes = node.args[1]
|
||||
if not isinstance(sizes, (tuple, list)) or len(sizes) != 3:
|
||||
return None
|
||||
if not all(isinstance(size, int) for size in sizes):
|
||||
return None
|
||||
q_size, k_size, v_size = sizes
|
||||
if k_size != v_size or q_size < k_size:
|
||||
return None
|
||||
return q_size, k_size
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None":
|
||||
for node in graph.nodes:
|
||||
if (sizes := cls._packed_sizes(node)) is None:
|
||||
continue
|
||||
q_size, kv_size = sizes
|
||||
qkv_node = upstream_linear(node.args[0], module)
|
||||
if qkv_node is None:
|
||||
continue
|
||||
qkv_name = str(qkv_node.target)
|
||||
# The split must consume the whole projection.
|
||||
if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size:
|
||||
continue
|
||||
# o_proj produces the module's output and consumes the query width.
|
||||
o_name = returned_linear(graph, module)
|
||||
if o_name == qkv_name or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features != q_size
|
||||
):
|
||||
o_name = None
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
qkv_name=qkv_name,
|
||||
o_name=o_name,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
)
|
||||
return None
|
||||
|
||||
def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call:
|
||||
"""The unique `self.<qkv_name>(...)....split((a, b, c), ...)` call."""
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "split"
|
||||
and node.args
|
||||
and isinstance(node.args[0], (ast.Tuple, ast.List))
|
||||
and len(node.args[0].elts) == 3
|
||||
and any(
|
||||
isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name
|
||||
for inner in ast.walk(node.func.value)
|
||||
)
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits")
|
||||
return calls[0]
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite the split sizes to the sharded projection's per-rank widths."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
split = self._split_call(funcdef)
|
||||
# (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes]
|
||||
sections = local_output_sizes(self.qkv_name)
|
||||
split.args[0] = ast.parse(sections, mode="eval").body
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible with a head-sharded packed GEMM."""
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
compatible = (
|
||||
self.q_size % head_size == 0
|
||||
and self.kv_size % head_size == 0
|
||||
and qkv.out_features == self.q_size + 2 * self.kv_size
|
||||
)
|
||||
if not compatible:
|
||||
logger.debug("%s is not compatible with packed QKV fusion", type(module))
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
merged = QKVParallelLinear(
|
||||
hidden_size=qkv.in_features,
|
||||
head_size=head_size,
|
||||
total_num_heads=self.q_size // head_size,
|
||||
total_num_kv_heads=self.kv_size // head_size,
|
||||
bias=qkv.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=qkv_prefix,
|
||||
return_bias=False,
|
||||
)
|
||||
setattr(module, self.qkv_name, merged)
|
||||
log_replacement(qkv_prefix, qkv, merged)
|
||||
# If there is an output projection, we know it must be rowwise.
|
||||
if self.o_name is not None:
|
||||
o_prefix = maybe_prefix(prefix, self.o_name)
|
||||
o_proj = module.get_submodule(self.o_name)
|
||||
new_o = replace_linear_class(
|
||||
o_proj, "rowwise", quant_config, prefix=o_prefix
|
||||
)
|
||||
setattr(module, self.o_name, new_o)
|
||||
log_replacement(o_prefix, o_proj, new_o)
|
||||
@@ -1,209 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""QKV projection fuser: `q(x), k(x), v(x)` -> a fused qkv linear + split."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
fused_head_size,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
block_chain,
|
||||
compile_forward,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
returned_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QKVFuser(StackedFuser):
|
||||
"""Fuser for the attention QKV pattern `q(x), k(x), v(x)`."""
|
||||
|
||||
q_name: str
|
||||
k_name: str
|
||||
v_name: str
|
||||
o_name: str | None
|
||||
merged_name: ClassVar[str] = "qkv_proj"
|
||||
merged_cls_name: ClassVar[str] = "QKVParallelLinear"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
return [(self.q_name, "q"), (self.k_name, "k"), (self.v_name, "v")]
|
||||
|
||||
@classmethod
|
||||
def _get_qkv_nodes(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Search `graph` for the QKV pattern `q(x), k(x), v(x)`."""
|
||||
by_input: dict[fx.Node, list[fx.Node]] = {}
|
||||
for node in graph.nodes:
|
||||
if (
|
||||
is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
by_input.setdefault(node.args[0], []).append(node)
|
||||
triples = [nodes for nodes in by_input.values() if len(nodes) == 3]
|
||||
if len(triples) != 1:
|
||||
return None
|
||||
|
||||
q_node, k_node, v_node = nodes = triples[0]
|
||||
outs = [module.get_submodule(node.target).out_features for node in nodes]
|
||||
if len(set(outs)) == 2:
|
||||
# q is identified as the larger projection (GQA)
|
||||
(q_node,) = (n for n, out in zip(nodes, outs) if outs.count(out) == 1)
|
||||
k_node, v_node = (n for n, out in zip(nodes, outs) if outs.count(out) == 2)
|
||||
if module.get_submodule(q_node.target).out_features != max(outs):
|
||||
return None
|
||||
elif len(set(outs)) != 1:
|
||||
return None
|
||||
return q_node, k_node, v_node
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "QKVFuser | None":
|
||||
if (qkv_nodes := cls._get_qkv_nodes(graph, module)) is None:
|
||||
return None
|
||||
q, k, v = qkv_nodes
|
||||
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
|
||||
# o_proj produces the module's output.
|
||||
o_name = returned_linear(graph, module)
|
||||
# o_proj must be compatible with the q/k/v projections.
|
||||
if o_name in names.values() or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features
|
||||
!= module.get_submodule(q.target).out_features
|
||||
):
|
||||
o_name = None
|
||||
names["o_name"] = o_name
|
||||
return cls(source_cls=type(module).__name__, **names)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source.
|
||||
|
||||
A projection may be guarded by an existence check -- a
|
||||
`self.<proj> is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare
|
||||
truthiness test -- which is folded to its constant value (see
|
||||
`bypass_existence_guard`). The calls may sit in different branches, so the
|
||||
fused GEMM is inserted before the earliest of them, in the innermost block
|
||||
that dominates all three.
|
||||
"""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
proj_names = (self.q_name, self.k_name, self.v_name)
|
||||
calls = self._unguarded_calls(funcdef, proj_names)
|
||||
arg_dumps = {ast.dump(call.args[0]) for call in calls}
|
||||
if len(arg_dumps) != 1:
|
||||
raise ValueError("projection inputs are written differently")
|
||||
# The trace may be partial, so prove projection exclusivity in source:
|
||||
# no other linear child may consume the same input (else the matched
|
||||
# three may not be q, k and v)
|
||||
other_linears = {
|
||||
name
|
||||
for name, child in module.named_children()
|
||||
if isinstance(child, nn.Linear)
|
||||
} - set(proj_names)
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr in other_linears
|
||||
and any(ast.dump(arg) in arg_dumps for arg in node.args)
|
||||
):
|
||||
raise ValueError("another linear consumes the same input")
|
||||
|
||||
# Insert the fused GEMM before the earliest call, in the innermost block
|
||||
# common to all three (the calls may be split across branches).
|
||||
chains = [block_chain(funcdef.body, call) for call in calls]
|
||||
if any(not chain for chain in chains):
|
||||
raise ValueError("projection calls not found in the function body")
|
||||
depth = 0
|
||||
for level in zip(*chains):
|
||||
if len({id(block) for block, _ in level}) != 1:
|
||||
break
|
||||
depth += 1
|
||||
block = chains[0][depth - 1][0]
|
||||
indices = [chain[depth - 1][1] for chain in chains]
|
||||
insert_index = min(indices)
|
||||
self._check_input_stable(funcdef, module, calls, block, indices)
|
||||
|
||||
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
|
||||
self._splice_merged_split(funcdef, calls, block, insert_index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
v = module.get_submodule(self.v_name)
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
compatible = (
|
||||
q.in_features == k.in_features == v.in_features
|
||||
and len({proj.bias is None for proj in (q, k, v)}) == 1
|
||||
and k.out_features == v.out_features
|
||||
and q.out_features % head_size == 0
|
||||
and k.out_features % head_size == 0
|
||||
)
|
||||
if not compatible:
|
||||
logger.debug("%s is not compatible with QKV fusion", type(module))
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
merged = QKVParallelLinear(
|
||||
hidden_size=q.in_features,
|
||||
head_size=head_size,
|
||||
total_num_heads=q.out_features // head_size,
|
||||
total_num_kv_heads=k.out_features // head_size,
|
||||
bias=q.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
)
|
||||
logger.debug(
|
||||
"%s: %s, %s: %s, %s: %s -> %s: %s",
|
||||
self.q_name,
|
||||
q,
|
||||
self.k_name,
|
||||
k,
|
||||
self.v_name,
|
||||
module.get_submodule(self.v_name),
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
# Drop the consumed submodules so their (meta) params are not expected.
|
||||
for name in (self.q_name, self.k_name, self.v_name):
|
||||
delattr(module, name)
|
||||
# If there is an output projection, we know it must be rowwise.
|
||||
if self.o_name is not None:
|
||||
o_proj_prefix = maybe_prefix(prefix, self.o_name)
|
||||
o_proj = module.get_submodule(self.o_name)
|
||||
new_o = replace_linear_class(
|
||||
o_proj, "rowwise", quant_config, prefix=o_proj_prefix
|
||||
)
|
||||
setattr(module, self.o_name, new_o)
|
||||
log_replacement(o_proj_prefix, o_proj, new_o)
|
||||
@@ -1,297 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RMSNorm fuser: detect the norm structurally and swap in vLLM's fused RMSNorm."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.distributed.parallel_state import model_parallel_is_initialized
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
forward_input_count,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import (
|
||||
GemmaRMSNorm,
|
||||
RMSNorm,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _operand(node: fx.Node, index: int, name: str) -> object | None:
|
||||
"""Operand `index` of `node`, whether it was passed positionally or as `name`."""
|
||||
if len(node.args) > index:
|
||||
return node.args[index]
|
||||
return node.kwargs.get(name)
|
||||
|
||||
|
||||
def _is_squared(node: object, x: fx.Node) -> bool:
|
||||
"""`x**2`, `x.square()` or `x * x`, through any dtype casts."""
|
||||
node = peel(node)
|
||||
if is_op(node, "pow"):
|
||||
return (
|
||||
peel(_operand(node, 0, "input")) is x and _operand(node, 1, "exponent") == 2
|
||||
)
|
||||
if is_op(node, "square"):
|
||||
return peel(_operand(node, 0, "input")) is x
|
||||
if is_op(node, "mul"):
|
||||
return (
|
||||
peel(_operand(node, 0, "input")) is x
|
||||
and peel(_operand(node, 1, "other")) is x
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _is_inverse_sqrt(node: object) -> bool:
|
||||
"""Detect `rsqrt(v)`, or the `pow(v, -0.5)` / `v ** -0.5` spelling of it."""
|
||||
if is_op(node, "rsqrt"):
|
||||
return True
|
||||
if is_op(node, "pow"):
|
||||
return _operand(node, 1, "exponent") == -0.5
|
||||
return False
|
||||
|
||||
|
||||
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
|
||||
"""`eps` from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
|
||||
add = peel(_operand(rsqrt, 0, "input"))
|
||||
if not is_op(add, "add"):
|
||||
return None
|
||||
operands = [_operand(add, 0, "input"), _operand(add, 1, "other")]
|
||||
consts = [a for a in operands if isinstance(a, (int, float))]
|
||||
nodes = [a for a in operands if isinstance(a, fx.Node)]
|
||||
if len(consts) != 1 or len(nodes) != 1:
|
||||
return None
|
||||
mean = peel(nodes[0])
|
||||
if not is_op(mean, "mean"):
|
||||
return None
|
||||
if not _is_squared(_operand(mean, 0, "input"), x):
|
||||
return None
|
||||
return float(consts[0])
|
||||
|
||||
|
||||
def _is_one_plus(node: object) -> bool:
|
||||
"""`1 + weight` in either operand order (marks a zero-centered weight)."""
|
||||
node = peel(node)
|
||||
if not is_op(node, "add"):
|
||||
return False
|
||||
return any(
|
||||
isinstance(a, (int, float)) and a == 1
|
||||
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
|
||||
)
|
||||
|
||||
|
||||
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
|
||||
"""Does the forward compute anything after `node` before returning?"""
|
||||
value = output_value(graph)
|
||||
return value is not None and peel(value) is not node
|
||||
|
||||
|
||||
class TPAwareNormMixin(nn.Module):
|
||||
"""Mixin for RMSNorms that reconstructs a TP-sharded input before normalizing."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if model_parallel_is_initialized():
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
else:
|
||||
self.tp_size, self.tp_rank = 1, 0
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor | None = None
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.tp_size > 1 and x.shape[-1] < (full := self.weight.shape[0]):
|
||||
if x.shape[-1] * self.tp_size != full:
|
||||
raise ValueError(
|
||||
f"Cannot gather norm of width {full}: a TP-sharded input of "
|
||||
f"width {x.shape[-1]} does not tile it evenly across "
|
||||
f"{self.tp_size} ranks (replicated or uneven sharding)."
|
||||
)
|
||||
x = tensor_model_parallel_all_gather(x.contiguous())
|
||||
x = super().forward(x)
|
||||
splits = split_tensor_along_last_dim(x, num_partitions=self.tp_size)
|
||||
return splits[self.tp_rank]
|
||||
return super().forward(x, residual)
|
||||
|
||||
|
||||
class TPAwareRMSNorm(TPAwareNormMixin, RMSNorm):
|
||||
"""`RMSNorm` that reconstructs a TP-sharded input before normalizing."""
|
||||
|
||||
|
||||
class TPAwareGemmaRMSNorm(TPAwareNormMixin, GemmaRMSNorm):
|
||||
"""`GemmaRMSNorm` that reconstructs a TP-sharded input before normalizing."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RMSNormFuser(BaseFuser):
|
||||
"""Fuser for RMSNorm patterns, including Gemma-style zero-centered weights."""
|
||||
|
||||
zero_centered: bool
|
||||
"""Gemma-style `(1 + weight)` scaling (weight initialised at zero)."""
|
||||
source_cls: str
|
||||
"""Class name of the norm this was matched from (for logging)."""
|
||||
eps_attr: str | None = None
|
||||
"""Attribute holding `eps`, read per instance in `fuse`."""
|
||||
eps: float | None = None
|
||||
"""`eps` itself, when it is not held in an attribute (see `_eps_source`)."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm"
|
||||
return f"Fused: {name} ({self.source_cls}) -> {norm} (CustomOp)"
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "RMSNormFuser | None":
|
||||
"""Match a graph to the RMSNorm pattern, returning a fuser if found."""
|
||||
if forward_input_count(type(module)) != 1:
|
||||
return None
|
||||
x = find_node(graph, lambda n: n.op == "placeholder")
|
||||
if x is None:
|
||||
return None
|
||||
# Handle native torch `rms_norm` op.
|
||||
rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm"))
|
||||
if rms_norm is not None and peel(_operand(rms_norm, 0, "input")) is x:
|
||||
if _has_trailing_compute(graph, rms_norm):
|
||||
return None
|
||||
eps_attr, eps = cls._eps_source(graph, module)
|
||||
return cls(
|
||||
zero_centered=False,
|
||||
source_cls=type(module).__name__,
|
||||
eps_attr=eps_attr,
|
||||
eps=eps,
|
||||
)
|
||||
# Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern.
|
||||
# The rsqrt over the mean-square variance is the spine of the norm.
|
||||
rsqrt = None
|
||||
for node in graph.nodes:
|
||||
if _is_inverse_sqrt(node) and _variance_eps(node, x) is not None:
|
||||
rsqrt = node
|
||||
break
|
||||
if rsqrt is None:
|
||||
return None
|
||||
# The `x * rsqrt(...)` normalize multiply.
|
||||
normalize = find_node(
|
||||
graph,
|
||||
lambda n: is_op(n, "mul")
|
||||
and rsqrt in map(peel, (_operand(n, 0, "input"), _operand(n, 1, "other"))),
|
||||
)
|
||||
if normalize is None:
|
||||
return None
|
||||
# An optional later `weight * normalized` (or `(1 + weight) * normalized`).
|
||||
tail, zero_centered = normalize, False
|
||||
for node in graph.nodes:
|
||||
if not is_op(node, "mul") or node is normalize:
|
||||
continue
|
||||
operands = [
|
||||
peel(a)
|
||||
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
|
||||
if isinstance(a, fx.Node)
|
||||
]
|
||||
if len(operands) == 2 and normalize in operands:
|
||||
weight = next(o for o in operands if o is not normalize)
|
||||
tail, zero_centered = node, _is_one_plus(weight)
|
||||
break
|
||||
# The norm must be the last compute in forward, or it is not a pure norm.
|
||||
if _has_trailing_compute(graph, tail):
|
||||
return None
|
||||
eps_attr, eps = cls._eps_source(graph, module)
|
||||
return cls(
|
||||
zero_centered=zero_centered,
|
||||
source_cls=type(module).__name__,
|
||||
eps_attr=eps_attr,
|
||||
eps=eps,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _eps_source(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[str | None, float | None]:
|
||||
"""Where `fuse` should read `eps` from, resolved once per class."""
|
||||
eps = cls._eps_from_graph(graph)
|
||||
if eps is None:
|
||||
return None, None
|
||||
# Whatever supplied the constant must still equal it.
|
||||
candidates = {
|
||||
name: value
|
||||
for name, value in vars(module).items()
|
||||
if isinstance(value, float) and value == eps
|
||||
}
|
||||
# Use unique markers and retrace to verify exactly which attribute is eps.
|
||||
markers = {float(-index - 1): name for index, name in enumerate(candidates)}
|
||||
marked = None
|
||||
if markers:
|
||||
try:
|
||||
for marker, name in markers.items():
|
||||
setattr(module, name, marker)
|
||||
if (remarked := trace(module)) is not None:
|
||||
marked = cls._eps_from_graph(remarked)
|
||||
finally:
|
||||
for name, value in candidates.items():
|
||||
setattr(module, name, value)
|
||||
if marked is not None and (name := markers.get(marked)) is not None:
|
||||
return name, None
|
||||
logger.debug_once(
|
||||
"%s does not hold its eps (%s) in an attribute. Every instance in this "
|
||||
"model will use the value traced from this instance. If this is not "
|
||||
"desired, consider storing and reading eps using attribute of %s.",
|
||||
type(module).__name__,
|
||||
eps,
|
||||
type(module).__name__,
|
||||
)
|
||||
return None, eps
|
||||
|
||||
@staticmethod
|
||||
def _eps_from_graph(graph: fx.Graph) -> float | None:
|
||||
"""Extract the `eps` constant from the graph, if present."""
|
||||
if (x := find_node(graph, lambda n: n.op == "placeholder")) is None:
|
||||
return None
|
||||
fused = find_node(graph, lambda n: is_op(n, "rms_norm"))
|
||||
if fused is not None and peel(_operand(fused, 0, "input")) is x:
|
||||
args, kwargs = fused.args, fused.kwargs
|
||||
eps = args[3] if len(args) > 3 else kwargs.get("eps")
|
||||
return float(eps) if isinstance(eps, (int, float)) else None
|
||||
for node in graph.nodes:
|
||||
if _is_inverse_sqrt(node) and (eps := _variance_eps(node, x)) is not None:
|
||||
return eps
|
||||
return None
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
return True
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
|
||||
weight = getattr(module, "weight", None)
|
||||
hidden_size = weight.size(0) if weight is not None else 0
|
||||
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
|
||||
if not isinstance(eps, (int, float)):
|
||||
# If eps was not detected, match torch behaviour.
|
||||
config_dtype = vllm_config.model_config.dtype
|
||||
dtype = weight.dtype if weight is not None else config_dtype
|
||||
eps = torch.finfo(dtype).eps
|
||||
if self.zero_centered:
|
||||
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
|
||||
return TPAwareRMSNorm(
|
||||
hidden_size=hidden_size,
|
||||
eps=eps,
|
||||
has_weight=weight is not None,
|
||||
dtype=weight.dtype if weight is not None else None,
|
||||
)
|
||||
@@ -1,818 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""fx tracing and forward-source rewriting for the Transformers backend fusers.
|
||||
|
||||
A small engine, independent of any particular pattern: trace a module's forward
|
||||
with `torch.fx` (tolerating a partial graph), inspect the resulting nodes, and
|
||||
rewrite the forward's *source* (AST) so only matched calls change while the rest
|
||||
stays live Python. `fusion.py` builds the concrete fusion patterns on top.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import inspect
|
||||
import operator
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from itertools import chain
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from torch.nn import functional as F
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_UNKNOWN: Any = object()
|
||||
"""Sentinel meta value for proxies whose concrete value could not be inferred.
|
||||
Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`)."""
|
||||
|
||||
_MODULE_CALL = nn.Module.__call__
|
||||
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
|
||||
`call_module` nodes; meta execution must call modules for real."""
|
||||
|
||||
_MODULE_GETATTR = nn.Module.__getattr__
|
||||
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
|
||||
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
|
||||
recurse until the stack runs out, truncating the graph."""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _unpatched_modules():
|
||||
"""Restores the module hooks fx patches, so meta execution records nothing."""
|
||||
with (
|
||||
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
|
||||
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
|
||||
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
|
||||
return isinstance(node, fx.Node) and node.meta.get("leaf_call", False)
|
||||
|
||||
|
||||
def _reference_weight(module: nn.Module) -> torch.Tensor | None:
|
||||
"""A weight whose trailing dim is the module's hidden size.
|
||||
|
||||
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
|
||||
for child in module.modules():
|
||||
if isinstance(child, nn.Linear):
|
||||
return child.weight
|
||||
for param in module.parameters():
|
||||
if param.ndim in (1, 2):
|
||||
return param
|
||||
return None
|
||||
|
||||
|
||||
class _MetaProxy(fx.Proxy):
|
||||
"""Proxy carrying the meta-tensor value of the traced expression.
|
||||
|
||||
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
|
||||
executing each op on the meta values, so PyTorch's meta kernels are the
|
||||
single source of shape inference — no per-op rules."""
|
||||
|
||||
meta: Any = _UNKNOWN
|
||||
|
||||
def __len__(self) -> int:
|
||||
if self.meta is not _UNKNOWN:
|
||||
return len(self.meta)
|
||||
return super().__len__() # type: ignore[misc]
|
||||
|
||||
def __getattr__(self, k: str) -> "_MetaAttribute":
|
||||
return _MetaAttribute(self, k)
|
||||
|
||||
def __setitem__(self, key: object, value: object) -> None:
|
||||
# Record `obj[key] = value` so tracing continues past in-place writes
|
||||
# into containers (e.g. Gemma 4's `shared_kv_states[...] = ...`).
|
||||
self.tracer.create_proxy(
|
||||
"call_function", operator.setitem, (self, key, value), {}
|
||||
)
|
||||
|
||||
|
||||
class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
|
||||
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
|
||||
|
||||
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too."""
|
||||
|
||||
def __init__(self, root: fx.Proxy, attr: str):
|
||||
super().__init__(root, attr)
|
||||
root_meta = getattr(root, "meta", _UNKNOWN)
|
||||
if root_meta is not _UNKNOWN:
|
||||
with contextlib.suppress(Exception):
|
||||
self.meta = getattr(root_meta, attr)
|
||||
|
||||
|
||||
class _AllLeafTracer(fx.Tracer):
|
||||
"""Tracer that treats every submodule as a leaf.
|
||||
|
||||
Each child stays one `call_module` node, so matching sees the module's own
|
||||
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
|
||||
Every traced op is also executed on meta tensors (see `_MetaProxy`) so
|
||||
shape unpacks and `*`-splats trace through; anything else untraceable ends
|
||||
the trace early and the partial graph is matched.
|
||||
"""
|
||||
|
||||
varkw: str | None = None
|
||||
"""Name of the traced forward's `**kwargs` parameter, if any."""
|
||||
|
||||
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
|
||||
return True
|
||||
|
||||
def proxy(self, node: fx.Node) -> fx.Proxy:
|
||||
return _MetaProxy(node, self)
|
||||
|
||||
def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs):
|
||||
proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs)
|
||||
if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN:
|
||||
# A failure stays _UNKNOWN; only fatal if a shape question is asked.
|
||||
with contextlib.suppress(Exception):
|
||||
proxy.meta = self._infer_meta(kind, target, args, kwargs)
|
||||
return proxy
|
||||
|
||||
def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict):
|
||||
"""Execute the op on meta tensors; PyTorch infers the output value."""
|
||||
if kind == "placeholder":
|
||||
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states.
|
||||
weight = _reference_weight(self.root)
|
||||
if str(target) == "hidden_states" and weight is not None:
|
||||
return torch.empty(
|
||||
1, 8, weight.shape[-1], dtype=weight.dtype, device="meta"
|
||||
)
|
||||
return _UNKNOWN
|
||||
if kind == "get_attr":
|
||||
with _unpatched_modules():
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = torch.empty_like(value, device="meta")
|
||||
return value
|
||||
unknown = False
|
||||
|
||||
def meta_of(arg: object) -> object:
|
||||
nonlocal unknown
|
||||
if isinstance(arg, fx.Proxy):
|
||||
meta = getattr(arg, "meta", _UNKNOWN)
|
||||
unknown = unknown or meta is _UNKNOWN
|
||||
return meta
|
||||
return arg
|
||||
|
||||
meta_args = fx.node.map_aggregate(args, meta_of)
|
||||
meta_kwargs = fx.node.map_aggregate(kwargs, meta_of)
|
||||
if unknown:
|
||||
return _UNKNOWN
|
||||
if kind == "call_function":
|
||||
assert callable(target)
|
||||
return target(*meta_args, **meta_kwargs)
|
||||
if kind == "call_method":
|
||||
receiver, *rest = meta_args
|
||||
return getattr(receiver, str(target))(*rest, **meta_kwargs)
|
||||
if kind == "call_module":
|
||||
# Run the child's forward with all its state on "meta", without
|
||||
# mutating it (at match time params may be meta but buffers real).
|
||||
child = self.root.get_submodule(str(target))
|
||||
state = {
|
||||
name: torch.empty_like(tensor, device="meta")
|
||||
for name, tensor in chain(
|
||||
child.named_parameters(), child.named_buffers()
|
||||
)
|
||||
}
|
||||
with _unpatched_modules():
|
||||
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
|
||||
return _UNKNOWN
|
||||
|
||||
def _is_varkw(self, node: object) -> bool:
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "placeholder"
|
||||
and str(node.target).lstrip("*") == self.varkw
|
||||
)
|
||||
|
||||
def iter(self, obj: fx.Proxy):
|
||||
# Assume kwargs is always empty to simplify tracing.
|
||||
node = obj.node
|
||||
if self._is_varkw(node) or (
|
||||
node.op == "call_method"
|
||||
and node.target == "keys"
|
||||
and self._is_varkw(node.args[0])
|
||||
):
|
||||
return iter(())
|
||||
meta = getattr(obj, "meta", _UNKNOWN)
|
||||
if meta is _UNKNOWN:
|
||||
return super().iter(obj)
|
||||
return iter([obj[i] for i in range(len(meta))])
|
||||
|
||||
|
||||
def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable:
|
||||
"""Wrap any callable so tracing records it as one opaque `call_function` node.
|
||||
|
||||
Lets the trace continue past untraceable bodies. Only the proxy arguments carry into
|
||||
the node's dataflow; the rest are dropped rather than lifted into the graph.
|
||||
`length` declares how many values the callable returns, so unpacking its result also
|
||||
traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough.
|
||||
"""
|
||||
|
||||
def leaf(*args, **kwargs):
|
||||
proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy))
|
||||
if not proxies:
|
||||
return fn(*args, **kwargs)
|
||||
proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {})
|
||||
proxy.node.meta["leaf_call"] = True
|
||||
if length is not None:
|
||||
# The body never executes, so fabricate a value of the declared length.
|
||||
proxy.meta = (_UNKNOWN,) * length
|
||||
return proxy
|
||||
|
||||
return leaf
|
||||
|
||||
|
||||
def _leaf_attention_interfaces():
|
||||
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
|
||||
|
||||
`vllm_attention_function` needs runtime context so it is untraceable.
|
||||
Every interface returns `(attn_output, attn_weights)`."""
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
original = AttentionInterface.get_interface
|
||||
|
||||
def get_interface(self, *args, **kwargs):
|
||||
return _as_leaf_call(original(self, *args, **kwargs), length=2)
|
||||
|
||||
return mock.patch.object(AttentionInterface, "get_interface", get_interface)
|
||||
|
||||
|
||||
def trace(module: nn.Module) -> fx.Graph | None:
|
||||
"""Trace `module.forward`, returning the partial graph on failure."""
|
||||
parameters = forward_parameters(type(module))
|
||||
# vLLM never passes `past_key_values` so it is always the default value of `None`.
|
||||
# Make this concrete to simplify tracing.
|
||||
concrete_args = None
|
||||
if "past_key_values" in parameters:
|
||||
concrete_args = {"past_key_values": None}
|
||||
# Get the name of the kwargs parameter passed to module.forward (usually "kwargs")
|
||||
tracer = _AllLeafTracer()
|
||||
tracer.varkw = next(
|
||||
(
|
||||
p.name
|
||||
for p in parameters.values()
|
||||
if p.kind is inspect.Parameter.VAR_KEYWORD
|
||||
),
|
||||
None,
|
||||
)
|
||||
try:
|
||||
with _leaf_attention_interfaces():
|
||||
return tracer.trace(module, concrete_args=concrete_args)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not fully trace %s: %s", type(module), exc)
|
||||
return getattr(tracer, "graph", None)
|
||||
|
||||
|
||||
def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
|
||||
"""Parse the source of `cls.forward`, ready for rewriting."""
|
||||
fn = inspect.unwrap(cls.forward)
|
||||
if fn.__code__.co_freevars:
|
||||
raise ValueError("forward is a closure")
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
|
||||
funcdef = tree.body[0]
|
||||
if not isinstance(funcdef, ast.FunctionDef):
|
||||
raise ValueError("source is not a plain function definition")
|
||||
# `fn` is already unwrapped; don't re-apply its decorators
|
||||
funcdef.decorator_list.clear()
|
||||
# Annotations may not evaluate outside the defining module (e.g. with
|
||||
# postponed evaluation); they're not needed at runtime
|
||||
funcdef.returns = None
|
||||
args = funcdef.args
|
||||
for arg in (
|
||||
*args.posonlyargs,
|
||||
*args.args,
|
||||
*args.kwonlyargs,
|
||||
*filter(None, (args.vararg, args.kwarg)),
|
||||
):
|
||||
arg.annotation = None
|
||||
# Recompiling outside the class body would break name mangling
|
||||
for node in ast.walk(funcdef):
|
||||
name = getattr(node, "attr", None) or getattr(node, "id", None)
|
||||
if name and name.startswith("__") and not name.endswith("__"):
|
||||
raise ValueError(f"{name} would be name mangled")
|
||||
return funcdef, fn
|
||||
|
||||
|
||||
def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
|
||||
"""`cls.forward`'s signature parameters, or empty if uninspectable."""
|
||||
try:
|
||||
return dict(inspect.signature(cls.forward).parameters)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def forward_input_count(cls: type[nn.Module]) -> int:
|
||||
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
|
||||
any `*args`/`**kwargs`. Read from the signature, so it is independent of
|
||||
whether the trace completes (unlike counting placeholders)."""
|
||||
params = list(forward_parameters(cls).values())
|
||||
if not params:
|
||||
return 1 # uninspectable: assume a single input and let matching decide
|
||||
fixed = (
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
return sum(1 for p in params[1:] if p.kind in fixed)
|
||||
|
||||
|
||||
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
|
||||
"""Compile `funcdef` in `fn`'s module so tracebacks point at the source."""
|
||||
module = ast.Module(body=[funcdef], type_ignores=[])
|
||||
ast.fix_missing_locations(module)
|
||||
ast.increment_lineno(module, fn.__code__.co_firstlineno - 1)
|
||||
code = compile(module, fn.__code__.co_filename, "exec")
|
||||
namespace: dict = {}
|
||||
exec(code, fn.__globals__, namespace)
|
||||
return namespace[funcdef.name]
|
||||
|
||||
|
||||
def self_call_and_refs(
|
||||
funcdef: ast.FunctionDef, name: str
|
||||
) -> tuple[ast.Call, list[ast.Attribute]]:
|
||||
"""The unique `self.<name>(arg)` call, plus every other `self.<name>` reference.
|
||||
|
||||
Raises unless exactly one single-argument `self.<name>(arg)` call exists, so
|
||||
one fx `call_module` node maps to one syntactic call site. The remaining
|
||||
references (e.g. `is not None` guards) are returned for the caller to resolve.
|
||||
"""
|
||||
uses = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Attribute)
|
||||
and node.attr == name
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
]
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and any(node.func is use for use in uses)
|
||||
and len(node.args) == 1
|
||||
and not isinstance(node.args[0], ast.Starred)
|
||||
and not node.keywords
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError(f"{name} is not called exactly once as self.{name}(arg)")
|
||||
call = calls[0]
|
||||
other_refs = [use for use in uses if use is not call.func]
|
||||
return call, other_refs
|
||||
|
||||
|
||||
def single_self_call(funcdef: ast.FunctionDef, name: str) -> ast.Call:
|
||||
"""The unique `self.<name>(arg)` call in `funcdef`.
|
||||
|
||||
Raises unless `name` appears exactly once, as such a call, so the source
|
||||
rewrite agrees with the fx match.
|
||||
"""
|
||||
call, other_refs = self_call_and_refs(funcdef, name)
|
||||
if other_refs:
|
||||
raise ValueError(f"{name} is referenced {len(other_refs) + 1} times")
|
||||
return call
|
||||
|
||||
|
||||
def _is_none(node: ast.expr) -> bool:
|
||||
return isinstance(node, ast.Constant) and node.value is None
|
||||
|
||||
|
||||
def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
|
||||
"""Is `ref` used only for its truth value (a test, or a `not` operand)?
|
||||
|
||||
In these positions the object's identity never escapes, so a reference that
|
||||
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
|
||||
yield an operand, so the module could escape (`x and self.<name>`)."""
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
|
||||
and node.test is ref
|
||||
):
|
||||
return True
|
||||
if (
|
||||
isinstance(node, ast.UnaryOp)
|
||||
and isinstance(node.op, ast.Not)
|
||||
and node.operand is ref
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def bypass_existence_guard(
|
||||
funcdef: ast.FunctionDef, ref: ast.Attribute, name: str
|
||||
) -> None:
|
||||
"""Fold a guard on `self.<name>`'s existence to its constant value.
|
||||
|
||||
The fuser deletes `self.<name>` and binds only instances where it exists as a
|
||||
truthy `nn.Linear` (guaranteed by the match, the cache key, and `validate`),
|
||||
so a guard testing its presence is a fusion invariant. Two forms are folded:
|
||||
|
||||
- an identity `None` check `self.<name> is None` -> `False`,
|
||||
`self.<name> is not None` -> `True` (either operand order). `==`/`!=` are
|
||||
*not* folded: `is_linear` accepts `nn.Linear` subclasses, which may override
|
||||
`__eq__`/`__ne__`, so equality is not guaranteed to track identity.
|
||||
- a bare truthiness test (`if self.<name>:`, `... if self.<name> else ...`,
|
||||
`not self.<name>`): the reference itself to `True`.
|
||||
|
||||
Any other surviving reference escapes the projection's value, which no longer
|
||||
exists after fusion, so refuse rather than change semantics."""
|
||||
for node in ast.walk(funcdef):
|
||||
# `self.<name> is (not) None`, either operand order.
|
||||
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
|
||||
continue
|
||||
(op,), (right,) = node.ops, node.comparators
|
||||
if not isinstance(op, (ast.Is, ast.IsNot)):
|
||||
continue
|
||||
if (ref is node.left and _is_none(right)) or (
|
||||
ref is right and _is_none(node.left)
|
||||
):
|
||||
value = isinstance(op, ast.IsNot)
|
||||
replace_expr(funcdef, node, ast.copy_location(ast.Constant(value), node))
|
||||
return
|
||||
# A bare truthiness test (statement `if`, ternary, or `not`); an `nn.Linear`
|
||||
# is always truthy, so the reference folds to `True`.
|
||||
if _in_boolean_context(funcdef, ref):
|
||||
replace_expr(funcdef, ref, ast.copy_location(ast.Constant(True), ref))
|
||||
return
|
||||
raise ValueError(f"{name} is referenced outside an existence guard")
|
||||
|
||||
|
||||
def block_chain(
|
||||
block: list[ast.stmt], node: ast.AST
|
||||
) -> list[tuple[list[ast.stmt], int]]:
|
||||
"""Path of (statement list, index) pairs from `block` down to `node`.
|
||||
|
||||
Each pair names a nested block and the index in it of the statement
|
||||
containing `node`; the last pair is `node`'s innermost block. Empty if
|
||||
`node` is not in `block`. The longest common prefix of several nodes' chains
|
||||
is the innermost block that dominates them all.
|
||||
"""
|
||||
for index, stmt in enumerate(block):
|
||||
if not any(child is node for child in ast.walk(stmt)):
|
||||
continue
|
||||
child_blocks = [
|
||||
getattr(stmt, fld, None) for fld in ("body", "orelse", "finalbody")
|
||||
]
|
||||
child_blocks += [h.body for h in getattr(stmt, "handlers", [])]
|
||||
child_blocks += [c.body for c in getattr(stmt, "cases", [])]
|
||||
for child_block in child_blocks:
|
||||
if (
|
||||
isinstance(child_block, list)
|
||||
and child_block
|
||||
and (tail := block_chain(child_block, node))
|
||||
):
|
||||
return [(block, index), *tail]
|
||||
return [(block, index)]
|
||||
return []
|
||||
|
||||
|
||||
def _base_name(node: ast.expr) -> str | None:
|
||||
"""The root `Name` of an attribute/subscript chain (`a.b[c]` -> `a`)."""
|
||||
while isinstance(node, (ast.Attribute, ast.Subscript)):
|
||||
node = node.value
|
||||
return node.id if isinstance(node, ast.Name) else None
|
||||
|
||||
|
||||
def _rebound_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names rebound outright within `region` (`x = ...`, `del x`).
|
||||
|
||||
These are `Name` nodes in a `Store`/`Del` context."""
|
||||
return {
|
||||
node.id
|
||||
for stmt in region
|
||||
for node in ast.walk(stmt)
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del))
|
||||
}
|
||||
|
||||
|
||||
def _inplace_target(node: ast.AST) -> str | None:
|
||||
"""Base name `node` mutates in place, if any.
|
||||
|
||||
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
|
||||
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
|
||||
plain `Name` store (see `_rebound_names`)."""
|
||||
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
|
||||
node.ctx, (ast.Store, ast.Del)
|
||||
):
|
||||
return _base_name(node)
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr.endswith("_")
|
||||
and not node.func.attr.endswith("__")
|
||||
):
|
||||
return _base_name(node.func.value)
|
||||
return None
|
||||
|
||||
|
||||
def _mutated_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names mutated in place within `region` (see `_inplace_target`)."""
|
||||
return {
|
||||
base
|
||||
for stmt in region
|
||||
for node in ast.walk(stmt)
|
||||
if (base := _inplace_target(node)) is not None
|
||||
}
|
||||
|
||||
|
||||
def _call_name(call: ast.Call) -> str:
|
||||
"""The called function's own name, without its qualifier."""
|
||||
if isinstance(call.func, ast.Attribute):
|
||||
return call.func.attr
|
||||
return call.func.id if isinstance(call.func, ast.Name) else ""
|
||||
|
||||
|
||||
def _arg_names(call: ast.Call) -> set[str]:
|
||||
"""Every name `call` reads through its arguments."""
|
||||
reads = [*call.args, *(kw.value for kw in call.keywords)]
|
||||
return {
|
||||
node.id
|
||||
for arg in reads
|
||||
for node in ast.walk(arg)
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
|
||||
}
|
||||
|
||||
|
||||
def _writes_through_args(call: ast.Call) -> bool:
|
||||
"""Does `call` write through an argument rather than return a new value?
|
||||
|
||||
Torch spells this three ways: an `out=` destination, `inplace=True`, and the
|
||||
trailing-underscore convention (`torch.relu_(x)`). The underscore is checked
|
||||
on the function's own name, so it catches the free-function form that
|
||||
`_inplace_target` cannot (there the base name is the module, `torch`).
|
||||
"""
|
||||
for keyword in call.keywords:
|
||||
if keyword.arg == "out":
|
||||
return True
|
||||
if keyword.arg == "inplace" and not (
|
||||
isinstance(keyword.value, ast.Constant) and keyword.value.value is False
|
||||
):
|
||||
return True
|
||||
name = _call_name(call)
|
||||
return name.endswith("_") and not name.endswith("__")
|
||||
|
||||
|
||||
def written_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names `region` may rebind or write through.
|
||||
|
||||
Beyond plain rebinding and writes through the name itself (`_rebound_names`,
|
||||
`_mutated_names`), a call can write through an argument without naming it as
|
||||
a target: `F.relu(x, inplace=True)`, `torch.add(x, y, out=x)`,
|
||||
`torch.relu_(x)`. A statement-level call whose result is discarded is treated
|
||||
the same way, since it can only be there for a side effect.
|
||||
"""
|
||||
written = _rebound_names(region) | _mutated_names(region)
|
||||
for stmt in region:
|
||||
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
|
||||
written |= _arg_names(stmt.value)
|
||||
for node in ast.walk(stmt):
|
||||
if isinstance(node, ast.Call) and _writes_through_args(node):
|
||||
written |= _arg_names(node)
|
||||
return written
|
||||
|
||||
|
||||
_METADATA_ATTRS = frozenset(
|
||||
{"shape", "dtype", "device", "ndim", "size", "dim", "numel", "item"}
|
||||
)
|
||||
|
||||
|
||||
def _aliasing_reads(node: ast.expr) -> set[str]:
|
||||
"""Names `node` reads in a way that could yield a view of them.
|
||||
|
||||
Metadata (`x.shape`, `x.size(0)`, `x.dtype`) is ints and tuples sharing no
|
||||
storage with `x`, so reaching a name only through metadata cannot alias it.
|
||||
Shape arithmetic is pervasive between projections, so propagating through it
|
||||
would taint nearly every later name.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
stack: list[ast.AST] = [node]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if isinstance(current, ast.Attribute) and current.attr in _METADATA_ATTRS:
|
||||
continue
|
||||
if (
|
||||
isinstance(current, ast.Call)
|
||||
and isinstance(current.func, ast.Attribute)
|
||||
and current.func.attr in _METADATA_ATTRS
|
||||
):
|
||||
stack.extend(current.args)
|
||||
stack.extend(keyword.value for keyword in current.keywords)
|
||||
continue
|
||||
if isinstance(current, ast.Name) and isinstance(current.ctx, ast.Load):
|
||||
names.add(current.id)
|
||||
stack.extend(ast.iter_child_nodes(current))
|
||||
return names
|
||||
|
||||
|
||||
def _returns_fresh(node: ast.expr, module: nn.Module) -> bool:
|
||||
"""Does evaluating `node` allocate, rather than return a view of its input?
|
||||
|
||||
A `nn.Linear` writes its output into new storage, so its result aliases
|
||||
nothing that went in, and a chain of view ops on top of it aliases only that
|
||||
fresh tensor. Walking the receiver chain down to a `self.<linear>(...)` call
|
||||
therefore proves the value cannot alias the projection input. A literal
|
||||
aliases nothing either, and a branch or sequence is fresh when every part is
|
||||
-- which is how a guarded projection (`p(x) if p is not None else None`)
|
||||
stays untracked. A call on an attribute that is missing or `None` cannot run
|
||||
at all, so a branch selecting a different configuration is fresh too.
|
||||
"""
|
||||
if isinstance(node, ast.Constant):
|
||||
return True
|
||||
if isinstance(node, ast.IfExp):
|
||||
if isinstance(node.test, ast.Constant):
|
||||
# A folded existence guard leaves the untaken branch as dead code.
|
||||
taken = node.body if node.test.value else node.orelse
|
||||
return _returns_fresh(taken, module)
|
||||
return _returns_fresh(node.body, module) and _returns_fresh(node.orelse, module)
|
||||
if isinstance(node, (ast.Tuple, ast.List)):
|
||||
return all(_returns_fresh(elt, module) for elt in node.elts)
|
||||
while isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
||||
func = node.func
|
||||
if isinstance(func.value, ast.Name) and func.value.id == "self":
|
||||
attribute = getattr(module, func.attr, None)
|
||||
if attribute is None or isinstance(attribute, nn.Linear):
|
||||
# A missing or `None` attribute cannot be called, so the branch
|
||||
# selecting it never runs (`q_proj` on a q-LoRA layer).
|
||||
return True
|
||||
# Anything else can only return storage it was handed, so it is
|
||||
# fresh exactly when its inputs are (`q_norm(q_proj(x))`).
|
||||
return bool(node.args) and all(
|
||||
_returns_fresh(arg, module) for arg in node.args
|
||||
)
|
||||
node = func.value
|
||||
return False
|
||||
|
||||
|
||||
def aliasing_names(
|
||||
funcdef: ast.FunctionDef, seed: set[str], module: nn.Module
|
||||
) -> set[str]:
|
||||
"""`seed` plus every name that may alias one of them.
|
||||
|
||||
A view shares storage with its base (`h = x.view(-1)`), so mutating `h`
|
||||
mutates `x`; tracking only `seed` would miss that. Any assignment reading a
|
||||
tracked name therefore taints its targets, transitively, unless its value is
|
||||
freshly allocated (`_returns_fresh`).
|
||||
"""
|
||||
names = set(seed)
|
||||
assigns = [
|
||||
(node, node.value)
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, (ast.Assign, ast.AnnAssign))
|
||||
and node.value is not None
|
||||
and not _returns_fresh(node.value, module)
|
||||
]
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node, value in assigns:
|
||||
if not _aliasing_reads(value) & names:
|
||||
continue
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
stores = {
|
||||
child.id
|
||||
for target in targets
|
||||
for child in ast.walk(target)
|
||||
if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store)
|
||||
}
|
||||
if stores - names:
|
||||
names |= stores
|
||||
changed = True
|
||||
return names
|
||||
|
||||
|
||||
def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
|
||||
"""Replace the expression `old` (by identity) with `new` within `module`."""
|
||||
|
||||
class _Replacer(ast.NodeTransformer):
|
||||
def visit(self, node: ast.AST) -> ast.AST:
|
||||
if node is old:
|
||||
return new
|
||||
return super().generic_visit(node)
|
||||
|
||||
_Replacer().visit(module)
|
||||
|
||||
|
||||
def is_complete(graph: fx.Graph) -> bool:
|
||||
"""Did the trace run to completion? A partial graph has no `output` node.
|
||||
|
||||
Only a complete graph can be read for the *absence* of an op; in a partial
|
||||
one the tail is exactly what is missing."""
|
||||
return any(node.op == "output" for node in graph.nodes)
|
||||
|
||||
|
||||
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
|
||||
"""The first node in `graph` matching `predicate`, or `None`."""
|
||||
return next((n for n in graph.nodes if predicate(n)), None)
|
||||
|
||||
|
||||
def output_value(graph: fx.Graph) -> object | None:
|
||||
"""The value the graph's `output` node returns, if the trace reached one."""
|
||||
output = find_node(graph, lambda n: n.op == "output")
|
||||
if output is None or not output.args:
|
||||
return None
|
||||
return output.args[0]
|
||||
|
||||
|
||||
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear producing `node`, walking back through splits/reshapes.
|
||||
|
||||
Non-linear submodules are transparent too (e.g. the dropout GPT-style
|
||||
attentions apply after their output projection). Never walks through a leaf
|
||||
call (e.g. an attention interface): its inputs are what attention consumes,
|
||||
not what produced the value."""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if not isinstance(current, fx.Node) or current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
if is_linear(current, module):
|
||||
return current
|
||||
if current.op in (
|
||||
"call_function",
|
||||
"call_method",
|
||||
"call_module",
|
||||
) and not is_leaf_call(current):
|
||||
stack.extend(current.args)
|
||||
return None
|
||||
|
||||
|
||||
def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): what crosses
|
||||
it is consumed by the attention computation, not projected."""
|
||||
queue = list(node.users)
|
||||
seen: set[fx.Node] = set()
|
||||
while queue:
|
||||
current = queue.pop(0)
|
||||
if current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
if is_linear(current, module):
|
||||
return current
|
||||
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
|
||||
queue.extend(current.users)
|
||||
return None
|
||||
|
||||
|
||||
def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None:
|
||||
"""Name of the Linear producing the graph's (first) output value."""
|
||||
value = output_value(graph)
|
||||
if isinstance(value, (tuple, list)) and value:
|
||||
value = value[0]
|
||||
linear = upstream_linear(value, module)
|
||||
return None if linear is None else str(linear.target)
|
||||
|
||||
|
||||
def is_linear(node: fx.Node, module: nn.Module) -> bool:
|
||||
"""Is node `nn.Linear.__call__()`."""
|
||||
return node.op == "call_module" and isinstance(
|
||||
module.get_submodule(node.target), nn.Linear
|
||||
)
|
||||
|
||||
|
||||
_DTYPE_CASTS = frozenset({"to", "float", "double", "half", "bfloat16", "type_as"})
|
||||
|
||||
|
||||
def peel(node: object) -> object:
|
||||
"""Strip dtype-cast wrappers (`.to(...)`, `.float()`, `.type_as(...)`)."""
|
||||
while (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_method"
|
||||
and node.target in _DTYPE_CASTS
|
||||
):
|
||||
node = node.args[0]
|
||||
return node
|
||||
|
||||
|
||||
def is_fn(node: object, target: object) -> TypeIs[fx.Node]:
|
||||
"""Is node `<target>()`."""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_function"
|
||||
and node.target is target
|
||||
)
|
||||
|
||||
|
||||
def is_method(node: object, name: str) -> TypeIs[fx.Node]:
|
||||
"""Is node `.<name>()`."""
|
||||
return (
|
||||
isinstance(node, fx.Node) and node.op == "call_method" and node.target == name
|
||||
)
|
||||
|
||||
|
||||
def is_op(node: object, name: str) -> TypeIs[fx.Node]:
|
||||
"""Is node `<mod>.<name>()` for torch, F, operator, or Tensor."""
|
||||
return any(
|
||||
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
|
||||
) or (hasattr(torch.Tensor, name) and is_method(node, name))
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layer provider resolution for the Transformers modeling backend.
|
||||
|
||||
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
|
||||
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
|
||||
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
|
||||
resolved source of every symbol is logged so it is clear which layers run
|
||||
hw-agnostic and which fell back to vLLM.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
|
||||
_VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used."""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
logger.info("Using hw-agnostic layer: %s", name)
|
||||
return obj
|
||||
except (ImportError, AttributeError):
|
||||
logger.warning(
|
||||
"hw-agnostic layer %s is not available; falling back to default", name
|
||||
)
|
||||
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
|
||||
|
||||
|
||||
RMSNorm = _resolve("layernorm", "RMSNorm")
|
||||
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
|
||||
|
||||
|
||||
def get_act_and_mul_fn(act_fn_name: str):
|
||||
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
|
||||
|
||||
Resolved per call because the op is name-parameterized: an activation with
|
||||
no hw-agnostic equivalent falls back to vLLM individually.
|
||||
"""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
from vllm.model_executor.hw_agnostic.layers.activation import (
|
||||
get_act_and_mul_fn as hw_fn,
|
||||
)
|
||||
|
||||
fn = hw_fn(act_fn_name)
|
||||
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
|
||||
return fn
|
||||
except (ImportError, KeyError):
|
||||
logger.warning_once(
|
||||
"hw-agnostic activation %s is not available; falling back to vLLM",
|
||||
act_fn_name,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
|
||||
|
||||
return vllm_fn(act_fn_name)
|
||||
@@ -1,80 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for legacy models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class LegacyMixin(Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(
|
||||
# Drop unsupported/unwanted output embeddings layers.
|
||||
orig_to_new_prefix={
|
||||
"model.lm_head.": None,
|
||||
"model.predictions.": None,
|
||||
"model.qa_outputs.": None,
|
||||
"model.embeddings_project.": None,
|
||||
"model.discriminator_predictions.": None,
|
||||
},
|
||||
orig_to_new_substr={
|
||||
# Some encoder models have the position_ids buffer in the checkpoint.
|
||||
# vLLM always passes position_ids as an argument, so drop the buffer.
|
||||
"position_ids": None,
|
||||
# Some encoder models have the bias of the final classifier layer in
|
||||
# the checkpoint. vLLM does not use this bias, so drop it.
|
||||
"score.bias": None,
|
||||
},
|
||||
)
|
||||
|
||||
# roberta-like models an extra padding in positions.
|
||||
# FIXME(Isotr0py): This is quite hacky for roberta edge case,
|
||||
# we should find a better way to handle this.
|
||||
self.is_roberta = "roberta" in self.text_config.model_type
|
||||
self.padding_idx = self.text_config.pad_token_id
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if self.is_roberta:
|
||||
# RoBERTa positions start at padding_idx + 1.
|
||||
# Non-in-place add to avoid mutating the persistent GPU buffer --
|
||||
# in-place += would accumulate on CUDA graph padding slots.
|
||||
positions = positions + self.padding_idx + 1
|
||||
return super().forward(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1,454 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .base import Base
|
||||
from .utils import log_replacement, maybe_per_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformersMoEState:
|
||||
topk_ids: torch.Tensor | None = None
|
||||
is_sequence_parallel: bool = False
|
||||
|
||||
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersMoERunner(MoERunner):
|
||||
"""Custom MoERunner for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._moe_state = moe_state
|
||||
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
"""In Transformers `experts.forward` will have this signature, which is why
|
||||
it deliberately does not match `MoERunner.forward`.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
hidden_states,
|
||||
topk_ids.to(torch.int32),
|
||||
topk_weights.to(torch.float32),
|
||||
self.layer_name,
|
||||
)
|
||||
|
||||
def _forward_super(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return super().forward(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def _transformers_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
"""Store the `topk_ids` in the layer and call the actual forward."""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._moe_state.topk_ids = topk_ids
|
||||
return self._forward_super(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def _transformers_moe_forward_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="transformers_moe_forward",
|
||||
op_func=_transformers_moe_forward,
|
||||
mutates_args=["hidden_states"],
|
||||
fake_impl=_transformers_moe_forward_fake,
|
||||
tags=(torch.Tag.needs_fixed_stride_order,),
|
||||
)
|
||||
|
||||
|
||||
def _hf_topk_router_precomputed(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
|
||||
weights arrive as `gating_output`, ids via `moe_state`."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
def _hf_topk_router(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
gate: nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
|
||||
_, topk_weights, topk_ids = gate(hidden_states)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
class TransformersRoutedExperts(RoutedExperts):
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name: str | None = None,
|
||||
ckpt_down_proj_name: str | None = None,
|
||||
ckpt_up_proj_name: str | None = None,
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
common_map = super().get_expert_mapping(
|
||||
ckpt_gate_proj_name or "gate_proj",
|
||||
ckpt_down_proj_name or "down_proj",
|
||||
ckpt_up_proj_name or "up_proj",
|
||||
include_fused,
|
||||
)
|
||||
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
|
||||
if not include_fused:
|
||||
return common_map + mixtral_map
|
||||
common_fused, common_unfused = common_map[:3], common_map[3:]
|
||||
mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:]
|
||||
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
|
||||
|
||||
|
||||
class MoEMixin(MixtureOfExperts, Base):
|
||||
mlp_layers: list[nn.Module]
|
||||
"""MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods."""
|
||||
|
||||
# Redeclared because `recursive_replace` assigns it below the first read
|
||||
num_local_physical_experts: int
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
num_local_physical_experts: int,
|
||||
):
|
||||
assert self.num_local_physical_experts == num_local_physical_experts
|
||||
self.num_physical_experts = num_physical_experts
|
||||
self.num_local_physical_experts = num_local_physical_experts
|
||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||
for moe_block in self.mlp_layers:
|
||||
moe_block.n_local_physical_experts = num_local_physical_experts
|
||||
moe_block.n_physical_experts = num_physical_experts
|
||||
moe_block.n_redundant_experts = self.num_redundant_experts
|
||||
moe_block.experts.update_expert_map()
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Initialize the MoE layers."""
|
||||
experts_name = "experts"
|
||||
text_config = self.text_config
|
||||
|
||||
# Positional arguments
|
||||
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
|
||||
num_experts = getattr_iter(text_config, num_experts_names, None)
|
||||
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
|
||||
top_k = getattr_iter(text_config, top_k_names, None)
|
||||
assert top_k is not None
|
||||
hidden_size = text_config.hidden_size
|
||||
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
|
||||
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
|
||||
assert intermediate_size is not None
|
||||
|
||||
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
|
||||
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
|
||||
|
||||
# Common kwargs
|
||||
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
|
||||
|
||||
# Routed scaling factor kwargs
|
||||
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
|
||||
# aiter applies routed_scaling_factor internally
|
||||
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
|
||||
routed_scaling_factor_kwargs = dict(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scale_to_output=apply_routed_scale_to_output,
|
||||
)
|
||||
|
||||
# Dtype the router computes in, if it is not the activation dtype.
|
||||
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
|
||||
if config_router_dtype is not None:
|
||||
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
|
||||
|
||||
# Grouped topk routing kwargs
|
||||
num_expert_group = getattr(text_config, "n_group", None)
|
||||
topk_group = getattr(text_config, "topk_group", None)
|
||||
use_grouped_topk = num_expert_group is not None and topk_group is not None
|
||||
grouped_topk_routing_kwargs = dict(
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
)
|
||||
|
||||
# MoE activation function
|
||||
activation = "silu"
|
||||
wrapped_arch = self.config.architectures[0].lower()
|
||||
if "gptoss" in wrapped_arch:
|
||||
activation = "swigluoai"
|
||||
|
||||
# Expert parallel load balancing kwargs
|
||||
enable_eplb = self.parallel_config.enable_eplb
|
||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||
|
||||
# MixtureOfExperts mixin settings
|
||||
ep_size = get_ep_group().world_size
|
||||
|
||||
self.mlp_layers = []
|
||||
self.moe_layers = []
|
||||
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
|
||||
self.num_logical_experts = num_experts
|
||||
self.num_physical_experts = num_experts + num_redundant_experts
|
||||
self.num_local_physical_experts = self.num_physical_experts // ep_size
|
||||
self.num_routed_experts = num_experts
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_redundant_experts = num_redundant_experts
|
||||
|
||||
# Down projections of shared experts consumed by FusedMoE
|
||||
shared_down_projs: list[tuple[nn.Module, str]] = []
|
||||
|
||||
# Recursively fuse MoE layers
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
# Naive implementations will have experts as ModuleList
|
||||
is_modulelist = isinstance(child_module, nn.ModuleList)
|
||||
# Packed implementations will have experts as 3D tensors of shapes like:
|
||||
# gate_up_proj = (num_experts, 2 * intermediate_size, hidden_size)
|
||||
# down_proj = (num_experts, intermediate_size, hidden_size)
|
||||
params = list(child_module.parameters())
|
||||
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
|
||||
if child_name == experts_name and (is_modulelist or is_3d):
|
||||
# Alias for readability
|
||||
moe_block = module
|
||||
experts = child_module
|
||||
# Class of the fused block (parent of gate/experts/shared)
|
||||
moe_block_cls = type(moe_block).__name__
|
||||
experts_cls = type(experts).__name__
|
||||
# Do the experts have biases
|
||||
has_bias = False
|
||||
for experts_param_name, _ in experts.named_parameters():
|
||||
if "bias" in experts_param_name:
|
||||
has_bias = True
|
||||
break
|
||||
# Whether the expert weights are stored in transposed format.
|
||||
# Comes from Transformers `use_experts_implementation` decorator.
|
||||
is_transposed = getattr(experts, "is_transposed", False)
|
||||
# If the config does not specify num_shared_experts, but
|
||||
# the model has shared experts, we assume there is one.
|
||||
if self.num_shared_experts == 0:
|
||||
for moe_block_param_name, _ in moe_block.named_parameters():
|
||||
if "shared_expert" in moe_block_param_name:
|
||||
self.num_shared_experts = 1
|
||||
break
|
||||
|
||||
# Only pay for the layer index if something is per-layer;
|
||||
# `extract_layer_index` raises for unnumbered module paths.
|
||||
per_layer = (top_k, intermediate_size, norm_topk_prob)
|
||||
layer_idx = (
|
||||
extract_layer_index(qual_name)
|
||||
if any(isinstance(v, list) for v in per_layer)
|
||||
else 0
|
||||
)
|
||||
layer_top_k = maybe_per_layer(top_k, layer_idx)
|
||||
layer_renormalize = maybe_per_layer(norm_topk_prob, layer_idx)
|
||||
if layer_renormalize is None:
|
||||
layer_renormalize = layer_top_k > 1
|
||||
|
||||
kwargs: dict[str, Any] = dict(
|
||||
num_experts=num_experts,
|
||||
top_k=layer_top_k,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=maybe_per_layer(intermediate_size, layer_idx),
|
||||
renormalize=layer_renormalize,
|
||||
quant_config=self.quant_config,
|
||||
prefix=qual_name,
|
||||
activation=activation,
|
||||
enable_eplb=enable_eplb,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
is_fused_checkpoint_transposed=is_transposed,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
# _maybe_apply_routed_scale_to_output edge case. Transformers
|
||||
# decoder layers do not compensate for dividing by scaling factor.
|
||||
divides_shared_output = (
|
||||
routed_scaling_factor != 1.0
|
||||
and apply_routed_scale_to_output
|
||||
and self.model_config.dtype == torch.float16
|
||||
)
|
||||
fuser = MoEBlockFuser.match(
|
||||
moe_block, experts_name, divides_shared_output
|
||||
)
|
||||
if fuser is not None:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in MoERunner.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
hf_shared = shared_experts.shared_experts
|
||||
glu_fuser = get_fuser(hf_shared, GLUFuser)
|
||||
if glu_fuser is not None:
|
||||
down_name = glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
if scoring_func := fuser.scoring_func:
|
||||
# Internal routing kwargs.
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=scoring_func,
|
||||
renormalize=(
|
||||
layer_renormalize
|
||||
if fuser.renormalize is None
|
||||
else fuser.renormalize
|
||||
),
|
||||
gate=gate,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
else:
|
||||
# External routing kwargs
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
_hf_topk_router,
|
||||
gate=getattr(moe_block, fuser.gate_name),
|
||||
),
|
||||
)
|
||||
|
||||
kwargs |= dict(
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
shared_experts=shared_experts,
|
||||
)
|
||||
fuser.rewrite_forward(moe_block)
|
||||
logger.info_once(
|
||||
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
|
||||
" + shared experts" if fuser.shared_name else "",
|
||||
moe_block_cls,
|
||||
"external" if fuser.scoring_func is None else "internal",
|
||||
)
|
||||
else:
|
||||
# MoE block forward is unmodified.
|
||||
# gate/router and shared expert (if any) runs in Transformers.
|
||||
# We then smuggle the topk_ids in using a custom op.
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
_hf_topk_router_precomputed, moe_state=moe_state
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
logger.info_once(
|
||||
"Fused: experts (%s) -> MoERunner (external routing)",
|
||||
experts_cls,
|
||||
)
|
||||
fused_experts = FusedMoEFactory(**kwargs)
|
||||
moe_block.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
# Update MixtureOfExperts mixin state
|
||||
self.mlp_layers.append(moe_block)
|
||||
self.moe_layers.append(fused_experts)
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
# Continue with the replacement of layers in Base
|
||||
super().recursive_replace()
|
||||
# GLUFuser likely fused shared_experts. The down projection after the GLU
|
||||
# normally immediately reduces but we want FusedMoE to handle the reduction.
|
||||
for hf_shared, down_name in shared_down_projs:
|
||||
hf_shared.get_submodule(down_name).reduce_results = False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,407 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend utilities."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.transformers_utils.config import iter_rope_parameters
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Copied from `accelerate`
|
||||
@contextmanager
|
||||
def init_on_device_without_buffers(device: torch.device):
|
||||
"""A context manager under which models are initialized with all
|
||||
parameters on the specified device. However buffers are not
|
||||
initialized on specified device.
|
||||
|
||||
Args:
|
||||
device (`torch.device`):
|
||||
Device to initialize all parameters on.
|
||||
|
||||
"""
|
||||
old_register_parameter = nn.Module.register_parameter
|
||||
|
||||
def register_empty_parameter(module, name, param):
|
||||
old_register_parameter(module, name, param)
|
||||
if param is not None:
|
||||
param_cls = type(module._parameters[name])
|
||||
kwargs = module._parameters[name].__dict__
|
||||
kwargs["requires_grad"] = param.requires_grad
|
||||
module._parameters[name] = param_cls(
|
||||
module._parameters[name].to(device), **kwargs
|
||||
)
|
||||
|
||||
tensor_constructors_to_patch: dict[str, Callable] = {}
|
||||
|
||||
def patch_tensor_constructor(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
kwargs["device"] = device
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
try:
|
||||
nn.Module.register_parameter = register_empty_parameter
|
||||
for torch_function_name in tensor_constructors_to_patch:
|
||||
setattr(
|
||||
torch,
|
||||
torch_function_name,
|
||||
patch_tensor_constructor(getattr(torch, torch_function_name)),
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
nn.Module.register_parameter = old_register_parameter
|
||||
for (
|
||||
torch_function_name,
|
||||
old_torch_function,
|
||||
) in tensor_constructors_to_patch.items():
|
||||
setattr(torch, torch_function_name, old_torch_function)
|
||||
|
||||
|
||||
Style = Literal[
|
||||
"colwise",
|
||||
"rowwise",
|
||||
"replicate",
|
||||
"colwise_gather_output",
|
||||
"rowwise_split_input",
|
||||
]
|
||||
|
||||
|
||||
def replace_linear_class(
|
||||
linear: nn.Linear,
|
||||
style: Style = "replicate",
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
|
||||
"""Replace nn.Linear with one of vLLM's tensor parallel linear classes.
|
||||
|
||||
Args:
|
||||
linear: `nn.Linear` to be replaced.
|
||||
style: Tensor parallel style of the new linear, e.g. "colwise".
|
||||
quant_config: Quantization config for the new linear.
|
||||
prefix: Module prefix of the new linear, used for quantization lookup.
|
||||
|
||||
Returns:
|
||||
The new linear.
|
||||
|
||||
"""
|
||||
if not isinstance(style, str):
|
||||
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
|
||||
|
||||
vllm_linear_cls, vllm_linear_kwargs = {
|
||||
"colwise": (ColumnParallelLinear, {}),
|
||||
"rowwise": (RowParallelLinear, {}),
|
||||
"replicate": (ReplicatedLinear, {}),
|
||||
"colwise_gather_output": (ColumnParallelLinear, {"gather_output": True}),
|
||||
"rowwise_split_input": (RowParallelLinear, {"input_is_parallel": False}),
|
||||
}.get(style, (ReplicatedLinear, {}))
|
||||
|
||||
return vllm_linear_cls(
|
||||
input_size=linear.in_features,
|
||||
output_size=linear.out_features,
|
||||
bias=linear.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
return_bias=False,
|
||||
**vllm_linear_kwargs,
|
||||
)
|
||||
|
||||
|
||||
TorchConv: TypeAlias = nn.Conv2d | nn.Conv3d
|
||||
VllmConv: TypeAlias = Conv2dLayer | Conv3dLayer
|
||||
|
||||
|
||||
def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
|
||||
"""Replace a Transformers Conv2d/Conv3d with vLLM's Conv2d/Conv3d.
|
||||
|
||||
Args:
|
||||
conv: `nn.Conv2d` or `nn.Conv3d` to be replaced.
|
||||
|
||||
Returns:
|
||||
The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported,
|
||||
returns the original conv module.
|
||||
|
||||
"""
|
||||
# vLLM does not handle non-zero padding modes
|
||||
if conv.padding_mode != "zeros":
|
||||
return conv
|
||||
|
||||
vllm_conv_cls = {
|
||||
nn.Conv2d: Conv2dLayer,
|
||||
nn.Conv3d: Conv3dLayer,
|
||||
}.get(type(conv))
|
||||
|
||||
if vllm_conv_cls is None:
|
||||
return conv
|
||||
|
||||
return vllm_conv_cls(
|
||||
in_channels=conv.in_channels,
|
||||
out_channels=conv.out_channels,
|
||||
kernel_size=conv.kernel_size,
|
||||
stride=conv.stride,
|
||||
padding=conv.padding,
|
||||
dilation=conv.dilation,
|
||||
groups=conv.groups,
|
||||
bias=conv.bias is not None,
|
||||
padding_mode=conv.padding_mode,
|
||||
params_dtype=conv.weight.dtype,
|
||||
)
|
||||
|
||||
|
||||
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
|
||||
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
|
||||
`LayerNorm`.
|
||||
|
||||
Args:
|
||||
layernorm: `nn.LayerNorm` to be replaced.
|
||||
|
||||
Returns:
|
||||
The new `LayerNorm`. If the layernorm is not supported (a subclass with
|
||||
its own behavior, multi-dim `normalized_shape`, or not both
|
||||
elementwise-affine and biased), returns the original module unchanged.
|
||||
|
||||
"""
|
||||
if (
|
||||
type(layernorm) is not nn.LayerNorm
|
||||
or len(layernorm.normalized_shape) != 1
|
||||
or not layernorm.elementwise_affine
|
||||
or layernorm.bias is None
|
||||
):
|
||||
return layernorm
|
||||
|
||||
return LayerNorm(
|
||||
layernorm.normalized_shape[0],
|
||||
eps=layernorm.eps,
|
||||
dtype=layernorm.weight.dtype,
|
||||
)
|
||||
|
||||
|
||||
def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
"""Set a possibly nested attribute, like the inverse of attrgetter."""
|
||||
parent, _, name = attr.rpartition(".")
|
||||
|
||||
def setter(obj: object, value: object):
|
||||
attr_parent = attrgetter(parent)(obj) if parent else obj
|
||||
setattr(attr_parent, name, value)
|
||||
|
||||
return setter
|
||||
|
||||
|
||||
class _UninitializedEmbedding(nn.Embedding):
|
||||
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
|
||||
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
|
||||
|
||||
|
||||
_rebased_embedding_classes: dict[type[nn.Embedding], type[VocabParallelEmbedding]] = {}
|
||||
|
||||
|
||||
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
|
||||
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
|
||||
|
||||
Args:
|
||||
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
|
||||
maps to the same class.
|
||||
|
||||
Returns:
|
||||
The new class, to assign to `__class__` of an instance of `cls`.
|
||||
|
||||
"""
|
||||
if cls not in _rebased_embedding_classes:
|
||||
rebased_cls = type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
|
||||
_rebased_embedding_classes[cls] = rebased_cls
|
||||
return _rebased_embedding_classes[cls]
|
||||
|
||||
|
||||
def replace_embedding_class(
|
||||
embedding: nn.Embedding,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> nn.Module:
|
||||
"""Replace `nn.Embedding` with `VocabParallelEmbedding`.
|
||||
|
||||
Args:
|
||||
embedding: The `nn.Embedding` holding a vocab table.
|
||||
quant_config: Quantization config for the new embedding.
|
||||
prefix: Qualname of `embedding`, used to look up its quantization method.
|
||||
|
||||
Returns:
|
||||
The module to install in `embedding`'s place.
|
||||
|
||||
"""
|
||||
# Tied tables are reached more than once, so don't rebase an already new embedding
|
||||
if isinstance(embedding, VocabParallelEmbedding):
|
||||
return embedding
|
||||
|
||||
kwargs = dict(
|
||||
num_embeddings=embedding.num_embeddings,
|
||||
embedding_dim=embedding.embedding_dim,
|
||||
params_dtype=embedding.weight.dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
# If `embedding` is a bare `nn.Embedding`, simple replace
|
||||
if type(embedding) is nn.Embedding:
|
||||
return VocabParallelEmbedding(**kwargs)
|
||||
|
||||
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
|
||||
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
|
||||
VocabParallelEmbedding.__init__(embedding, **kwargs)
|
||||
return embedding
|
||||
|
||||
|
||||
def recursive_replace_linear(
|
||||
model: nn.Module,
|
||||
quant_config: "QuantizationConfig | None",
|
||||
prefix: str = "",
|
||||
):
|
||||
"""Recursively replace linear modules in the model as needed."""
|
||||
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
new_module = child_module
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
# Replace modules as needed
|
||||
if isinstance(child_module, nn.Linear):
|
||||
new_module = replace_linear_class(
|
||||
child_module,
|
||||
"replicate",
|
||||
quant_config,
|
||||
prefix=qual_name,
|
||||
)
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
if new_module is not child_module:
|
||||
setattr(module, child_name, new_module)
|
||||
|
||||
_recursive_replace(model, prefix=prefix)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def maybe_per_layer(value: T | list[T], layer_idx: int) -> T:
|
||||
"""Pick `layer_idx`'s entry from a config field that may be sized per layer."""
|
||||
return value[layer_idx] if isinstance(value, list) else value
|
||||
|
||||
|
||||
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""`module`'s own state (i.e. named parameters and buffers)."""
|
||||
return chain(module.named_parameters(), module.named_buffers())
|
||||
|
||||
|
||||
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
|
||||
logger.debug("%s: %s -> %s", name, old_module, new_module)
|
||||
|
||||
|
||||
def get_feature_request_tip(
|
||||
model: str,
|
||||
trust_remote_code: bool,
|
||||
) -> str:
|
||||
hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new"
|
||||
gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose"
|
||||
url = hf_url if trust_remote_code else gh_url
|
||||
prefix = f"Please open {url} to request support for this feature. "
|
||||
if Path(model).exists():
|
||||
prefix = ""
|
||||
doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models"
|
||||
tip = f"See {doc_url} for instructions on how to add support yourself."
|
||||
return f"{prefix}{tip}"
|
||||
|
||||
|
||||
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
|
||||
|
||||
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
|
||||
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
|
||||
if max_position_embeddings is None:
|
||||
return False
|
||||
return vllm_config.model_config.max_model_len <= max_position_embeddings
|
||||
|
||||
|
||||
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
|
||||
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
|
||||
|
||||
Defaults to `True` but is disabled in the following situations:
|
||||
|
||||
- The model uses dynamic rope scaling that can still rescale within
|
||||
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
|
||||
if "dynamic" not in rope_types:
|
||||
return True
|
||||
# Layer types using longrope still sync, even once dynamic rope is disabled
|
||||
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
|
||||
|
||||
|
||||
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
|
||||
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
|
||||
|
||||
The update syncs on `torch.max(position_ids)` every forward, which breaks
|
||||
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
|
||||
"default" skips it while keeping the `inv_freq` computed at init.
|
||||
"""
|
||||
if not dynamic_rope_update_is_noop(vllm_config):
|
||||
return
|
||||
for module in model.modules():
|
||||
rope_type = getattr(module, "rope_type", None)
|
||||
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
|
||||
continue
|
||||
if isinstance(rope_type, dict):
|
||||
module.rope_type = {
|
||||
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
|
||||
}
|
||||
elif rope_type == "dynamic":
|
||||
module.rope_type = "default"
|
||||
@@ -6,7 +6,6 @@ happen during model execution.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -134,25 +133,6 @@ def _warmup_bf16x3_router_gemm(
|
||||
logger.info_once("Warmed up BF16x3 router GEMM configs: %s.", configs)
|
||||
|
||||
|
||||
def _warmup_gemm_rs_ar() -> None:
|
||||
# Model construction (Kimi-K3, DeepSeek-V4.1) imports this module only
|
||||
# when GEMM-RS/AR is enabled and initializes its singleton before
|
||||
# kernel_warmup runs. Avoid importing it here so other models do not
|
||||
# compile the RS/AR variants.
|
||||
module = sys.modules.get("vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar")
|
||||
if module is None:
|
||||
return
|
||||
compiled = module.warmup_gemm_rs_ar()
|
||||
if compiled:
|
||||
logger.info_once("Warmed up %d GEMM-RS/AR variants.", compiled)
|
||||
|
||||
|
||||
def _autotune_kimi_k3_kda_qkvg(model: torch.nn.Module) -> None:
|
||||
module = sys.modules.get("vllm.models.kimi_k3.nvidia.low_latency_gemm")
|
||||
if module is not None:
|
||||
module.autotune_kda_qkvg(model)
|
||||
|
||||
|
||||
def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if not worker.use_v2_model_runner:
|
||||
# The KV-block zeroing kernel is driven by the scheduler's
|
||||
@@ -202,8 +182,6 @@ def kernel_warmup(worker: "Worker", *, process_local_only: bool = False):
|
||||
if current_platform.has_device_capability(90):
|
||||
_warmup_ll_bf16_router_gemm(worker.get_model())
|
||||
|
||||
_warmup_gemm_rs_ar()
|
||||
|
||||
if worker.vllm_config.kernel_config.enable_cutedsl_warmup:
|
||||
# TODO(roberto): Remove after registered CuTeDSL warmups are migrated
|
||||
# to the shared JIT warmup infrastructure.
|
||||
@@ -478,16 +456,11 @@ def flashinfer_autotune(runner: "GPUModelRunner", *, skip_attn: bool = False) ->
|
||||
)
|
||||
skip_attn = skip_attn or hisparse_enabled
|
||||
if hisparse_enabled:
|
||||
# HiSparse hot-buffer attention is bounded by decode batch
|
||||
# size, not the prefill-sized batch used for the full model.
|
||||
from vllm.model_executor.warmup.flashinfer_sparse_mla_warmup import (
|
||||
autotune_hisparse_flashinfer_attention,
|
||||
)
|
||||
|
||||
autotune_hisparse_flashinfer_attention(runner)
|
||||
# HiSparse config still bounds decode-batch autotune even
|
||||
# though the connector was removed from this build.
|
||||
logger.info_once("HiSparse configured; skipping attention autotune.")
|
||||
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
|
||||
replayssm_autotune_warmup(runner)
|
||||
_autotune_kimi_k3_kda_qkvg(runner.get_model())
|
||||
finally:
|
||||
set_autotune_process_group(None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user