Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -6,6 +6,7 @@ from collections.abc import Callable
from typing import Any
import torch
import torch.distributed as dist
import vllm.envs as envs
from vllm.model_executor.determinism.batch_invariant_configs import (
@@ -59,6 +60,7 @@ def matmul_kernel_persistent(
stride_bn,
stride_cm,
stride_cn,
stride_bias,
BLOCK_SIZE_M: tl.constexpr, #
BLOCK_SIZE_N: tl.constexpr, #
BLOCK_SIZE_K: tl.constexpr, #
@@ -128,7 +130,7 @@ def matmul_kernel_persistent(
c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
if HAS_BIAS:
bias_ptrs = bias_ptr + offs_cn
bias_ptrs = bias_ptr + offs_cn * stride_bias
bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
accumulator += bias
c = accumulator.to(c_ptr.dtype.element_ty)
@@ -305,6 +307,8 @@ def matmul_persistent(
),
)
if not _BLOCK_SIZES_RESOLVED:
_resolve_block_sizes()
configs = {
torch.bfloat16: {
"BLOCK_SIZE_M": 128,
@@ -345,6 +349,7 @@ def matmul_persistent(
b.stride(1), #
c.stride(0),
c.stride(1), #
bias.stride(0) if bias is not None else 1,
NUM_SMS=NUM_SMS, #
A_LARGE=a.numel() > 2**31,
B_LARGE=b.numel() > 2**31,
@@ -489,16 +494,19 @@ def bmm_kernel(
@triton.jit
def _log_softmax_kernel(
def _softmax_kernel(
input_ptr,
output_ptr,
input_row_stride,
input_col_stride,
output_row_stride,
n_cols,
BLOCK_SIZE: tl.constexpr,
LOG: tl.constexpr = False,
):
"""Compute log_softmax along the last dimension of a 2D tensor.
Each block handles one row of the input tensor.
"""Compute softmax, or log_softmax when ``LOG``, along the last dimension of a
2D tensor. Each block handles one row of the input tensor, in a fixed block
order, so the result never depends on the row count.
"""
# Get the row index for this block
row_idx = tl.program_id(0).to(tl.int64)
@@ -514,7 +522,9 @@ def _log_softmax_kernel(
mask = col_idx < n_cols
# Load values
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=-float("inf"))
vals = tl.load(
row_start_ptr + col_idx * input_col_stride, mask=mask, other=-float("inf")
)
# Update maximum
max_val = tl.max(tl.maximum(vals, max_val))
@@ -526,25 +536,26 @@ def _log_softmax_kernel(
mask = col_idx < n_cols
# Load values
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0)
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask, other=0.0)
# Compute exp(x - max_val) and accumulate
exp_vals = tl.exp(vals - max_val)
sum_exp += tl.sum(tl.where(mask, exp_vals, 0.0))
# Compute log(sum_exp)
log_sum_exp = tl.log(sum_exp)
log_sum_exp = tl.log(sum_exp) if LOG else 0.0
# Step 3: Compute final log_softmax values: x - max_val - log_sum_exp
# Step 3: normalise
for col_offset in range(0, n_cols, BLOCK_SIZE):
col_idx = col_offset + tl.arange(0, BLOCK_SIZE)
mask = col_idx < n_cols
# Load values
vals = tl.load(row_start_ptr + col_idx, mask=mask)
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask)
# Compute log_softmax
output = vals - max_val - log_sum_exp
if LOG:
output = vals - max_val - log_sum_exp
else:
output = tl.exp(vals - max_val) / sum_exp
# Store results
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
@@ -570,30 +581,55 @@ def log_softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
# Flatten all dimensions except the last one
original_shape = input.shape
input_2d = input.reshape(-1, input.shape[-1])
input_2d = input_2d.contiguous()
n_rows, n_cols = input_2d.shape
# Allocate output tensor
output = torch.empty_like(input_2d)
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
# Choose block size based on the number of columns
BLOCK_SIZE = 1024
# Launch kernel with one block per row
grid = (n_rows,)
_log_softmax_kernel[grid](
_softmax_kernel[grid](
input_2d,
output,
input_2d.stride(0),
input_2d.stride(1),
output.stride(0),
n_cols,
BLOCK_SIZE=BLOCK_SIZE,
LOG=True,
)
# Reshape output back to original shape
return output.reshape(original_shape)
def softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
"""Compute softmax along the last dimension using a Triton kernel."""
if dim != -1 and dim != input.ndim - 1:
raise ValueError(
"This implementation only supports softmax along the last dimension"
)
original_shape = input.shape
input_2d = input.reshape(-1, input.shape[-1])
n_rows, n_cols = input_2d.shape
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
_softmax_kernel[(n_rows,)](
input_2d,
output,
input_2d.stride(0),
input_2d.stride(1),
output.stride(0),
n_cols,
BLOCK_SIZE=1024,
)
return output.reshape(original_shape)
@triton.jit
def mean_kernel(
input_ptr,
@@ -735,6 +771,9 @@ def mean_dim(
def mm_batch_invariant(a, b, bias=None):
if a.device.type == "xpu":
# The descriptor kernel reads the bias at unit stride.
if bias is not None:
bias = bias.contiguous()
return matmul_descriptor_persistent(a, b, bias=bias)
return matmul_persistent(a, b, bias=bias)
@@ -820,6 +859,8 @@ def bmm_batch_invariant(a, b, *, out=None):
assert out.dtype == dtype and out.device == a.device, "out tensor mismatch"
c = out
if not _BLOCK_SIZES_RESOLVED:
_resolve_block_sizes()
configs = {
torch.bfloat16: {
"BLOCK_SIZE_M": 128,
@@ -877,8 +918,50 @@ def bmm_batch_invariant(a, b, *, out=None):
return c
def addmm_batch_invariant(bias, a, b):
return mm_batch_invariant(a, b, bias=bias)
def _addmm_fused_bias(bias, beta, alpha, n):
# The kernel adds bias into its fp32 accumulator indexed by output column,
# so it can only absorb a row vector of width n, unscaled by alpha.
if beta == 0 or alpha != 1:
return None
if bias.dim() != 1 or bias.shape[0] != n:
return None
# fp32 keeps the scaling exact; the kernel casts the bias to fp32 anyway.
return bias if beta == 1 else bias.float() * beta
def _addmm_impl(bias, a, b, beta, alpha, out):
fused_bias = _addmm_fused_bias(bias, beta, alpha, b.shape[1])
result = mm_batch_invariant(a, b, bias=fused_bias)
if fused_bias is None:
if alpha != 1:
result = result * alpha
if beta != 0:
result = result + (bias if beta == 1 else beta * bias)
if out is None:
return result
return out.copy_(result)
def addmm_batch_invariant(bias, a, b, *, beta=1, alpha=1):
return _addmm_impl(bias, a, b, beta, alpha, None)
# Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``, which
# dispatches the ``.out`` overload rather than the default one. Registering only
# the default leaves the compiled path on the vendor GEMM, so a compiled
# ``torch.addmm`` is batch variant while the eager one is not.
def mm_out_batch_invariant(a, b, *, out):
return matmul_batch_invariant(a, b, out=out)
def addmm_out_batch_invariant(bias, a, b, *, beta=1, alpha=1, out):
return _addmm_impl(bias, a, b, beta, alpha, out)
def bmm_out_batch_invariant(a, b, *, out):
return bmm_batch_invariant(a, b, out=out)
def _log_softmax_batch_invariant(input, dim, _half_to_float):
@@ -887,10 +970,16 @@ def _log_softmax_batch_invariant(input, dim, _half_to_float):
return log_softmax(input, dim=dim)
def softmax_batch_invariant(input, dim, dtype=None):
# Compute softmax in a deterministic way
# First subtract max for numerical stability (standard practice)
def softmax_batch_invariant(input, dim, half_to_float=False):
if half_to_float:
input = input.float()
if dim == -1 or dim == input.ndim - 1:
return softmax(input, dim=-1)
# Off the last dimension this falls back to torch ops, whose reduction split
# depends on the tensor shape.
input_max = torch.amax(input, dim=dim, keepdim=True)
# First subtract max for numerical stability (standard practice)
input = input - input_max
exp_x = torch.exp(input)
sum_exp_x = torch.sum(exp_x, dim=dim, keepdim=True)
@@ -1035,11 +1124,150 @@ def rms_norm_batch_invariant(
def linear_batch_invariant(input, weight, bias=None):
output = matmul_batch_invariant(input, weight.t())
if bias is not None:
output = output + bias
return output
bias = bias.expand(weight.shape[0])
output = mm_batch_invariant(
input.reshape(-1, input.shape[-1]), weight.t(), bias=bias
)
return output.view(*input.shape[:-1], output.shape[-1])
@triton.jit
def _fixed_order_sum_kernel(
src_ptr, # (WORLD_SIZE, numel) gathered contributions, rank-major
out_ptr, # (numel,)
numel,
stride_rank,
WORLD_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
LARGE: tl.constexpr,
):
"""Sum ``WORLD_SIZE`` contributions in ascending rank order, fp32 accumulator.
An output element sums exactly ``WORLD_SIZE`` values in a compile-time
constant order and rounds once, so the result depends only on the rank
ordering and never on how many elements are being reduced.
"""
pid = tl.program_id(0)
if LARGE:
pid = pid.to(tl.int64)
stride_rank = stride_rank.to(tl.int64)
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offs < numel
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
for r in tl.static_range(WORLD_SIZE):
vals = tl.load(src_ptr + r * stride_rank + offs, mask=mask, other=0.0)
acc += vals.to(tl.float32)
tl.store(out_ptr + offs, acc.to(out_ptr.dtype.element_ty), mask=mask)
def all_reduce_batch_invariant(
input_: torch.Tensor, group: "torch.distributed.ProcessGroup | None" = None
) -> torch.Tensor:
"""Sum all-reduce whose result does not depend on the message size.
Library all-reduces (NCCL/RCCL) pick their algorithm, channel count and chunk
boundaries from the message size, so the order in which a given element's
contributions are summed changes with the number of tokens in the batch.
Instead, all-gather the contributions -- pure data movement, so bitwise
reproducible at any size -- and reduce them with ``_fixed_order_sum_kernel``.
"""
world_size = dist.get_world_size(group)
if world_size == 1:
return input_
x = input_.contiguous()
numel = x.numel()
gathered = torch.empty((world_size, numel), dtype=x.dtype, device=x.device)
dist.all_gather_into_tensor(gathered, x.view(-1), group=group)
out = torch.empty_like(x)
BLOCK_SIZE = 1024
grid = (triton.cdiv(numel, BLOCK_SIZE),)
_fixed_order_sum_kernel[grid](
gathered,
out.view(-1),
numel,
numel,
WORLD_SIZE=world_size,
BLOCK_SIZE=BLOCK_SIZE,
LARGE=gathered.numel() > 2**31,
)
return out
def reduce_scatter_batch_invariant(
input_: torch.Tensor,
group: "torch.distributed.ProcessGroup | None" = None,
sizes: list[int] | None = None,
) -> torch.Tensor:
"""Sum reduce-scatter over dim 0 whose result does not depend on the size.
``ncclReduceScatter`` picks its chunking from the message size just as
``ncclAllReduce`` does. Route around it the same way
``all_reduce_batch_invariant`` does: an all-to-all sends rank ``d`` exactly
the rows it owns from every rank -- pure data movement, so bitwise
reproducible however the library chunks it -- and lands them rank-major,
which is the layout ``_fixed_order_sum_kernel`` already consumes.
``sizes`` (the reduce-scatterv case) changes only *which* rows a rank
receives, never how the received contributions are summed, so variable
shard sizes are invariant for the same reason.
Args:
input_: Contiguous full-size input; dim 0 is the scattered axis.
group: Process group, defaults to the world group.
sizes: Per-rank row counts. Uniform split when ``None``.
Returns:
This rank's shard of the sum.
"""
world_size = dist.get_world_size(group)
if world_size == 1:
return input_
x = input_.contiguous()
if sizes is None:
assert x.shape[0] % world_size == 0
sizes = [x.shape[0] // world_size] * world_size
else:
assert len(sizes) == world_size
assert x.shape[0] == sum(sizes)
rows = sizes[dist.get_rank(group)]
out = torch.empty((rows, *x.shape[1:]), dtype=x.dtype, device=x.device)
recv = torch.empty(
(world_size * rows, *x.shape[1:]), dtype=x.dtype, device=x.device
)
# Every rank takes part even when its own shard is empty: it still has rows
# to send to the others.
dist.all_to_all_single(
recv,
x,
output_split_sizes=[rows] * world_size,
input_split_sizes=sizes,
group=group,
)
numel = out.numel()
if numel == 0:
return out
BLOCK_SIZE = 1024
grid = (triton.cdiv(numel, BLOCK_SIZE),)
_fixed_order_sum_kernel[grid](
recv.view(-1),
out.view(-1),
numel,
numel,
WORLD_SIZE=world_size,
BLOCK_SIZE=BLOCK_SIZE,
LARGE=recv.numel() > 2**31,
)
return out
_batch_invariant_MODE = False
@@ -1047,9 +1275,33 @@ _batch_invariant_LIB = None
_fp16_block_size_n = 256
_fp32_block_size_n = 128
_fp32_num_stages = 3
_BLOCK_SIZES_RESOLVED = False
_NUM_SMS: int = 0
def _resolve_block_sizes():
"""Size the tiles to the shared memory, to avoid triton OutOfResources.
Runs when the mode is enabled, or on first use by a caller that never
enables it, such as a kernel unit test.
"""
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
global _BLOCK_SIZES_RESOLVED
_BLOCK_SIZES_RESOLVED = True
if not current_platform.is_cuda_alike():
return
if get_max_shared_memory_bytes() > 106496:
_fp16_block_size_n = 256
_fp32_block_size_n = 128
_fp32_num_stages = 3
else:
_fp16_block_size_n = 128
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
# Only N=1 OORs; keep 128/stages=3 for every other N.
_fp32_block_size_n = 32
_fp32_num_stages = 2
def enable_batch_invariant_mode():
global _batch_invariant_MODE, _batch_invariant_LIB
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
@@ -1062,12 +1314,17 @@ def enable_batch_invariant_mode():
_batch_invariant_LIB = torch.library.Library("aten", "IMPL")
key = current_platform.dispatch_key
if current_platform.is_cuda():
if current_platform.is_device_capability_family(80):
# SM80 (Ampere) cannot rely on cuBLASLt-only determinism; install the
# triton persistent matmul overrides for mm/addmm/matmul/linear.
if current_platform.is_cuda_alike():
if current_platform.is_rocm() or current_platform.is_device_capability_family(
80
):
# Neither cuBLASLt on SM80 (Ampere) nor hipBLASLt can be made batch
# invariant; install the triton persistent matmul overrides for
# mm/addmm/matmul/linear.
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
else:
@@ -1077,18 +1334,7 @@ def enable_batch_invariant_mode():
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1"
# Query the shared memory size and set block size
# accordingly to avoid triton OutOfResources.
if get_max_shared_memory_bytes() > 106496:
_fp16_block_size_n = 256
_fp32_block_size_n = 128
_fp32_num_stages = 3
else:
_fp16_block_size_n = 128
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
# Only N=1 OORs; keep 128/stages=3 for every other N.
_fp32_block_size_n = 32
_fp32_num_stages = 2
_resolve_block_sizes()
elif current_platform.is_xpu():
# Tensor descriptors need a global-memory allocator; must be set outside
# torch.compile regions (triton.set_allocator modifies global state).
@@ -1100,6 +1346,8 @@ def enable_batch_invariant_mode():
_NUM_SMS = num_compute_units(0)
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
@@ -1123,7 +1371,9 @@ def enable_batch_invariant_mode():
_batch_invariant_LIB.impl(
"aten::bmm", bmm_batch_invariant, key, allow_override=True
)
torch.bmm = bmm_batch_invariant
_batch_invariant_LIB.impl(
"aten::bmm.out", bmm_out_batch_invariant, key, allow_override=True
)
reduced_precision_val = (
(False, False) if is_torch_equal_or_newer("2.10.0") else False
@@ -1145,28 +1395,32 @@ def override_envs_for_invariance():
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
# unlike the pins below, it also applies to communicators shared with other
# processes.
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
pin_nccl_env(
{
"NCCL_COLLNET_ENABLE": "0",
"NCCL_NVLS_ENABLE": "0",
"NCCL_P2P_NET_DISABLE": "1",
"NCCL_MIN_NCHANNELS": "1",
"NCCL_MAX_NCHANNELS": "1",
"NCCL_PROTO": "Simple",
# NCCL >= 2.31 zero-fills the algorithm table of every collective
# when NCCL_ALGO is set and re-enables only the named ones; together
# with the NCCL_PROTO above, collectives not named here end up with
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
# Tree for all collectives, then pin AllReduce to Tree.
"NCCL_ALGO": "ring,tree;allreduce:tree",
"NCCL_NTHREADS": "1",
"NCCL_SOCKET_NTHREADS": "1",
}
)
if current_platform.is_rocm():
# The skinny GEMMs pick their split of K from the batch size.
os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
else:
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
# unlike the pins below, it also applies to communicators shared with other
# processes.
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
pin_nccl_env(
{
"NCCL_COLLNET_ENABLE": "0",
"NCCL_NVLS_ENABLE": "0",
"NCCL_P2P_NET_DISABLE": "1",
"NCCL_MIN_NCHANNELS": "1",
"NCCL_MAX_NCHANNELS": "1",
"NCCL_PROTO": "Simple",
# NCCL >= 2.31 zero-fills the algorithm table of every collective
# when NCCL_ALGO is set and re-enables only the named ones; together
# with the NCCL_PROTO above, collectives not named here end up with
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
# Tree for all collectives, then pin AllReduce to Tree.
"NCCL_ALGO": "ring,tree;allreduce:tree",
"NCCL_NTHREADS": "1",
"NCCL_SOCKET_NTHREADS": "1",
}
)
# torch.compile settings
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
@@ -3,7 +3,14 @@
import torch
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select
from vllm.model_executor.layers.indexer_topk import (
deep_select_topk,
get_deep_select_stride_requirement,
is_deep_select_supported,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
@triton.jit
@@ -17,38 +24,47 @@ def _block_scores_kernel(
starts,
ends,
scores,
valid_blocks,
stride_row,
stride_col,
stride_start,
stride_end,
stride_scores,
width,
nblocks,
BLOCK_SIZE: tl.constexpr,
HAS_STARTS: tl.constexpr,
ROW_REPEAT: tl.constexpr,
TILE: tl.constexpr,
FULL_WIDTH: tl.constexpr,
):
# Writes each row's valid block count; FULL_WIDTH also scores the -inf tail.
row = tl.program_id(0).to(tl.int64)
blocks = tl.program_id(1) * TILE + tl.arange(0, TILE)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
end = tl.load(ends + row // ROW_REPEAT * stride_end)
col_end = tl.minimum(end, width)
num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE)
if tl.program_id(1) == 0:
tl.store(valid_blocks + row, num_valid)
num_blocks = nblocks if FULL_WIDTH else num_valid
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(blocks[:, None] < nblocks)
& (offsets[None, :] < BLOCK_SIZE)
& (cols < end)
& (cols < width),
other=-float("inf"),
)
reduced = tl.reduce(values, 1, _max_with_nan)
reduced = tl.where(
(end > start) & (blocks == (end - start - 1) // BLOCK_SIZE),
float("inf"),
reduced,
)
tl.store(scores + row * nblocks + blocks, reduced, blocks < nblocks)
num_tiles = tl.cdiv(num_blocks, TILE)
for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)):
blocks = tile_id * TILE + tl.arange(0, TILE)
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(offsets[None, :] < BLOCK_SIZE) & (cols < col_end),
other=-float("inf"),
).to(tl.float32)
reduced = tl.reduce(values, 1, _max_with_nan)
# Pin the newest block. DeepSelect leaves NaN rows partially written,
# so also pin NaN blocks (torch.topk ranks NaN first too).
pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE)
if not FULL_WIDTH:
pinned |= reduced != reduced
reduced = tl.where(pinned, float("inf"), reduced)
tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks)
@triton.jit(do_not_specialize=["k"])
@@ -160,22 +176,42 @@ def select_candidate_blocks(
out.fill_(-1)
return
nblocks = triton.cdiv(width, block_size)
scores = logits.new_empty((rows, nblocks))
_block_scores_kernel[(rows, triton.cdiv(nblocks, 128))](
# Logits are max_model_len wide (CUDA-graph static) but rows are shorter:
# DeepSelect stops at each row's valid block count. Order does not matter.
use_deep_select = has_deep_select() and nblocks >= topk_blocks
if use_deep_select:
in_align, out_align = get_deep_select_stride_requirement()
padded = round_up(nblocks, in_align // logits.element_size())
scores = logits.new_empty((rows, padded))[:, :nblocks]
use_deep_select = (
out.stride(1) == 1
and out.stride(0) * out.element_size() % out_align == 0
and is_deep_select_supported(scores, topk_blocks)
)
else:
scores = logits.new_empty((rows, nblocks))
valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device)
_block_scores_kernel[(rows, 32)](
logits,
row_ks,
row_ke,
scores,
valid_blocks,
*logits.stride(),
row_ks.stride(0) if row_ks is not None else 0,
row_ke.stride(0),
scores.stride(0),
width,
nblocks,
block_size,
row_ks is not None,
row_repeat,
128,
TILE=128,
FULL_WIDTH=not use_deep_select,
)
if use_deep_select:
deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out)
return
# Keep the existing top-k tie behavior.
top = scores.topk(min(topk_blocks, nblocks), dim=-1)
_store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))](
@@ -123,6 +123,9 @@ from vllm.model_executor.kernels.linear.mxfp8.humming import (
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
TorchMxFp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
@@ -135,6 +138,7 @@ from vllm.model_executor.kernels.linear.nvfp4 import (
)
from vllm.model_executor.kernels.linear.nvfp4.b12x import (
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4.cutlass import (
CutlassNvFp4LinearKernel,
@@ -218,6 +222,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
)
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFp8BlockScaledMMKernel,
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
@@ -253,6 +258,20 @@ def _get_linear_backend(*, quantization: str) -> str:
return config.kernel_config.linear_backend
def _batch_invariant_fp8_kernel(kernel, config, quantization: str):
can_implement, reason = is_supported_and_can_implement_kernel(kernel, config, None)
if not can_implement:
raise ValueError(f"Batch invariance on ROCm: {reason}")
linear_backend = _get_linear_backend(quantization=quantization)
if linear_backend not in ("auto", "triton"):
logger.warning_once(
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
"using the Triton backend for deterministic execution.",
linear_backend,
)
return kernel
# Kernel classes covered by each --linear-backend value.
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"b12x": {
@@ -260,6 +279,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
B12xMxFp4LinearKernel,
B12xMxfp8LinearKernel,
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
B12xTensorFP8ScaledMMLinearKernel,
},
"cutlass": {
@@ -321,6 +341,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
ChannelWiseTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
TorchMxFp8LinearKernel,
TorchNvFp4LinearKernel,
},
"aiter": {
@@ -541,6 +562,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
EmulationMxfp8LinearKernel,
HummingMxfp8LinearKernel,
FlashInferTrtllmMxfp8LinearKernel,
TorchMxFp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -550,6 +572,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
],
PlatformEnum.XPU: [
XPUMxFp8LinearKernel,
TorchMxFp8LinearKernel,
EmulationMxfp8LinearKernel,
],
}
@@ -567,6 +590,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
FlashInferCudnnNvFp4LinearKernel,
FbgemmNvFp4LinearKernel,
B12xNvFp4LinearKernel,
B12xNvFp4W4A16LinearKernel,
TorchNvFp4LinearKernel,
EmulationNvFp4LinearKernel,
HummingNvFp4LinearKernel,
@@ -723,6 +747,13 @@ def init_fp8_linear_kernel(
)
if activation_quant_key.scale.group_shape.is_per_group():
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
force_kernel = _batch_invariant_fp8_kernel(
TritonFp8BlockScaledMMKernel,
scaled_mm_linear_kernel_config,
"fp8_block_w8a8",
)
kernel_type = choose_scaled_mm_linear_kernel(
config=scaled_mm_linear_kernel_config,
possible_kernels=_POSSIBLE_FP8_BLOCK_KERNELS, # type: ignore[misc]
@@ -755,6 +786,13 @@ def init_fp8_linear_kernel(
)
else:
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
force_kernel = _batch_invariant_fp8_kernel(
TritonFP8ScaledMMLinearKernel,
scaled_mm_linear_kernel_config,
"fp8_w8a8",
)
kernel_type = choose_scaled_mm_linear_kernel(
config=scaled_mm_linear_kernel_config,
possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type]
@@ -780,6 +818,19 @@ def init_fp8_linear_kernel(
)
def fp8_scaled_mm_is_batch_invariant(
kernel: FP8ScaledMMLinearKernel | Fp8BlockScaledMMLinearKernel,
) -> bool:
"""Whether a row's output from the kernel is independent of the other rows."""
if not isinstance(
kernel, (CutlassFP8ScaledMMLinearKernel, TritonFP8ScaledMMLinearKernel)
):
return False
# A dynamic per-tensor activation scale is an amax over every row.
scale = kernel.config.activation_quant_key.scale
return scale.static or not scale.group_shape.is_per_tensor()
def init_int8_linear_kernel(
is_channelwise: bool,
is_static_input_scheme: bool,
@@ -1085,6 +1136,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
current platform."""
config = NvFp4LinearLayerConfig()
a16_kernels = (
B12xNvFp4W4A16LinearKernel,
FlashInferCuteDslNvFp4W4A16LinearKernel,
MarlinNvFp4LinearKernel,
HummingNvFp4LinearKernel,
@@ -1250,6 +1302,7 @@ def register_linear_kernel(
__all__ = [
"fp8_scaled_mm_is_batch_invariant",
"init_fp8_linear_kernel",
"init_int8_linear_kernel",
"init_nvfp4_linear_kernel",
@@ -1298,6 +1351,7 @@ __all__ = [
"B12xMxfp8LinearKernel",
"B12xMxFp4LinearKernel",
"B12xNvFp4LinearKernel",
"B12xNvFp4W4A16LinearKernel",
"init_mxfp4_linear_kernel",
"MxFp4LinearKernel",
"MxFp4LinearLayerConfig",
@@ -1315,6 +1369,7 @@ __all__ = [
"FlashInferTrtllmMxfp8LinearKernel",
"MarlinMxfp8LinearKernel",
"XPUMxFp8LinearKernel",
"TorchMxFp8LinearKernel",
"EmulationMxfp8LinearKernel",
"CutlassNvFp4LinearKernel",
"EmulationNvFp4LinearKernel",
@@ -0,0 +1,457 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import cutlass
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
from cutlass import const_expr
class LLFp32WDotprod:
"""FP32-weight router GEMM kernel based on CTA-local dot products.
This kernel computes C[M, N] = A[M, K] @ B[N, K]^T for bf16/fp16/fp32
activations, fp32 weights, and fp32 output. It launches one CTA per
small output-column tile, distributes K across CTA threads with vectorized
loads, accumulates one fp32 dot product per token/expert, and reduces
through warp shuffles plus shared memory.
:param k: Compile-time K dimension specialized into the generated kernel.
:type k: int
:param bs: Threads per CTA and K-stripe width used by the reduction.
:type bs: int
:param main_vec_width: Elements loaded per thread in the main loop.
:type main_vec_width: int
:param tail_vec_width: Elements loaded per thread in the vector tail.
:type tail_vec_width: int
:param use_pdl: Whether to launch with Programmatic Dependent Launch.
:type use_pdl: bool
:note: Supported A/B data types:
- BFloat16/Float32
- Float16/Float32
- Float32/Float32
:note: Supported accumulator data types:
- Float32
:note: Supported C data types:
- Float32
:note: Constraints:
- Inputs must be contiguous row-major 2D tensors.
:note: K is handled as vectorized main/tail loops plus scalar remainder.
:compile-key: ``(M, K, bs, epb, act_dtype)`` selects the token count,
hidden size, CTA thread/K-stripe width, experts per CTA, and activation
input type.
"""
def __init__(
self,
m: int,
k: int,
bs: int = 128,
main_vec_width: int = 8,
tail_vec_width: int = 4,
token_groups: int = 1,
epb: int = 1,
use_pdl: bool = False,
):
"""Initialize the dot-product kernel configuration.
This configuration fixes the CTA thread count, reduction warp count,
vector widths, and K-loop decomposition used by the generated kernel.
:param m: Token count M selected by the compile key.
:type m: int
:param k: Hidden size K used to specialize the K-loop decomposition.
:type k: int
:param bs: Threads per CTA and K-stripe width for the reduction.
:type bs: int
:param main_vec_width: Elements loaded per thread in the main loop.
:type main_vec_width: int
:param tail_vec_width: Elements loaded per thread in the vector tail.
:type tail_vec_width: int
:param use_pdl: Whether to launch with Programmatic Dependent Launch.
:type use_pdl: bool
"""
self.m = m
self.m_per_group = m // token_groups
self.bs = bs
self.main_vec_width = main_vec_width
self.tail_vec_width = tail_vec_width
self.token_groups = token_groups
self.epb = epb
self.use_pdl = use_pdl
self.num_warps = bs // cute.arch.WARP_SIZE
self._init_k_tiles(k)
def _vectorized_elems(self, k_extent: int, vec_width: int) -> int:
vector_tile = vec_width * self.bs
return (k_extent // vector_tile) * vector_tile
def _init_k_tiles(self, k: int) -> None:
"""Split K into vector loops, scalar rounds, and ragged tail."""
self.k_main_elems = self._vectorized_elems(k, self.main_vec_width)
self.k_after_main = k - self.k_main_elems
self.k_tail_elems = self._vectorized_elems(
self.k_after_main, self.tail_vec_width
)
self.k_done_all = self.k_main_elems + self.k_tail_elems
self.scalar_rem = k - self.k_done_all
self.ks_full = self.scalar_rem // self.bs
self.ks_part = self.scalar_rem % self.bs
self.k_scalar_full = self.ks_full * self.bs
self.k_part_offset = self.k_done_all + self.k_scalar_full
self.main_tiles = self.k_main_elems // (self.main_vec_width * self.bs)
self.tail_tiles = self.k_tail_elems // (self.tail_vec_width * self.bs)
@cute.jit
def _vector_dotprod(
self,
acc: cute.Tensor,
tA: cute.Tensor,
tB: cute.Tensor,
M: cutlass.Constexpr,
epb: cutlass.Constexpr,
num_tiles: cutlass.Constexpr,
):
for tile in cutlass.range_constexpr(num_tiles):
bt0 = tB[0, None, tile]
br0 = cute.make_rmem_tensor_like(bt0)
cute.autovec_copy(bt0, br0)
br0_f32 = br0.load().to(cutlass.Float32)
if const_expr(epb == 2):
bt1 = tB[1, None, tile]
br1 = cute.make_rmem_tensor_like(bt1)
cute.autovec_copy(bt1, br1)
br1_f32 = br1.load().to(cutlass.Float32)
for m in cutlass.range_constexpr(M):
at = tA[m, None, tile]
ar = cute.make_rmem_tensor_like(at)
cute.autovec_copy(at, ar)
vec_width: cutlass.Constexpr = cute.size(ar)
for v in cutlass.range_constexpr(vec_width):
av = ar[v].to(cutlass.Float32)
acc[m, 0] = acc[m, 0] + av * br0_f32[v]
if const_expr(epb == 2):
acc[m, 1] = acc[m, 1] + av * br1_f32[v]
def _make_thread_vector_slice(
self,
gA_vec: cute.Tensor,
gB_vec: cute.Tensor,
tidx: cutlass.Int32,
bs: cutlass.Constexpr,
):
# A: (M, K_TILE, K_LANE, K_VEC); B: (epb, K_TILE, K_LANE, K_VEC).
tA = cute.logical_divide(gA_vec, (None, (None, bs)))
tB = cute.logical_divide(gB_vec, (None, (None, bs)))
return tA[None, (None, (tidx, None))], tB[None, (None, (tidx, None))]
def _make_m_slice(
self,
gX: cute.Tensor,
m_offset: cutlass.Int32,
m_extent: cutlass.Constexpr,
):
return cute.local_tile(
cute.domain_offset((m_offset, 0), gX),
(m_extent, cute.size(gX, mode=[1])),
(0, 0),
)
def _make_n_tile(
self,
gX: cute.Tensor,
n_tile_idx: cutlass.Int32,
n_extent: cutlass.Constexpr,
):
return cute.local_tile(
gX,
(n_extent, cute.size(gX, mode=[1])),
(n_tile_idx, 0),
)
def _make_sm_m_slice(
self,
sm: cute.Tensor,
m_offset: cutlass.Int32,
m_extent: cutlass.Constexpr,
epb: cutlass.Constexpr,
num_warps: cutlass.Constexpr,
):
return cute.local_tile(
cute.domain_offset((m_offset, 0, 0), sm),
(m_extent, epb, num_warps),
(0, 0, 0),
)
def _make_k_slice(
self,
gX: cute.Tensor,
k_offset: cutlass.Constexpr,
k_extent: cutlass.Constexpr,
):
k_layout_extent: cutlass.Constexpr = (
1 if const_expr(k_extent == 0) else k_extent
)
if const_expr(k_offset == 0):
return cute.local_tile(
gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0)
)
return cute.local_tile(
cute.domain_offset((0, k_offset), gX),
(cute.size(gX, mode=[0]), k_layout_extent),
(0, 0),
)
@cute.jit
def _compute_token_group(
self,
gA: cute.Tensor,
gB: cute.Tensor,
sm: cute.Tensor,
n_tile_idx: cutlass.Int32,
tidx: cutlass.Int32,
local_wid: cutlass.Int32,
m_per_group: cutlass.Constexpr,
m_offset: cutlass.Int32,
main_vec_width: cutlass.Constexpr,
tail_vec_width: cutlass.Constexpr,
bs: cutlass.Constexpr,
epb: cutlass.Constexpr,
num_warps: cutlass.Constexpr,
k_main_elems: cutlass.Constexpr,
k_tail_elems: cutlass.Constexpr,
k_done_all: cutlass.Constexpr,
ks_full: cutlass.Constexpr,
ks_part: cutlass.Constexpr,
k_scalar_full: cutlass.Constexpr,
k_part_offset: cutlass.Constexpr,
main_tiles: cutlass.Constexpr,
tail_tiles: cutlass.Constexpr,
):
# Each CTA reuses A for epb adjacent experts.
acc = cute.make_rmem_tensor((m_per_group, epb), cutlass.Float32)
acc.fill(0.0)
gA_group = self._make_m_slice(gA, m_offset, m_per_group)
gB_epb = self._make_n_tile(gB, n_tile_idx, epb)
sm_group = self._make_sm_m_slice(sm, m_offset, m_per_group, epb, num_warps)
if const_expr(k_main_elems > 0):
gA_main = self._make_k_slice(gA_group, 0, k_main_elems)
gB_main = self._make_k_slice(gB_epb, 0, k_main_elems)
gA_vec = cute.logical_divide(gA_main, (None, main_vec_width))
gB_vec = cute.logical_divide(gB_main, (None, main_vec_width))
tA, tB = self._make_thread_vector_slice(gA_vec, gB_vec, tidx, bs)
self._vector_dotprod(acc, tA, tB, m_per_group, epb, main_tiles)
if const_expr(k_tail_elems > 0):
gA_tail = self._make_k_slice(gA_group, k_main_elems, k_tail_elems)
gB_tail = self._make_k_slice(gB_epb, k_main_elems, k_tail_elems)
gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width))
gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width))
tA_t, tB_t = self._make_thread_vector_slice(
gA_tail_vec, gB_tail_vec, tidx, bs
)
self._vector_dotprod(acc, tA_t, tB_t, m_per_group, epb, tail_tiles)
if const_expr(ks_full > 0):
gA_scalar = self._make_k_slice(gA_group, k_done_all, k_scalar_full)
gB_scalar = self._make_k_slice(gB_epb, k_done_all, k_scalar_full)
gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1))
gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1))
tA_s, tB_s = self._make_thread_vector_slice(
gA_scalar_vec, gB_scalar_vec, tidx, bs
)
self._vector_dotprod(acc, tA_s, tB_s, m_per_group, epb, ks_full)
if const_expr(ks_part > 0):
gA_part = self._make_k_slice(gA_group, k_part_offset, ks_part)
gB_part = self._make_k_slice(gB_epb, k_part_offset, ks_part)
if tidx < ks_part:
for local_m in cutlass.range_constexpr(m_per_group):
av = gA_part[local_m, tidx].to(cutlass.Float32)
for e in cutlass.range_constexpr(epb):
bv = gB_part[e, tidx].to(cutlass.Float32)
acc[local_m, e] = acc[local_m, e] + av * bv
for local_m in cutlass.range_constexpr(m_per_group):
for e in cutlass.range_constexpr(epb):
acc[local_m, e] = cute.arch.warp_reduction_sum(acc[local_m, e])
with cute.arch.elect_one():
for local_m in cutlass.range_constexpr(m_per_group):
for e in cutlass.range_constexpr(epb):
sm_group[local_m, e, local_wid] = acc[local_m, e]
@cute.jit
def _store_outputs(
self,
gC: cute.Tensor,
sm: cute.Tensor,
n_tile_idx: cutlass.Int32,
N_dim: cutlass.Int32,
tidx: cutlass.Int32,
group_idx: cutlass.Int32,
M: cutlass.Constexpr,
epb: cutlass.Constexpr,
):
n_tile_layout = cute.make_layout(
(cute.ceil_div(N_dim, epb), epb), stride=(epb, 1)
)
n_coords = cute.make_tensor(0, n_tile_layout)
if group_idx == 0 and tidx < M:
for e in cutlass.range_constexpr(epb):
n = n_coords[n_tile_idx, e]
if n < N_dim:
partials = sm[tidx, e, None].load()
gC[tidx, n] = partials.reduce(
cute.ReductionOp.ADD,
init_val=cutlass.Float32(0.0),
reduction_profile=0,
)
@cute.jit
def __call__(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gC: cute.Tensor,
N_dim: cutlass.Int32,
stream: CUstream,
):
"""Execute the dot-product GEMM operation in steps:
- Launch one CTA per small output-column tile with ``bs`` threads.
- Keep one FP32 accumulator per token/expert in each thread.
- Traverse K with vectorized main/tail, scalar, and ragged-tail loops
from the precomputed K decomposition.
- Reduce each token accumulator first within the warp, then across
warps through shared memory, and store ``C[:, n]``.
:param gA: Input tensor A with shape ``[M, K]``.
:type gA: cute.Tensor
:param gB: Input tensor B with shape ``[N, K]``.
:type gB: cute.Tensor
:param gC: Output tensor C with shape ``[M, N]``.
:type gC: cute.Tensor
:param N_dim: Output column count used for the launch grid.
:type N_dim: cutlass.Int32
:param stream: CUDA stream for asynchronous execution.
:type stream: CUstream
"""
self.kernel(
gA,
gB,
gC,
N_dim,
self.m,
self.m_per_group,
self.main_vec_width,
self.tail_vec_width,
self.bs,
self.num_warps,
self.token_groups,
self.epb,
self.k_main_elems,
self.k_tail_elems,
self.k_done_all,
self.ks_full,
self.ks_part,
self.k_scalar_full,
self.k_part_offset,
self.main_tiles,
self.tail_tiles,
).launch(
grid=[cute.ceil_div(N_dim, self.epb), 1, 1],
block=[self.bs, self.token_groups, 1],
smem=self.m * self.epb * 4 * self.num_warps,
stream=stream,
use_pdl=self.use_pdl,
min_blocks_per_mp=1,
)
@cute.kernel
def kernel(
self,
gA: cute.Tensor,
gB: cute.Tensor,
gC: cute.Tensor,
N_dim: cutlass.Int32,
M: cutlass.Constexpr,
m_per_group: cutlass.Constexpr,
main_vec_width: cutlass.Constexpr,
tail_vec_width: cutlass.Constexpr,
bs: cutlass.Constexpr,
num_warps: cutlass.Constexpr,
token_groups: cutlass.Constexpr,
epb: cutlass.Constexpr,
k_main_elems: cutlass.Constexpr,
k_tail_elems: cutlass.Constexpr,
k_done_all: cutlass.Constexpr,
ks_full: cutlass.Constexpr,
ks_part: cutlass.Constexpr,
k_scalar_full: cutlass.Constexpr,
k_part_offset: cutlass.Constexpr,
main_tiles: cutlass.Constexpr,
tail_tiles: cutlass.Constexpr,
):
tidx, group_idx, _ = cute.arch.thread_idx()
n_tile_idx, _, _ = cute.arch.block_idx()
local_wid = tidx // cute.arch.WARP_SIZE
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_wait()
smem_red_layout = cute.make_layout(
(M, epb, num_warps), stride=(epb * num_warps, num_warps, 1)
)
smem = cutlass.utils.SmemAllocator()
sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16)
if const_expr(token_groups == 1):
m_offset: cutlass.Constexpr = 0
else:
m_offset = group_idx * m_per_group
self._compute_token_group(
gA,
gB,
sm,
n_tile_idx,
tidx,
local_wid,
m_per_group,
m_offset,
main_vec_width,
tail_vec_width,
bs,
epb,
num_warps,
k_main_elems,
k_tail_elems,
k_done_all,
ks_full,
ks_part,
k_scalar_full,
k_part_offset,
main_tiles,
tail_tiles,
)
cute.arch.sync_threads()
self._store_outputs(gC, sm, n_tile_idx, N_dim, tidx, group_idx, M, epb)
if const_expr(self.use_pdl):
cute.arch.griddepcontrol_launch_dependents()
def make_host_fp32w(
k_val: int, bs: int = 128, token_groups: int = 1, epb: int = 1, m: int = 1
):
return LLFp32WDotprod(m=m, k=k_val, bs=bs, token_groups=token_groups, epb=epb)
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import logging
from typing import Any
import torch
logger = logging.getLogger(__name__)
_cutedsl_available: bool | None = None
_cute_ctx: tuple[Any, Any] | None = None
def is_cutedsl_available(kernel_name: str) -> bool:
global _cutedsl_available
if _cutedsl_available is not None:
return _cutedsl_available
try:
import cutlass # noqa: F401
import cutlass.cute # noqa: F401
_cutedsl_available = True
except ImportError:
_cutedsl_available = False
logger.info(
"cuteDSL (CUTLASS Python) not available, %s disabled",
kernel_name,
)
return _cutedsl_available
def cute_context():
global _cute_ctx
if _cute_ctx is not None:
return _cute_ctx
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
_cute_ctx = (cute, CUstream)
return _cute_ctx
def current_cuda_stream():
_, CUstream = cute_context()
from vllm.utils.torch_utils import current_stream
return CUstream(current_stream().cuda_stream)
def use_pdl() -> bool:
from vllm.platforms import current_platform
return current_platform.is_arch_support_pdl()
def cutlass_dtype(dtype: torch.dtype):
from cutlass import BFloat16, Float16, Float32
if dtype == torch.bfloat16:
return BFloat16
if dtype == torch.float16:
return Float16
if dtype == torch.float32:
return Float32
raise ValueError(f"unsupported router GEMM dtype: {dtype}")
def make_fake_gemm_tensors(
*,
M,
K,
N,
a_dtype: torch.dtype,
b_dtype: torch.dtype,
divisibility: int,
):
from cutlass import Float32
from quack.compile_utils import make_fake_tensor
hidden_states = make_fake_tensor(
cutlass_dtype(a_dtype), (M, K), divisibility=divisibility
)
router_weight = make_fake_tensor(
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
)
output = make_fake_tensor(Float32, (M, N), divisibility=1)
return hidden_states, router_weight, output
def validate_common_gemm_inputs(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
output_dtype: torch.dtype,
*,
op_name: str,
k_multiple: int | None = None,
) -> None:
if hidden_states.dim() != 2 or router_weight.dim() != 2:
raise ValueError("hidden_states and router_weight must be 2D tensors")
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
raise ValueError("hidden_states and router_weight must have device_type=cuda")
if hidden_states.device != router_weight.device:
raise ValueError(
"hidden_states and router_weight must be on the same CUDA device"
)
if output_dtype != torch.float32:
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
if hidden_states.shape[1] != router_weight.shape[1]:
raise ValueError(
"hidden_states and router_weight must have matching K dimensions"
)
if k_multiple is not None and hidden_states.shape[1] % k_multiple != 0:
raise ValueError(f"{op_name} requires K to be divisible by {k_multiple}")
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
raise ValueError(
"hidden_states and router_weight must be contiguous row-major inputs"
)
@@ -11,24 +11,20 @@ from typing import Any, Literal
import torch
logger = logging.getLogger(__name__)
from ._ll_router_common import (
current_cuda_stream,
cute_context,
is_cutedsl_available,
make_fake_gemm_tensors,
use_pdl,
validate_common_gemm_inputs,
)
_cutedsl_available: bool | None = None
logger = logging.getLogger(__name__)
def is_available() -> bool:
global _cutedsl_available
if _cutedsl_available is not None:
return _cutedsl_available
try:
import cutlass # noqa: F401
import cutlass.cute # noqa: F401
_cutedsl_available = True
except ImportError:
_cutedsl_available = False
logger.info("cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled")
return _cutedsl_available
return is_cutedsl_available("ll_bf16_gemm")
# Default configs
@@ -96,33 +92,6 @@ def _arch_tuned_configs() -> tuple[
return {}, {}
_cute_ctx = None
def _cute():
global _cute_ctx
if _cute_ctx is not None:
return _cute_ctx
import cutlass.cute as cute
from cuda.bindings.driver import CUstream
_cute_ctx = (cute, CUstream)
return _cute_ctx
def _stream():
_, CUstream = _cute()
from vllm.utils.torch_utils import current_stream
return CUstream(current_stream().cuda_stream)
def _use_pdl() -> bool:
from vllm.platforms import current_platform
return current_platform.is_arch_support_pdl()
class LLBf16Gemm:
@dataclass(frozen=True, slots=True)
class CompileKey:
@@ -165,16 +134,17 @@ class LLBf16Gemm:
@staticmethod
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
from cutlass import BFloat16, Float32
from quack.compile_utils import make_fake_tensor
hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=divisibility)
router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=divisibility)
output = make_fake_tensor(Float32, (M, N), divisibility=1)
return hidden_states, router_weight, output
return make_fake_gemm_tensors(
M=M,
K=K,
N=N,
a_dtype=torch.bfloat16,
b_dtype=torch.bfloat16,
divisibility=divisibility,
)
def _compile_splitk(self, compile_key: CompileKey) -> None:
cute, _ = _cute()
cute, _ = cute_context()
from ._ll_bf16_splitk import LLBf16SplitK
hidden_states, router_weight, output = self._fake_gemm_tensors(
@@ -189,14 +159,14 @@ class LLBf16Gemm:
num_stages=compile_key.num_stages,
num_dma_warps=4,
split_k=compile_key.split_k,
use_pdl=_use_pdl(),
use_pdl=use_pdl(),
)
compiled = cute.compile(
gemm,
hidden_states,
router_weight,
output,
_stream(),
current_cuda_stream(),
options="--enable-tvm-ffi",
)
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
@@ -207,7 +177,7 @@ class LLBf16Gemm:
)
def _compile_dotprod(self, compile_key: CompileKey) -> None:
cute, _ = _cute()
cute, _ = cute_context()
from ._ll_bf16_dotprod import LLBf16Dotprod
N = cute.sym_int()
@@ -221,7 +191,7 @@ class LLBf16Gemm:
gemm = LLBf16Dotprod(
k=compile_key.K,
bs=compile_key.bs,
use_pdl=_use_pdl(),
use_pdl=use_pdl(),
prefetch_pdl_weights=self._prefetch_pdl_weights,
)
compiled = cute.compile(
@@ -232,7 +202,7 @@ class LLBf16Gemm:
compile_key.M,
compile_key.K,
1, # runtime N placeholder for fake-tensor compile
_stream(),
current_cuda_stream(),
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
)
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
@@ -269,34 +239,19 @@ class LLBf16Gemm:
router_weight: torch.Tensor,
output_dtype: torch.dtype,
) -> None:
if hidden_states.dim() != 2 or router_weight.dim() != 2:
raise ValueError("hidden_states and router_weight must be 2D tensors")
if (
hidden_states.dtype != torch.bfloat16
or router_weight.dtype != torch.bfloat16
):
raise ValueError("hidden_states and router_weight must have dtype=bfloat16")
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
raise ValueError(
"hidden_states and router_weight must have device_type=cuda"
)
if hidden_states.device != router_weight.device:
raise ValueError(
"hidden_states and router_weight must be on the same CUDA device"
)
if output_dtype != torch.float32:
raise ValueError("ll_bf16_gemm only supports output_dtype=torch.float32")
if hidden_states.shape[1] != router_weight.shape[1]:
raise ValueError(
"hidden_states and router_weight must have matching K dimensions"
)
# Kernels use vectorized bf16 loads and require 16-byte row alignment.
if hidden_states.shape[1] % 8 != 0:
raise ValueError("ll_bf16_gemm requires K to be divisible by 8")
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
raise ValueError(
"hidden_states and router_weight must be contiguous row-major inputs"
)
validate_common_gemm_inputs(
hidden_states,
router_weight,
output_dtype,
op_name="ll_bf16_gemm",
k_multiple=8,
)
def __call__(
self,
@@ -320,7 +275,7 @@ class LLBf16Gemm:
self.compile(compile_key)
kernel = self._compiled_cache[dotprod_cache_key]
stream = _stream()
stream = current_cuda_stream()
output = torch.empty(M, N, dtype=output_dtype, device=hidden_states.device)
if compile_key.backend == "splitk":
kernel(hidden_states, router_weight, output, stream, 1.0)
@@ -0,0 +1,197 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import logging
import math
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, zip_inputs
from ._ll_router_common import (
current_cuda_stream,
cute_context,
is_cutedsl_available,
make_fake_gemm_tensors,
use_pdl,
validate_common_gemm_inputs,
)
logger = logging.getLogger(__name__)
def is_available() -> bool:
return is_cutedsl_available("ll_fp32w_gemm")
_DEFAULT_DOTPROD_CONFIG = (192, 1, 1)
_DEFAULT_DOTPROD_GROUPED_CONFIG = (192, 2, 1)
_TUNED_DOTPROD_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int, int]]] = {
(6144, 128): {
**{
M: (384, 1, 1)
for M in (
*range(1, 6),
*range(9, 14, 2),
17,
*range(23, 32, 2),
)
},
**{M: (256, 1, 1) for M in (7, 15, *range(19, 22, 2))},
**{M: (384, 2, 1) for M in range(6, 12, 2)},
**{M: (192, 2, 1) for M in (*range(12, 26, 2), 32)},
**{M: (128, 2, 1) for M in range(26, 32, 2)},
},
(6144, 256): {
**{M: (384, 1, 1) for M in range(1, 4)},
**{M: (128, 1, 1) for M in (*range(4, 7, 2), 11)},
**{M: (128, 1, 2) for M in range(5, 11, 2)},
**{M: (128, 2, 2) for M in range(8, 33, 2)},
**{M: (256, 1, 2) for M in range(13, 33, 2)},
},
}
_SUPPORTED_ACTIVATION_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
class LLFp32WGemm(VllmJitKernel["LLFp32WGemm.CompileKey"]):
# Dot-prod is specialized on M/K, tile shape, and activation dtype.
@dataclass(frozen=True, slots=True)
class CompileKey:
m: int
k: int
bs: int
a_dtype: torch.dtype
token_groups: int = 1
epb: int = 1
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
from ._ll_fp32w_dotprod import LLFp32WDotprod
return LLFp32WDotprod(
m=compile_key.m,
k=compile_key.k,
bs=compile_key.bs,
token_groups=compile_key.token_groups,
epb=compile_key.epb,
use_pdl=use_pdl(),
)
def dispatch( # type: ignore[override]
self,
*,
M: int,
K: int,
N: int,
a_dtype: torch.dtype,
) -> CompileKey:
default_config = (
_DEFAULT_DOTPROD_GROUPED_CONFIG
if N <= 128 and M >= 12 and M % 2 == 0
else _DEFAULT_DOTPROD_CONFIG
)
tuned_configs = _TUNED_DOTPROD_CONFIGS.get((K, N))
config = (
tuned_configs.get(M, default_config) if tuned_configs else default_config
)
return self.CompileKey(
m=M,
k=K,
bs=config[0],
a_dtype=a_dtype,
token_groups=config[1],
epb=config[2],
)
def get_warmup_keys(
self,
*,
shapes: Iterable[tuple[int, int]],
m_values: Iterable[int],
a_dtypes: Iterable[torch.dtype] = (torch.bfloat16,),
) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
zip_inputs(*(dict(K=K, N=N) for K, N in shapes)),
M=tuple(m_values),
a_dtype=tuple(a_dtypes),
)
def compile(self, compile_key: CompileKey) -> None:
if compile_key in self._compiled_cache:
return
if compile_key.token_groups not in (1, 2):
raise ValueError("ll_fp32w_gemm supports token_groups 1 or 2")
if compile_key.epb not in (1, 2):
raise ValueError("ll_fp32w_gemm supports epb 1 or 2")
if compile_key.m % compile_key.token_groups != 0:
raise ValueError("M must be divisible by token_groups")
cute, _ = cute_context()
N = cute.sym_int()
stride_divisibility = math.gcd(8, compile_key.k)
hidden_states, router_weight, output = make_fake_gemm_tensors(
M=compile_key.m,
K=compile_key.k,
N=N,
a_dtype=compile_key.a_dtype,
b_dtype=torch.float32,
divisibility=stride_divisibility,
)
compiled = cute.compile(
self.kernel(compile_key),
hidden_states,
router_weight,
output,
1, # runtime N placeholder for fake-tensor compile
current_cuda_stream(),
options="--enable-tvm-ffi",
)
self._compiled_cache[compile_key] = compiled
@staticmethod
def _validate_inputs(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
output_dtype: torch.dtype,
) -> None:
if hidden_states.dtype not in _SUPPORTED_ACTIVATION_DTYPES:
raise ValueError("hidden_states must have dtype bf16, fp16, or fp32")
if router_weight.dtype != torch.float32:
raise ValueError("router_weight must have dtype=float32")
validate_common_gemm_inputs(
hidden_states,
router_weight,
output_dtype,
op_name="ll_fp32w_gemm",
)
def __call__(
self,
hidden_states: torch.Tensor, # [M, K] bf16/fp16/fp32
router_weight: torch.Tensor, # [N, K] fp32
output_dtype: torch.dtype = torch.float32,
) -> torch.Tensor: # [M, N] fp32
self._validate_inputs(hidden_states, router_weight, output_dtype)
M, K = hidden_states.shape
N = router_weight.shape[0]
compile_key = self.dispatch(M=M, K=K, N=N, a_dtype=hidden_states.dtype)
compiled = self._get_or_compile(
compile_key,
runtime_context={"M": M, "K": K, "N": N},
)
stream = current_cuda_stream()
output = torch.empty(M, N, dtype=output_dtype, device=hidden_states.device)
compiled(hidden_states, router_weight, output, N, stream)
return output
LL_FP32W_GEMM_KERNEL = LLFp32WGemm()
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
@@ -71,7 +72,7 @@ class HummingLinearKernel(MPLinearKernel):
input_schema=input_schema,
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -57,9 +57,21 @@ def _on_gfx1151() -> bool:
return on_gfx1151()
def _on_gfx115x() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx115x
return on_gfx115x()
# Maximum activation rows supported by HIP skinny (Python M maps to C++ N_in).
# When K*M exceeds regular LDS capacity, the C++ medium kernel caches the
# prefix in LDS and reads the remaining activation elements from global memory.
# On gfx115x the K*M term is dropped entirely: deep-K shapes stage every
# activation row in LDS one K chunk at a time, and only when N is not a
# multiple of the kernel's row tile does the medium kernel fall back to reading
# the rows that do not fit from global memory.
MAX_SKINNY_BATCH_SIZE = 5
# 64 KiB per-workgroup LDS limit expressed in fp16 elements.
# (AMD RDNA has 128 KiB total LDS per CU, but 64 KiB per workgroup.)
@@ -86,6 +98,7 @@ def _triton_w4a16_skinny_fmt_kernel(
K,
K8, # K // 8
num_groups, # K // group_size
stride_sn, # row stride of scales_ptr and zp_ptr, in groups
stride_bn, # b_ptr row stride; >= K8, the rows may be padded
stride_am, # a_ptr row stride in elements; >= K, rows may be padded
# Quantization parameters
@@ -146,12 +159,12 @@ def _triton_w4a16_skinny_fmt_kernel(
b = (b >> shifts_full) & 0xF
group_idx = (k_start * BLOCK_K) // group_size
scale_ptrs = scales_ptr + offs_n * num_groups + group_idx
scale_ptrs = scales_ptr + offs_n * stride_sn + group_idx
scale_mask = offs_n < N
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
if HAS_ZP:
zp_ptrs = zp_ptr + (offs_n // 8) * num_groups + group_idx
zp_ptrs = zp_ptr + (offs_n // 8) * stride_sn + group_idx
zp_word = tl.load(zp_ptrs, mask=scale_mask, other=0)
zp_raw = (zp_word >> (4 * (offs_n % 8))) & 0xF
b_fp = (b - zp_raw[:, None]).to(scales.dtype) * scales[:, None]
@@ -216,7 +229,8 @@ def triton_w4a16_skinny_fmt_gemm(
"""
assert a.stride(1) == 1, "Activation rows must be contiguous"
assert b_q.stride(1) == 1, "Weight rows must be contiguous"
assert scales.is_contiguous(), "Scales must be contiguous"
assert scales.stride(1) == 1, "Scale rows must be contiguous"
stride_sn = scales.stride(0)
M, K = a.shape
N = b_q.shape[0]
@@ -230,7 +244,10 @@ def triton_w4a16_skinny_fmt_gemm(
f"scales shape mismatch: {scales.shape} vs ({N}, {num_groups})"
)
if zp is not None:
assert zp.is_contiguous(), "Zero-points must be contiguous"
assert zp.stride(1) == 1, "Zero-point rows must be contiguous"
assert zp.stride(0) == stride_sn, (
f"zp row stride {zp.stride(0)} must match scales {stride_sn}"
)
assert N % 8 == 0, f"N must be divisible by 8 for packed zp, got {N}"
assert zp.shape == (N // 8, num_groups), (
f"zp shape mismatch: {zp.shape} vs ({N // 8}, {num_groups})"
@@ -337,6 +354,7 @@ def triton_w4a16_skinny_fmt_gemm(
K,
K8,
num_groups,
stride_sn,
stride_bn,
stride_am,
group_size=group_size,
@@ -466,7 +484,9 @@ def _rdna_hybrid_w4a16_apply_impl(
K = x_2d.shape[1]
N = w_q.shape[0]
if M <= MAX_SKINNY_BATCH_SIZE and K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS:
if M <= MAX_SKINNY_BATCH_SIZE and (
K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS or _on_gfx115x()
):
# record_function is not torch.compile-safe; use nullcontext when
# compiling to keep the op traceable.
ctx = (
@@ -186,7 +186,7 @@ class XPUW4A8IntLinearKernel(MPLinearKernel):
# Free the original unpacked int8 weight (still registered as "weight")
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
layer.register_parameter("weight", None)
delattr(layer, "weight")
def apply_weights(
self,
@@ -62,7 +62,12 @@ if is_aiter_found_and_supported():
N = weight.shape[0]
K = weight.shape[1]
if rocm_use_aiter_fp4_asm_gemm:
if M <= 64 and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K):
# The preshuffle kernel's tuned configs split K in some M buckets.
if (
M <= 64
and not envs.VLLM_BATCH_INVARIANT
and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
):
if x_scales is None:
# use hip quant kernel for performance
if M >= 32:
@@ -116,7 +121,16 @@ if is_aiter_found_and_supported():
x_q.shape[0], weight.shape[0], device=x_q.device, dtype=out_dtype
)
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
config = None
if envs.VLLM_BATCH_INVARIANT:
# The configs split K in some M buckets (up to M <= 256 for
# tuned shapes). Keep the tile, drop the split.
from aiter.ops.triton.gemm_afp4wfp4 import _get_config
config, _ = _get_config(M, N, K)
config = dict(config, NUM_KSPLIT=1)
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y, config)
return y
def gemm_with_dynamic_quant_fake(
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
quant_key_to_input_schema,
)
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
@@ -55,12 +56,20 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
input_schema = quant_key_to_input_schema(self.config.activation_quant_key)
act_key = self.config.activation_quant_key
input_schema = quant_key_to_input_schema(act_key)
# Keep checkpoint-declared FP8 activations instead of falling back to
# A16 for group-scaled weights.
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_schema=input_schema
layer,
quant_config,
input_schema=input_schema,
allow_input_schema_fallback=(
act_key is None or act_key.dtype != current_platform.fp8_dtype()
),
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp6E2M3Static
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
@@ -66,7 +67,7 @@ class HummingMxFp6LinearKernel(MxFp6LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
@@ -57,7 +58,7 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
mxfp8_e4m3_quantize,
swizzle_mxfp8_scale,
xpu_mxfp8_quantize,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
_E8M0 = torch.float8_e8m0fnu
class TorchMxFp8LinearKernel(Mxfp8LinearKernel):
"""MXFP8 W8A8 GEMM using the native ``torch._scaled_mm`` dispatch.
Supported on XPU (oneDNN) and CUDA (cuBLASLt, SM90+).
"""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if current_platform.is_xpu():
return True, None
if current_platform.is_cuda() and current_platform.has_device_capability(90):
return True, None
return False, "requires XPU or CUDA (>=sm_90) for native torch._scaled_mm"
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# weight: [N, K] fp8_e4m3; weight_scale: [N, K//32] (E8M0-as-uint8).
weight = layer.weight.data # [N, K]
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
if current_platform.is_xpu():
# oneDNN: un-swizzled scales; store column-major weight (B operand).
replace_parameter(layer, "weight", weight.t())
weight_scale = weight_scale.t().contiguous().view(_E8M0)
else:
# CUDA: L4-padded SWIZZLE_32_4_4 scales; keep [N, K] weight and
# transpose to column-major at apply time.
replace_parameter(layer, "weight", weight.contiguous())
weight_scale = swizzle_mxfp8_scale(weight_scale, M=N, K=K).view(_E8M0)
replace_parameter(layer, "weight_scale", weight_scale)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
weight = layer.weight
weight_scale = layer.weight_scale
input_shape = x.shape
if current_platform.is_xpu():
# weight stored column-major [K, N].
K, N = weight.shape
mat_b = weight
input_2d = x.reshape(-1, K)
# xpu_mxfp8_quantize already returns float8_e8m0fnu, [M, K//32].
x_fp8, x_scale = xpu_mxfp8_quantize(input_2d)
else:
# weight stored [N, K]; transpose to column-major for _scaled_mm.
N, K = weight.shape
mat_b = weight.t()
input_2d = x.reshape(-1, K)
x_fp8, x_scale = mxfp8_e4m3_quantize(input_2d, is_sf_swizzled_layout=True)
x_scale = x_scale.view(_E8M0)
out = torch._scaled_mm(
x_fp8,
mat_b,
scale_a=x_scale,
scale_b=weight_scale,
bias=bias,
out_dtype=out_dtype,
)
return out.reshape(*input_shape[:-1], N)
@@ -12,6 +12,7 @@ instead, since ``can_implement`` does not filter on K.
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
@@ -246,7 +247,10 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
block_scale = _as_block32_scale(weight_scale)
# rocm_mxfp8_block32_gemm's split-K varies with M.
block_scale = (
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
)
if block_scale is not None:
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
# back from the loader expanded per row; keep the blocks and use
@@ -138,4 +138,64 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel):
)
__all__ = ["B12xNvFp4LinearKernel"]
class B12xNvFp4W4A16LinearKernel(B12xNvFp4LinearKernel):
"""BF16 x NVFP4 GEMM without activation quantization."""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
blockscaled = _import_b12x_blockscaled()
assert blockscaled is not None
layer.b12x_nvfp4_packed_weight = blockscaled.pack_weight(
layer.weight,
layer.weight_scale,
recipe="nvfp4",
global_scale=layer.weight_global_scale,
)
def get_b12x_warmup_unit(
self,
layer: torch.nn.Module,
token_counts: tuple[int, ...],
output_dtype: torch.dtype,
) -> B12xWarmupUnit:
def compile() -> None:
for tokens in token_counts:
source = torch.zeros(
(tokens, layer.weight.shape[1] * 2),
dtype=output_dtype,
device=layer.weight.device,
)
self.apply_weights(layer, source)
return B12xWarmupUnit(
name="NVFP4 W4A16",
key=(
type(self),
layer.weight.device,
tuple(layer.weight.shape),
output_dtype,
),
compile=compile,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if x.dtype != torch.bfloat16:
raise ValueError(f"b12x NVFP4 W4A16 requires BF16 input, got {x.dtype}")
blockscaled = _import_b12x_blockscaled()
assert blockscaled is not None
output = blockscaled.mm(
x.reshape(-1, x.shape[-1]).contiguous(),
layer.b12x_nvfp4_packed_weight,
required_mode="a16",
)
if bias is not None:
output = output + bias
return output.view(*x.shape[:-1], layer.weight.shape[0])
__all__ = ["B12xNvFp4LinearKernel", "B12xNvFp4W4A16LinearKernel"]
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
@@ -71,7 +72,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -39,6 +39,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFP8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
@@ -65,6 +66,7 @@ __all__ = [
"PerTensorTorchFP8ScaledMMLinearKernel",
"RowWiseTorchFP8ScaledMMLinearKernel",
"ROCmFP8ScaledMMLinearKernel",
"TritonFP8ScaledMMLinearKernel",
"TritonInt8ScaledMMLinearKernel",
"ZentorchInt8ScaledMMLinearKernel",
"Fp8BlockScaledMMLinearKernel",
@@ -104,7 +104,8 @@ class CutlassInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
# azp_adj is the AZP adjustment term, used to account for weights.
# It does not depend on scales or azp, so it is the same for
# static and dynamic quantization.
# For more details, see csrc/quantization/w8a8/cutlass/Epilogues.md
# For more details, see
# csrc/quantization/w8a8/cutlass/Epilogues.md
# https://github.com/vllm-project/vllm/blob/main/csrc/quantization/w8a8/cutlass/Epilogues.md
if not config.input_symmetric:
weight = getattr(layer, w_q_name)
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
prepare_humming_linear_layer_config,
)
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .ScaledMMLinearKernel import (
@@ -81,7 +82,7 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -148,7 +149,7 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -20,6 +20,8 @@ from .BlockScaledMMLinearKernel import (
)
from .cutlass import CutlassInt8ScaledMMLinearKernel
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
Int8ScaledMMLinearLayerConfig,
)
@@ -131,9 +133,7 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
x.contiguous(), i_s, i_zp, symmetric=symmetric
)
out = triton_scaled_mm(
x_q, w_q, scale_a=x_s, scale_b=w_s, out_dtype=x.dtype, bias=bias
)
out = torch.ops.vllm.int8_scaled_mm_func(x_q, w_q, x_s, w_s, x.dtype, bias)
if azp_adj is not None:
# Asymmetric quantization: subtract the zero-point correction.
@@ -156,6 +156,46 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
return out
class TritonFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda_alike():
return False, "requires ROCm or CUDA."
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
act_group = c.activation_quant_key.scale.group_shape
weight_group = c.weight_quant_key.scale.group_shape
if not (act_group.is_per_tensor() or act_group.is_per_token()):
return False, "requires per-tensor or per-token activation scales."
if not (
weight_group.is_per_tensor()
or weight_group.is_per_channel()
or weight_group.is_per_token()
):
return False, "requires per-tensor, per-channel or per-token weight scales."
return True, None
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
out = triton_scaled_mm(
A, B, scale_a=As, scale_b=Bs, out_dtype=out_dtype, bias=bias
)
return out.view(*output_shape)
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
@classmethod
def is_supported(cls, compute_capability=None):
@@ -180,6 +220,44 @@ class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
)
def _int8_scaled_mm_func(
qx: torch.Tensor,
weight: torch.Tensor,
x_scale: torch.Tensor,
weight_scale: torch.Tensor,
output_dtype: torch.dtype,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return triton_scaled_mm(
qx,
weight,
scale_a=x_scale,
scale_b=weight_scale,
out_dtype=output_dtype,
bias=bias,
)
def _int8_scaled_mm_fake(
qx: torch.Tensor,
weight: torch.Tensor,
x_scale: torch.Tensor,
weight_scale: torch.Tensor,
output_dtype: torch.dtype,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return torch.empty(
(qx.size(0), weight.size(1)), dtype=output_dtype, device=qx.device
)
direct_register_custom_op(
"int8_scaled_mm_func",
_int8_scaled_mm_func,
fake_impl=_int8_scaled_mm_fake,
)
# TODO we should be able to change the type of block_size to GroupShape
# after we resolve GroupShape compilation issue
# https://github.com/vllm-project/vllm/issues/25270
@@ -261,6 +261,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down, round_up
from vllm.utils.torch_utils import (
@@ -661,6 +662,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM120",
"FLASH_ATTN_MLA_SPARSE_FA4",
"DEEPSEEK_V32_INDEXER",
):
from vllm.v1.attention.backends.mla.compressor_utils import (
@@ -708,6 +710,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
use_pcp=self.use_pcp,
)
# TODO(ROCm): both BMM kernels re-quantize kv_b_proj online, ignoring
# the precision the checkpoint asked for, so they now default to off.
# Turning them back on needs long-context accuracy validation, and the
# per-layer decision belongs in --quantization-config.targets rather
# than in this layer.
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
@@ -751,7 +758,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
def _uses_flat_kv_cache(self) -> bool:
backend = self.attn_backend.get_name()
return backend == "FLASHINFER_MLA_SPARSE" or (
return backend in ("FLASHINFER_MLA_SPARSE", "FLASH_ATTN_MLA_SPARSE_FA4") or (
backend == "FLASHMLA_SPARSE"
and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla")
)
@@ -1376,14 +1383,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
is_index_group_leader=self.indexer is not None,
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
)
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
uses_tma_rows = (
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
and current_platform.is_device_capability_family(100)
)
if self._uses_flat_kv_cache() or uses_tma_rows:
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
return spec
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
@@ -1742,7 +1741,7 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
hf_text_config = model_config.hf_text_config
# Check if this is a DeepseekV4 config (uses unified head_dim + rope_head_dim)
if hasattr(hf_text_config, "compress_ratios"):
if get_compress_ratios(hf_text_config):
# DeepseekV4 style config: unified head_dim with rope_head_dim
head_dim = hf_text_config.head_dim
rope_head_dim = hf_text_config.qk_rope_head_dim
@@ -1865,6 +1864,9 @@ def _use_masked_mha(
seq_len: int,
has_context: bool,
) -> bool:
if backend_name == "FLASH_ATTN_MLA_SPARSE_FA4":
# Its prefill batches run on FlashInfer's kernel, so it shares those rows.
backend_name = "FLASHINFER_MLA_SPARSE"
thresholds = (
_MASKED_MHA_THRESHOLDS.get((qk_head_dim, v_head_dim), {})
.get(backend_name, {})
@@ -2548,16 +2550,26 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
"Non-causal multi-token MLA requires an explicitly supported "
"attention group."
)
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
num_active_reqs = int(torch.count_nonzero(query_lens > 0))
uniform_active_queries = num_active_reqs > 0 and bool(
torch.all(query_lens[:num_active_reqs] == query_lens[0])
# query_start_loc_cpu is host-side and holds one entry per
# request, so this check costs more in tensor dispatch than in
# the work it does. Read it out once and validate on Python ints:
# the block runs on every build, between decode steps.
qsl = query_start_loc_cpu.tolist()
query_lens = [qsl[i + 1] - qsl[i] for i in range(len(qsl) - 1)]
num_active_reqs = 0
for qlen in query_lens:
if qlen > 0:
num_active_reqs += 1
uniform_active_queries = num_active_reqs > 0 and all(
qlen == query_lens[0] for qlen in query_lens[:num_active_reqs]
)
trailing_graph_padding = all(
qlen == 0 for qlen in query_lens[num_active_reqs:]
)
trailing_graph_padding = bool(torch.all(query_lens[num_active_reqs:] == 0))
if not (uniform_active_queries and trailing_graph_padding):
raise ValueError(
"Non-causal MLA requires a uniform query block; got query "
f"lengths {query_lens.tolist()}."
f"lengths {query_lens}."
)
# Use exact GPU sequence lengths instead of the prefill path's CPU
# context-length upper bounds.
@@ -36,12 +36,18 @@ from vllm.utils.torch_utils import (
is_quantized_kv_cache,
np_to_pinned_tensor,
)
from vllm.v1.attention.backend import AttentionMetadataBuilder
from vllm.v1.attention.backend import AttentionLayer, AttentionMetadataBuilder
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
from vllm.v1.attention.backends.mla.index_group import (
HiSparseMLAIndexGroup,
SparseMLAIndexGroup,
SparseMLAIndexGroupBuilder,
)
from vllm.v1.attention.backends.mla.sparse_utils import (
flat_kv_row_view,
triton_convert_req_index_to_global_index,
triton_filter_and_convert_dcp_index,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
@@ -630,6 +636,9 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
"""Sparse MLA base with dense and masked-MHA prefill paths."""
is_sparse = True
# Reuse the index group's top-k conversion only for uniform decode batches
# of multi-layer groups; other batches convert per layer.
group_conversion_decode_only: ClassVar[bool] = False
def __init__(
self,
@@ -728,6 +737,162 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
return_valid_counts=return_valid_counts,
)
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: T,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
assert self.topk_indices_buffer is not None
num_tokens = q[0].shape[0] if isinstance(q, tuple) else q.shape[0]
topk = self.topk_indices_buffer[:num_tokens]
num_decode_tokens = attn_metadata.num_decode_tokens
block_size = attn_metadata.block_size
index_group = self.index_group
if not isinstance(index_group, HiSparseMLAIndexGroup):
_, block_stride_rows = flat_kv_row_view(kv_c_and_k_pe_cache, block_size)
if self.dcp_world_size > 1:
physical_topk, valid_counts = triton_filter_and_convert_dcp_index(
attn_metadata.req_id_per_token[:num_tokens],
attn_metadata.block_table,
topk,
dcp_size=self.dcp_world_size,
dcp_rank=self.dcp_rank,
cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size,
BLOCK_SIZE=block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk.shape[1],
return_valid_counts=True,
)
elif self.group_conversion_decode_only and not (
index_group is not None
and index_group.num_layers > 1
and num_decode_tokens >= num_tokens
):
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_tokens],
attn_metadata.block_table,
topk,
BLOCK_SIZE=block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=topk.shape[1],
return_valid_counts=True,
)
else:
physical_topk, valid_counts = self._convert_logical_to_physical_topk(
topk,
attn_metadata,
block_stride_rows=block_stride_rows,
return_valid_counts=True,
)
return self._forward_mqa_kernel(
q,
kv_c_and_k_pe_cache,
physical_topk,
valid_counts,
layer=layer,
block_size=block_size,
is_decode=num_decode_tokens >= num_tokens,
)
assert self.dcp_world_size == 1
layer_index = self.index_group_index
hot_cache = index_group.physical_kv_cache(layer_index).view(
kv_c_and_k_pe_cache.dtype
)
assert hot_cache.is_contiguous(), "HiSparse hot buffer must be contiguous"
if (
num_decode_tokens >= num_tokens
or index_group.cache(layer_index).all_context_pages_resident
):
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
layer_index,
topk,
attn_metadata,
block_stride_rows=None,
return_valid_counts=True,
)
return self._forward_mqa_kernel(
q,
hot_cache,
physical_topk,
valid_counts,
layer=layer,
block_size=block_size,
is_decode=num_decode_tokens >= num_tokens,
)
decode_out = decode_lse = None
if num_decode_tokens:
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
layer_index,
topk[:num_decode_tokens],
attn_metadata,
block_stride_rows=None,
return_valid_counts=True,
)
q_decode = (
(q[0][:num_decode_tokens], q[1][:num_decode_tokens])
if isinstance(q, tuple)
else q[:num_decode_tokens]
)
decode_out, decode_lse = self._forward_mqa_kernel(
q_decode,
hot_cache,
physical_topk,
valid_counts,
layer=layer,
block_size=block_size,
is_decode=True,
)
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
layer_index, kv_c_and_k_pe_cache, attn_metadata
)
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
req_ids,
block_table,
topk[num_decode_tokens:],
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=topk.shape[1],
return_valid_counts=True,
)
q_prefill = (
(q[0][num_decode_tokens:], q[1][num_decode_tokens:])
if isinstance(q, tuple)
else q[num_decode_tokens:]
)
prefill_out, prefill_lse = self._forward_mqa_kernel(
q_prefill,
prefill_cache,
physical_topk,
valid_counts,
layer=layer,
block_size=block_size,
is_decode=False,
)
if decode_out is None:
return prefill_out, prefill_lse
out = torch.cat((decode_out, prefill_out))
if decode_lse is None:
return out, None
assert prefill_lse is not None
return out, torch.cat((decode_lse, prefill_lse))
def _forward_mqa_kernel(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_cache: torch.Tensor,
topk_indices: torch.Tensor,
valid_counts: torch.Tensor,
*,
layer: AttentionLayer,
block_size: int,
is_decode: bool,
) -> tuple[torch.Tensor, torch.Tensor | None]:
raise NotImplementedError
@staticmethod
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
"""Whether this prefill batch's top-k masks fit the workspace."""
@@ -106,7 +106,11 @@ if HAS_TRITON:
BatchedTritonExperts,
)
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterFp8BlockExperts,
AiterFp8ChannelExperts,
AiterFp8TensorExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
@@ -132,7 +136,11 @@ if HAS_TRITON:
)
__all__ += [
"AiterExperts",
"AiterUnquantizedExperts",
"AiterFp8BlockExperts",
"AiterFp8TensorExperts",
"AiterFp8ChannelExperts",
"AiterMxfp4Experts",
"fused_topk",
"fused_experts",
"get_config_file_name",
@@ -329,14 +329,24 @@ def maybe_make_prepare_finalize(
elif moe.use_mori_kernels:
assert quant_config is not None
# MXFP4 activations are dispatched as packed FP4 with their e8m0
# block scales, which the experts consume without re-quantizing.
use_fp4_dispatch = quant_config.use_mxfp4_w4a4
# Note: We may want to use FP8 dispatch just to reduce
# data movement.
use_fp8_dispatch = (
use_fp8_dispatch = not use_fp4_dispatch and (
quant_config.is_per_act_token
or quant_config.is_block_quantized
or quant_config.is_per_tensor
)
if use_fp8_dispatch:
if use_fp4_dispatch:
# One e8m0 scale per 32-element block. token_hidden_size stays
# the unpacked hidden size: mori sizes its token buffers by the
# BF16 input type, which also covers the BF16 combine.
quant_dtype = torch.float4_e2m1fn_x2
scale_dim = moe.hidden_dim // 32
scale_type_size = torch.float8_e8m0fnu.itemsize
elif use_fp8_dispatch:
# For PTPC (per token per channel) or per-tensor quant,
# scale dim is 1. For 1x128 quant, scale dim is
# hidden_dim // 128
@@ -346,18 +356,20 @@ def maybe_make_prepare_finalize(
if (quant_config.is_per_act_token or quant_config.is_per_tensor)
else moe.hidden_dim // 128
)
scale_type_size = torch.float32.itemsize
else:
# Unquantized dispatch (e.g. AITER with defer_input_quant):
# dispatch raw BF16/FP16 data, no scales needed.
quant_dtype = moe.in_dtype
scale_dim = 0
scale_type_size = 0
all_to_all_args = dict(
rank=all2all_manager.rank,
num_ep_ranks=all2all_manager.world_size,
quant_dtype=quant_dtype,
token_hidden_size=moe.hidden_dim,
scale_dim=scale_dim,
scale_type_size=0 if scale_dim == 0 else torch.float32.itemsize,
scale_type_size=scale_type_size,
max_num_tokens_per_dp_rank=moe.max_num_tokens,
input_dtype=moe.in_dtype,
num_local_experts=moe.num_experts // all2all_manager.world_size,
@@ -370,6 +382,7 @@ def maybe_make_prepare_finalize(
max_tokens_per_rank=moe.max_num_tokens,
num_dispatchers=all2all_manager.world_size,
use_fp8_dispatch=use_fp8_dispatch,
use_fp4_dispatch=use_fp4_dispatch,
)
elif moe.use_fi_nvl_two_sided_kernels:
+42 -7
View File
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.b12x import (
B12X_BLOCK_CODECS,
B12xWarmupUnit,
get_b12x_fused_moe,
reuse_packed_weight_storage,
@@ -38,6 +39,7 @@ _B12X_MOE_MODES: dict[
tuple[torch.dtype | str | None, torch.dtype | str | None],
tuple[str, str, str],
] = {
**{(codec, None): ("w4a16", codec, "w31") for codec in B12X_BLOCK_CODECS},
("mxfp4", "mxfp8"): ("w4a8_mx", "fp4_e8m0_k32", "w31"),
("mxfp4", None): ("w4a16", "fp4_e8m0_k32", "w31"),
("nvfp4", "nvfp4"): ("nvfp4", "modelopt_nvfp4", "w31"),
@@ -206,10 +208,13 @@ class B12xExperts(mk.FusedMoEExpertsModular):
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
if quant_config.weight_quant_dtype not in ("mxfp4", "nvfp4"):
if quant_config.weight_quant_dtype not in (
"mxfp4",
"nvfp4",
*B12X_BLOCK_CODECS,
):
raise ValueError(
"b12x MoE requires MXFP4 or NVFP4 weights, got "
f"{quant_config.weight_quant_dtype}"
f"unsupported b12x MoE weight format: {quant_config.weight_quant_dtype}"
)
scheme = (
quant_config.weight_quant_dtype,
@@ -293,6 +298,31 @@ class B12xExperts(mk.FusedMoEExpertsModular):
raise RuntimeError(
"b12x MoE weights must be prepared before CUDA graph capture"
)
if self._source_format in B12X_BLOCK_CODECS:
fused_moe = _require_b12x_fused_moe()
block_size, _ = B12X_BLOCK_CODECS[self._source_format]
plan = fused_moe.plan_weights(
source=fused_moe.PackedSource(
format=self._source_format, w13_layout=self._w13_layout
),
activation=fused_moe.ActivationSpec(
mode="a16",
nonlinearity=_b12x_activation_name(activation),
io_dtype=params_dtype,
),
geometry=fused_moe.MoEGeometry(
num_experts=w1.shape[0],
hidden_size=w2.shape[1],
intermediate_size=w2.shape[2] * block_size,
),
)
prepared = fused_moe.prepare_weights(
plan=plan,
weights=fused_moe.BlockQuantWeights(
w13=w1, w2=w2, codec=self._source_format
),
)
return prepared._impl
if self.w1_scale is None or self.w2_scale is None:
raise ValueError("b12x MoE requires w1 and w2 block scales")
@@ -345,6 +375,8 @@ class B12xExperts(mk.FusedMoEExpertsModular):
)
def _refresh_quant_config(self, layer: torch.nn.Module) -> None:
if self._source_format in B12X_BLOCK_CODECS:
return
self.quant_config._w1.scale = layer.w13_weight_scale
self.quant_config._w2.scale = layer.w2_weight_scale
if self._source_format != "modelopt_nvfp4":
@@ -446,8 +478,10 @@ class B12xExperts(mk.FusedMoEExpertsModular):
):
return (
False,
"MXFP4 W4A8 requires hidden size divisible by 256 and "
"per-rank intermediate size divisible by 32",
(
"MXFP4 W4A8 requires hidden size divisible by 256 and "
"per-rank intermediate size divisible by 32"
),
)
return mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
@@ -530,14 +564,15 @@ class B12xExperts(mk.FusedMoEExpertsModular):
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
if w1.numel() and w2.numel():
if w1.numel() and w2.numel() and self._source_format not in B12X_BLOCK_CODECS:
return super().moe_problem_size(a1, w1, w2, topk_ids)
prepared = self._prepared()
tokens = int(a1.shape[0] if a1.ndim == 2 else a1.shape[1])
return (
int(prepared.num_experts),
tokens,
int(prepared.intermediate_size) * 2,
int(prepared.intermediate_size)
* (2 if self.moe_config.activation.is_gated else 1),
int(a1.shape[-1]),
int(topk_ids.shape[1]),
)
@@ -1293,6 +1293,8 @@ class FusedMoEConfig:
has_bias: bool = False
is_lora_enabled: bool = False
has_hash_routing: bool = False
# Routes are adjusted between routing and expert execution.
require_decomposed_backend: bool = False
shared_expert_prefix: str | None = None
# When True, the MoE skips its final cross-rank all-reduce (and the separate
@@ -18,19 +18,15 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
def ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
"""Reinterpret a uint8 UE8M0 exponent tensor as float32 scale values."""
return (sf.to(torch.int32) << 23).view(torch.float32)
class DeepGemmMegaMoEBackend:
"""Abstract MegaMoE backend that hides deep_gemm-specific details.
A backend advertises its expected hidden-state quantization layout via
``hidden_quant`` and its expected MMA type via ``mma_type``. It is
responsible for the routed-expert weight transform run at load time, the
optional shared-expert weight transform, and the actual MegaMoE kernel
invocation at inference time. Runtime capability checks live in
``hidden_quant`` and its expected MMA type via ``mma_type``. It owns the
kernel weight layout: :meth:`create_weights` registers parameters in that
layout and :meth:`load_weight` writes each checkpoint tensor into it, so
loading (and reloading) needs no post-load transform. It also runs the
MegaMoE kernel. Runtime capability checks live in
:func:`get_deep_gemm_mega_moe_backend`: obtaining a backend already implies
that the current device and shapes are supported.
"""
@@ -39,49 +35,39 @@ class DeepGemmMegaMoEBackend:
mma_type: str = "fp8xfp4"
hidden_quant: QuantKey = kMxfp8Dynamic
def transform_weights(
def create_weights(
self,
layer: nn.Module,
*,
w13_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w2_weight: torch.Tensor,
w2_weight_scale: torch.Tensor,
num_local_experts: int,
hidden_size: int,
intermediate_size: int,
activation: str | None = None,
) -> tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]:
num_shared_experts: int,
) -> None:
"""Register the expert (and fused shared-expert) kernel-layout parameters:
``w13_weight``, ``w13_weight_scale``, ``w2_weight``, ``w2_weight_scale``
and, if ``num_shared_experts``, ``shared_l{1,2}_weight[_scale]``. Routed
parameters are indexed by local expert on dim 0, which EPLB moves."""
raise NotImplementedError
def supports_shared_experts(self, deep_gemm) -> bool:
return False
def transform_shared_expert_weights(
def load_weight(
self,
*,
shared_experts: nn.Module,
num_shared_experts: int,
hidden_size: int,
intermediate_size: int,
prefix: str,
) -> (
tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]
| None
):
"""Return ``(l1, l2)`` transformed shared-expert weights, or ``None``.
dst: torch.Tensor,
src: torch.Tensor,
shard_id: str,
is_scale: bool,
) -> None:
"""Write checkpoint shard ``src`` (``w1``/``w2``/``w3``) into ``dst``, one
expert's slice or a shared-expert parameter, in the kernel layout."""
raise NotImplementedError
``shared_experts`` must expose ``gate_up_proj`` and ``down_proj``
linear layers. A ``None`` return means the backend cannot fuse shared
experts for this module; the caller should fall back to a separate
shared MLP.
def supports_shared_experts(self, shared_experts: nn.Module) -> bool:
"""Whether ``shared_experts`` can be fused into the MegaMoE kernel.
Decided at construction from the installed kernel API and the shared
MLP's parameters, so the fused layout is fixed before any weight loads.
"""
return None
return False
def run_mega_moe(
self,
@@ -133,229 +119,153 @@ class DeepGemmSm100MegaMoEBackend(DeepGemmMegaMoEBackend):
mma_type = "fp8xfp4"
hidden_quant = kMxfp8Dynamic
def transform_weights(
def create_weights(
self,
layer: nn.Module,
*,
w13_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w2_weight: torch.Tensor,
w2_weight_scale: torch.Tensor,
num_local_experts: int,
hidden_size: int,
intermediate_size: int,
activation: str | None = None,
) -> tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]:
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
w13_scale = deep_gemm.transform_sf_into_required_layout(
ue8m0_uint8_to_float(w13_weight_scale).contiguous(),
2 * intermediate_size,
hidden_size,
(1, 32),
num_local_experts,
)
w2_scale = deep_gemm.transform_sf_into_required_layout(
ue8m0_uint8_to_float(w2_weight_scale).contiguous(),
hidden_size,
intermediate_size,
(1, 32),
num_local_experts,
)
kwargs = {} if activation is None else {"activation": activation}
return deep_gemm.transform_weights_for_mega_moe(
(w13_weight.view(torch.int8).contiguous(), w13_scale),
(w2_weight.view(torch.int8).contiguous(), w2_scale),
**kwargs,
)
def supports_shared_experts(self, deep_gemm) -> bool:
"""Check the Python API before touching a symmetric-memory group.
This also gives users of an older precompiled vLLM wheel a safe serial
fallback instead of failing halfway through multi-rank buffer setup.
"""
try:
buffer_params = signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters
kernel_params = signature(deep_gemm.fp8_fp4_mega_moe).parameters
except (TypeError, ValueError):
return False
return (
hasattr(deep_gemm, "get_block_m_for_mega_moe")
and hasattr(deep_gemm, "transform_weights_for_mega_moe")
and "num_shared_experts" in buffer_params
and "shared_l1_weights" in kernel_params
and "shared_l2_weights" in kernel_params
)
def transform_shared_expert_weights(
self,
*,
shared_experts: nn.Module,
num_shared_experts: int,
hidden_size: int,
intermediate_size: int,
prefix: str,
) -> (
tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]
| None
):
from vllm.utils.deep_gemm import _import_deep_gemm
) -> None:
def register(name: str, data: torch.Tensor) -> None:
layer.register_parameter(name, nn.Parameter(data, requires_grad=False))
deep_gemm = _import_deep_gemm()
gate_up = shared_experts.gate_up_proj
down = shared_experts.down_proj
gate_up_weight = gate_up.weight.data
gate_up_scale = (
gate_up.weight_scale
if hasattr(gate_up, "weight_scale")
else gate_up.weight_scale_inv
).data
down_weight = down.weight.data
down_scale = (
down.weight_scale
if hasattr(down, "weight_scale")
else down.weight_scale_inv
).data
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
# while the checkpoint uses coarser block-FP8 scales (usually
# 128x128). Build a dedicated, numerically equivalent scale view.
ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32)
if (
gate_up_scale.dtype in ue8m0_scale_dtypes
and down_scale.dtype in ue8m0_scale_dtypes
):
gate_up_scale = self._prepare_shared_expert_scale(
deep_gemm,
gate_up,
gate_up_scale,
gate_up_weight.shape[0],
gate_up_weight.shape[1],
prefix,
)
down_scale = self._prepare_shared_expert_scale(
deep_gemm,
down,
down_scale,
down_weight.shape[0],
down_weight.shape[1],
prefix,
)
if gate_up_scale is None or down_scale is None:
return None
shared_intermediate_size = intermediate_size * num_shared_experts
expected_gate_up_shape = (
2 * shared_intermediate_size,
hidden_size,
# FP4 weights two per byte; UE8M0 scales MN-major, four per int32 along K.
e, h, i = num_local_experts, hidden_size, intermediate_size
register("w13_weight", torch.zeros(e, 2 * i, h // 2, dtype=torch.int8))
register(
"w13_weight_scale",
torch.zeros(e, h // 128, 2 * i, dtype=torch.int32).transpose(1, 2),
)
expected_down_shape = (hidden_size, shared_intermediate_size)
if (
gate_up_weight.dtype != torch.float8_e4m3fn
or down_weight.dtype != torch.float8_e4m3fn
or gate_up_scale.dtype != torch.int32
or down_scale.dtype != torch.int32
or tuple(gate_up_weight.shape) != expected_gate_up_shape
or tuple(down_weight.shape) != expected_down_shape
):
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: expected "
"replicated block-FP8 weights with gate_up=%s, down=%s, and "
"DeepGEMM int32 scales; got gate_up=%s/%s/%s and down=%s/%s/%s.",
prefix,
expected_gate_up_shape,
expected_down_shape,
tuple(gate_up_weight.shape),
gate_up_weight.dtype,
gate_up_scale.dtype,
tuple(down_weight.shape),
down_weight.dtype,
down_scale.dtype,
)
return None
transformed_l1, transformed_l2 = deep_gemm.transform_weights_for_mega_moe(
(gate_up_weight, gate_up_scale),
(down_weight, down_scale),
register("w2_weight", torch.zeros(e, h, i // 2, dtype=torch.int8))
register(
"w2_weight_scale",
torch.zeros(e, i // 128, h, dtype=torch.int32).transpose(1, 2),
)
# L1 interleaving allocates a full copy. Re-home the loader Parameter on
# that storage so the original 2*intermediate*hidden FP8 tensor can be
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
# The generic linear post-load hook may still repack the serial scales,
# but this shared MLP is never called after native fusion is enabled.
gate_up.weight.data = transformed_l1[0]
transformed_l1 = (gate_up.weight.data, transformed_l1[1])
return transformed_l1, transformed_l2
if num_shared_experts == 0:
return
# The shared expert is FP8 with the layout of one routed expert slice.
s, fp8 = i * num_shared_experts, torch.float8_e4m3fn
register("shared_l1_weight", torch.zeros(2 * s, h, dtype=fp8))
register(
"shared_l1_weight_scale",
torch.zeros(h // 128, 2 * s, dtype=torch.int32).transpose(0, 1),
)
register("shared_l2_weight", torch.zeros(h, s, dtype=fp8))
register(
"shared_l2_weight_scale",
torch.zeros(s // 128, h, dtype=torch.int32).transpose(0, 1),
)
def load_weight(
self,
dst: torch.Tensor,
src: torch.Tensor,
shard_id: str,
is_scale: bool,
) -> None:
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"Unsupported expert shard id: {shard_id}")
rows = dst.shape[0] // (1 if shard_id == "w2" else 2)
if is_scale:
src = self._pack_scale(src, rows, dst.shape[1] * 4)
else:
src = src.view(dst.dtype)
if tuple(src.shape) != (rows, dst.shape[1]):
raise ValueError(
f"MegaMoE {shard_id} weight shape {tuple(src.shape)} does not "
f"match the kernel shard ({rows}, {dst.shape[1]})"
)
dst, src = self._kernel_layout_shard(dst, src, shard_id, is_scale)
dst.copy_(src)
@staticmethod
def _prepare_shared_expert_scale(
deep_gemm,
linear: nn.Module,
scale: torch.Tensor,
mn: int,
k: int,
prefix: str,
) -> torch.Tensor | None:
block_size = getattr(linear, "weight_block_size", None)
if block_size is None or len(block_size) != 2:
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: "
"shared FP8 weight block size is unavailable.",
prefix,
def _pack_scale(sf: torch.Tensor, rows: int, cols: int) -> torch.Tensor:
"""Expand block UE8M0 scales to one per 1x32 and pack four along K per int32."""
sf = sf.view(torch.uint8)
if rows % sf.shape[0] or cols % sf.shape[1]:
raise ValueError(
f"Scale shape {tuple(sf.shape)} does not tile ({rows}, {cols})"
)
return None
sf = sf.repeat_interleave(rows // sf.shape[0], 0)
sf = sf.repeat_interleave(cols // sf.shape[1], 1)
return sf.contiguous().clone().view(torch.int32)
block_m, block_k = block_size
num_k_blocks = (k + block_k - 1) // block_k
# The linear hook's DeepGEMM layout packs four k-block bytes per int32.
packed = scale.dtype == torch.int32
expected_shape = (
(mn, (num_k_blocks + 3) // 4)
if packed
else ((mn + block_m - 1) // block_m, num_k_blocks)
)
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: "
"cannot convert shared scale shape %s with block size %s "
"to MegaMoE's 1x32 layout for weight (%d, %d).",
prefix,
tuple(scale.shape),
tuple(block_size),
mn,
k,
@staticmethod
def _kernel_layout_shard(
dst: torch.Tensor,
src: torch.Tensor,
shard_id: str,
is_scale: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pair a checkpoint shard with its destination in the MegaMoE layout."""
k = src.shape[-1]
if shard_id == "w2":
if not is_scale:
return dst, src
# UTCCP: scale row 128*b + 32*j + i -> 128*b + 4*i + j.
return dst.view(-1, 32, 4, k).transpose(1, 2), src.reshape(-1, 4, 32, k)
half = 0 if shard_id == "w1" else 1
if not is_scale:
# Interleave gate/up by 8 rows: r -> (r // 8) * 16 + 8 * half + r % 8.
return dst.view(-1, 2, 8, k)[:, half], src.reshape(-1, 8, k)
# Interleave + UTCCP: 64b + 16j + 8q + m -> 128b + 64q + 32half + 4m + j.
dst = dst.view(-1, 2, 2, 8, 4, k)[:, :, half].permute(0, 3, 1, 2, 4)
return dst, src.reshape(-1, 4, 2, 8, k)
def supports_shared_experts(self, shared_experts: nn.Module) -> bool:
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
weight = linear.weight
scale = getattr(linear, "weight_scale", None)
if scale is None:
scale = getattr(linear, "weight_scale_inv", None)
if not self._tiles_1x32(weight, scale):
logger.warning_once(
"Disabling native MegaMoE shared-expert fusion: needs FP8 "
"weights with UE8M0 block scales that tile to 1x32; got "
"weight %s/%s and scale %s/%s.",
tuple(weight.shape),
weight.dtype,
None if scale is None else tuple(scale.shape),
None if scale is None else scale.dtype,
)
return False
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
try:
api = {
*signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters,
*signature(deep_gemm.fp8_fp4_mega_moe).parameters,
}
except (AttributeError, TypeError, ValueError):
api = set()
if not (
hasattr(deep_gemm, "get_block_m_for_mega_moe")
and {"num_shared_experts", "shared_l1_weights", "shared_l2_weights"} <= api
):
logger.warning_once(
"Disabling native MegaMoE shared-expert fusion because the "
"installed DeepGEMM Python API is older than the vLLM "
"source. Rebuild the vendored _deep_gemm_C extension to enable it.",
)
return None
return False
return True
if packed:
row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks]
else:
row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn]
scale_1x32 = (
ue8m0_uint8_to_float(row_scale)
.repeat_interleave(block_k // 32, dim=1)[:, : k // 32]
.contiguous()
@staticmethod
def _tiles_1x32(weight: torch.Tensor, scale: torch.Tensor | None) -> bool:
if scale is None or weight.dim() != 2 or scale.dim() != 2:
return False
(rows, cols), (scale_rows, scale_cols) = weight.shape, scale.shape
return (
weight.dtype == torch.float8_e4m3fn
and scale.dtype in (torch.uint8, torch.float8_e8m0fnu)
and rows % scale_rows == 0
and cols % scale_cols == 0
and (cols // scale_cols) % 32 == 0
)
# The grouped API is used with a singleton dimension to request the
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
# 2D layout required for a shared expert.
return deep_gemm.transform_sf_into_required_layout(
scale_1x32.unsqueeze(0),
mn,
k,
(1, 32),
1,
).squeeze(0)
def run_mega_moe(
self,
@@ -0,0 +1,380 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Constant expert substitution for MoNE-pruned checkpoints.
MoNE (https://arxiv.org/abs/2507.00390) replaces redundant experts with
"novices" that return a constant vector. Checkpoints list the replaced experts
per decoder layer either in ``config.approximate_experts``, with constants named
``<experts prefix>.<expert id>.approx_value``, or in
``compression_config.transform_config.expert_substitution``, with explicitly
named constants. Retained experts keep their logical checkpoint names and are
loaded into compact physical rows.
"""
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
import torch
from torch import nn
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
APPROX_VALUE_SUFFIX = ".approx_value"
def decoder_layer_index(path: str) -> int | None:
"""Return the only integer component of a module path, if unique."""
indices = [int(part) for part in path.split(".") if part.isdigit()]
return indices[0] if len(indices) == 1 else None
@dataclass(frozen=True)
class ExpertSubstitutionSpec:
# Sorted substituted expert IDs per decoder layer.
experts: dict[int, tuple[int, ...]]
# Explicitly named constant tensors and the (layer, expert) pairs they
# supply. Without an entry, constants use the ``approx_value`` naming.
value_names: dict[str, tuple[tuple[int, int], ...]] = field(default_factory=dict)
# The only router semantics that the constant-v1 format supports.
_ROUTER_SEMANTICS = {
"preserve_logical_expert_ids": True,
"preserve_router_weights": True,
"renormalize_after_substitution": False,
}
def _expert_ids(layer_idx: int, expert_ids: Any) -> tuple[int, ...]:
ids = tuple(sorted(int(expert_id) for expert_id in expert_ids or ()))
if len(set(ids)) != len(ids) or any(expert_id < 0 for expert_id in ids):
raise ValueError(
f"substituted experts of layer {layer_idx} must be unique, "
f"non-negative expert IDs, got {list(ids)}"
)
return ids
def _parse_approximate_experts(raw: Any) -> ExpertSubstitutionSpec:
if not isinstance(raw, Mapping):
raise ValueError("approximate_experts must map layer indices to expert IDs")
experts: dict[int, tuple[int, ...]] = {}
for layer, expert_ids in raw.items():
try:
layer_idx = int(layer)
experts[layer_idx] = _expert_ids(layer_idx, expert_ids)
except (TypeError, ValueError) as e:
raise ValueError(
f"approximate_experts has an invalid entry for layer {layer!r}"
) from e
return ExpertSubstitutionSpec({k: v for k, v in experts.items() if v})
def _parse_expert_substitution(raw: Any) -> ExpertSubstitutionSpec:
experts: dict[int, tuple[int, ...]] = {}
value_names: dict[str, list[tuple[int, int]]] = {}
try:
if raw["version"] != 1 or raw["router_semantics"] != _ROUTER_SEMANTICS:
raise ValueError("only version 1 with unchanged routing is supported")
for path, target in raw["targets"].items():
layer_idx = decoder_layer_index(path)
if layer_idx is None or layer_idx in experts:
raise ValueError(f"target {path!r} must name one decoder layer")
if target["weight_layout"] != "compact_retained_experts":
raise ValueError(f"unsupported weight layout for {path!r}")
for expert_id, replacement in target["replacements"].items():
if replacement["format"] != "constant-v1":
raise ValueError(f"unsupported replacement format for {path!r}")
value_names.setdefault(replacement["tensors"]["value"], []).append(
(layer_idx, int(expert_id))
)
experts[layer_idx] = _expert_ids(layer_idx, target["replacements"])
except (AttributeError, KeyError, TypeError, ValueError) as e:
raise ValueError(f"invalid expert_substitution metadata: {e}") from e
return ExpertSubstitutionSpec(
experts, {name: tuple(pairs) for name, pairs in value_names.items()}
)
def get_expert_substitution_spec(model_config: Any) -> ExpertSubstitutionSpec | None:
"""Parse the substituted experts declared in a model's HF config."""
hf_configs = [
getattr(model_config, name, None) for name in ("hf_text_config", "hf_config")
]
approximate = None
for config in hf_configs:
approximate = getattr(config, "approximate_experts", None)
if approximate is not None:
break
compression_config = getattr(hf_configs[1], "compression_config", None)
transform_config = (
compression_config.get("transform_config")
if isinstance(compression_config, Mapping)
else None
)
declared = (
transform_config.get("expert_substitution")
if isinstance(transform_config, Mapping)
else None
)
if approximate is not None and declared is not None:
raise ValueError(
"config declares both approximate_experts and "
"compression_config.transform_config.expert_substitution"
)
if approximate is not None:
spec = _parse_approximate_experts(approximate)
elif declared is not None:
spec = _parse_expert_substitution(declared)
else:
return None
return spec if spec.experts else None
class ConstantExpertSubstitution(nn.Module):
"""Constant outputs for the substituted experts of one MoE layer.
Router outputs keep logical expert IDs. ``transform_routes`` maps retained
experts to compact physical rows and turns substituted routes into
zero-weight routes to physical expert 0, so any decomposed MoE backend can
run them unchanged.
"""
def __init__(
self,
layer_idx: int,
num_logical_experts: int,
substituted_expert_ids: tuple[int, ...],
hidden_size: int,
params_dtype: torch.dtype,
) -> None:
super().__init__()
substituted = set(substituted_expert_ids)
if not substituted <= set(range(num_logical_experts)):
raise ValueError(
f"approximate_experts for layer {layer_idx} must be within "
f"[0, {num_logical_experts}), got {sorted(substituted)}"
)
if len(substituted) == num_logical_experts:
raise ValueError(
f"approximate_experts for layer {layer_idx} must retain at least "
"one expert"
)
self.layer_idx = layer_idx
self.substituted_expert_ids = tuple(sorted(substituted))
retained = [i for i in range(num_logical_experts) if i not in substituted]
self.num_compute_experts = len(retained)
# Retained experts map to their physical row r >= 0, substituted
# experts to -1 - (their row in `values`).
routes = [0] * num_logical_experts
for row, expert_id in enumerate(retained):
routes[expert_id] = row
for row, expert_id in enumerate(self.substituted_expert_ids):
routes[expert_id] = -1 - row
self._routes = tuple(routes)
self.register_buffer(
"expert_substitution_routes",
torch.tensor(routes, dtype=torch.int32),
persistent=False,
)
# NaN marks values that the checkpoint has not provided yet.
self.values = nn.Parameter(
torch.full((len(substituted), hidden_size), torch.nan, dtype=params_dtype),
requires_grad=False,
)
self.values.weight_loader = self.weight_loader
@property
def num_logical_experts(self) -> int:
return len(self._routes)
def physical_expert_id(self, expert_id: int) -> int:
"""Physical row of a retained expert, or -1 for a substituted one."""
return max(self._routes[expert_id], -1)
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, expert_id: int
) -> None:
if not 0 <= expert_id < self.num_logical_experts or (
self._routes[expert_id] >= 0
):
raise ValueError(
f"expert {expert_id} of layer {self.layer_idx} is not listed in "
"approximate_experts"
)
row = param.data[-1 - self._routes[expert_id]]
if loaded_weight.shape != row.shape:
raise ValueError(
f"approx_value for expert {expert_id} of layer {self.layer_idx} has "
f"shape {tuple(loaded_weight.shape)}, expected {tuple(row.shape)}"
)
row.copy_(loaded_weight)
def validate_loaded_values(self, prefix: str) -> None:
missing_rows = torch.isnan(self.values).any(dim=1).nonzero().flatten().tolist()
if missing_rows:
missing = [self.substituted_expert_ids[row] for row in missing_rows]
raise ValueError(
f"{prefix} is missing approx_value for {len(missing)} "
f"expert(s), first missing expert IDs: {missing[:8]}"
)
def transform_routes(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Remap routes in place and return the constant experts' output."""
valid = (topk_ids >= 0) & (topk_ids < self.num_logical_experts)
routes = self.expert_substitution_routes[
torch.where(valid, topk_ids, torch.zeros_like(topk_ids)).long()
].long()
substituted = valid & (routes < 0)
# Sum each token's router weights per substituted expert in FP32; other
# routes land in a trailing column that is dropped.
num_values, value_size = self.values.shape
scales = topk_weights.new_zeros(
(topk_ids.shape[0], num_values + 1), dtype=torch.float32
)
scales.scatter_add_(
1,
torch.where(substituted, -1 - routes, num_values),
topk_weights.float(),
)
output = torch.zeros_like(hidden_states)
output[..., :value_size] = scales[:, :num_values] @ self.values.float()
topk_weights.masked_fill_(~valid | substituted, 0.0)
topk_ids.copy_(routes.clamp_min(0).to(topk_ids.dtype))
return output
def make_expert_substitution(
model_config: Any,
prefix: str,
num_experts: int,
hidden_size: int,
params_dtype: torch.dtype | None,
) -> ConstantExpertSubstitution | None:
spec = get_expert_substitution_spec(model_config)
if spec is None:
return None
layer_idx = decoder_layer_index(prefix)
if layer_idx is None:
raise ValueError(
f"cannot match MoE layer {prefix!r} to substituted experts: expected "
"exactly one layer index in its prefix"
)
expert_ids = spec.experts.get(layer_idx)
if expert_ids is None:
return None
return ConstantExpertSubstitution(
layer_idx,
num_experts,
expert_ids,
hidden_size,
params_dtype or torch.get_default_dtype(),
)
class SubstitutedRoutedExperts(RoutedExperts):
"""Routed experts that hold weights only for retained experts."""
def __init__(
self, *args: Any, expert_substitution: ConstantExpertSubstitution, **kwargs: Any
):
super().__init__(*args, **kwargs)
self.expert_substitution = expert_substitution
moe_config = self.moe_config
parallel_config = moe_config.moe_parallel_config
unsupported = [
name
for name, enabled in (
(
"quantized MoE weights",
not isinstance(self.quant_method, UnquantizedFusedMoEMethod),
),
("monolithic MoE backends", self.quant_method.is_monolithic),
("MoE LoRA", moe_config.is_lora_enabled),
(
"EPLB",
parallel_config.enable_eplb
or moe_config.num_experts
!= expert_substitution.num_compute_experts,
),
(
"fused shared experts",
self.expert_map_manager.num_fused_shared_experts > 0,
),
("expert parallelism", parallel_config.use_ep),
("data parallelism", parallel_config.dp_size > 1),
("prefill context parallelism", parallel_config.pcp_size > 1),
("sequence parallelism", parallel_config.is_sequence_parallel),
("router weights on expert inputs", self.apply_router_weight_on_input),
)
if enabled
]
if unsupported:
raise NotImplementedError(
"expert substitution does not support: " + ", ".join(unsupported)
)
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
# Checkpoints name retained experts by their logical ID.
physical_id = self.expert_substitution.physical_expert_id(expert_id)
if physical_id < 0:
return -1
return super()._map_global_expert_id_to_local_expert_id(physical_id)
def get_expert_mapping(
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
# Fused checkpoint tensors would cover every logical expert at once.
return self.build_expert_params_mapping(
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
ckpt_down_proj_name or self.ckpt_down_proj_name,
ckpt_up_proj_name or self.ckpt_up_proj_name,
num_experts=self.expert_substitution.num_logical_experts,
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
)
def forward_modular(
self,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None" = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
constant_output = self.expert_substitution.transform_routes(
x, topk_weights, topk_ids
)
output = super().forward_modular(
x, topk_weights, topk_ids, shared_experts, shared_experts_input
)
if isinstance(output, UnfinalizedMoEOutput):
raise NotImplementedError(
"expert substitution does not support deferred MoE finalize"
)
# The constant output is replicated across TP ranks, whose partial
# outputs are all-reduced later, so add it on one rank only.
if self.moe_config.tp_rank == 0:
output.add_(constant_output[..., : output.shape[-1]])
return output
@@ -51,6 +51,10 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
def _supports_parallel_config(moe_parallel_config) -> bool:
return True
@staticmethod
def _supports_batch_invariance() -> bool:
return False
@staticmethod
def is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
@@ -973,34 +973,6 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
)
# ===========================================================================
# INT8 W8A8 MoE
# ===========================================================================
def prepare_int8_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Prepack INT8 MoE weights for the current CPU architecture."""
# SMMLA packing for AArch64
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
return (
cpu_prepack_moe_weight_int8(w13, "neon"),
cpu_prepack_moe_weight_int8(w2, "neon"),
)
# VSX packing for POWER
if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC:
return (
cpu_prepack_moe_weight_int8(w13, "vsx"),
cpu_prepack_moe_weight_int8(w2, "vsx"),
)
# VNNI packing for x86
packed_w13 = torch.ops._C.convert_weight_packed(w13)
packed_w2 = torch.ops._C.convert_weight_packed(w2)
return packed_w13, packed_w2
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
modular MoE experts."""
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -412,10 +413,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# CutlassExpertsFp8 does not support expert map, which is
# needed for STANDARD activation format kernels in DP/EP mode.
# Note that the BATCHED activation format does not use
# the expert map for identifying experts.
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
@@ -491,6 +491,15 @@ FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
def _prepare_topk_ids_for_cutlass_fp4_moe(
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> torch.Tensor:
if expert_map is None:
return topk_ids
return remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
def run_cutlass_moe_fp4(
output: torch.Tensor,
a: torch.Tensor,
@@ -513,6 +522,7 @@ def run_cutlass_moe_fp4(
e: int,
device: torch.device,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
@@ -543,6 +553,8 @@ def run_cutlass_moe_fp4(
assumes that topk < k < n to satisfy - up/down projection expectations.
"""
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
is_gated = activation.is_gated
# For gated activations (e.g. SiLU), w1 output is 2*n (gate + up).
# For non-gated activations (e.g. SiLU_NO_MUL), w1 output is n (up only).
@@ -729,9 +741,14 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# CutlassExpertsFp4 does not support expert map, which is
# needed for STANDARD activation format kernels in EP mode.
return moe_parallel_config.ep_size == 1
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
@staticmethod
def _supports_batch_invariance() -> bool:
@@ -806,6 +823,7 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
e=e,
device=hidden_states.device,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
activation_config=self.activation_config,
)
@@ -828,10 +846,13 @@ def run_cutlass_moe_mxfp4(
e: int,
device: torch.device,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
"""MXFP4 x MXFP4 MoE implementation using CUTLASS grouped GEMM."""
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
is_gated = activation.is_gated
w1_n = n * 2 if is_gated else n
@@ -1036,7 +1057,14 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return moe_parallel_config.ep_size == 1
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -1103,6 +1131,7 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
e=e,
device=hidden_states.device,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
activation_config=self.activation_config,
)
@@ -450,6 +450,8 @@ def invoke_moe_batched_triton_kernel(
BLOCK_M = config["BLOCK_SIZE_M"]
BLOCK_N = config["BLOCK_SIZE_N"]
BLOCK_K = config["BLOCK_SIZE_K"]
if block_shape is not None:
BLOCK_K = min(BLOCK_K, min(block_shape[0], block_shape[1]))
grid = (
expert_num_tokens.size(0),
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
fused_globalize_align_block_size,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
@@ -63,6 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.worker.workspace import current_workspace_manager
@@ -137,7 +141,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
"w13": humming_quant_config.w1_humming_config,
"w2": humming_quant_config.w2_humming_config,
}
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
self.locks = create_humming_locks(moe_config.device)
self.num_experts = moe_config.num_local_experts
self.global_num_experts = moe_config.num_experts
self.quant_config = humming_quant_config
@@ -214,12 +218,20 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
if scatter_idx is not None:
layout = "scatter"
hadamard_block_size = getattr(quant_config, f"{prefix}_hadamard_block_size")
has_input_processing = (
mode.should_quantize
or (hadamard_block_size is not None and hadamard_block_size > 1)
or activation is not None
or scatter_idx is not None
or num_valid_tokens is not None
)
inputs, group_scales, token_scales = may_process_input(
config,
inputs=inputs,
outputs=quanted_input,
outputs=quanted_input if has_input_processing else None,
token_scales=scale2 if mode.has_secondary_scale else scale,
hadamard_block_size=getattr(quant_config, f"{prefix}_hadamard_block_size"),
hadamard_block_size=hadamard_block_size,
layout=layout,
expert_tokens=expert_tokens if layout == "grouped_mask" else None,
scatter_idx=scatter_idx,
@@ -702,16 +714,45 @@ class HummingIndexedExperts(HummingExpertsBase):
num_sms = num_compute_units(topk_ids.get_device())
use_scatter = topk_ids.numel() > 2 * num_sms
alignment = moe_align_block_size(
topk_ids=topk_ids,
block_size=moe_block_size,
num_experts=self.global_num_experts,
expert_map=expert_map,
ignore_invalid_experts=True,
return_scatter_idx=use_scatter,
# When DeepEP-v2 deferred globalization, fuse globalize + align + sort
# into one launch (topk_ids globalized in place); else align only.
rank_expert_offset = (
expert_tokens_meta.rank_expert_offset
if expert_tokens_meta is not None
else None
)
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
scatter_idx = alignment[3] if len(alignment) == 4 else None
psum = (
expert_tokens_meta.psum_recv_per_rank
if expert_tokens_meta is not None
else None
)
if (
current_platform.is_cuda()
and rank_expert_offset is not None
and psum is not None
):
alignment = fused_globalize_align_block_size(
recv_topk_idx=topk_ids,
psum_recv_per_rank=psum,
rank_expert_offset=rank_expert_offset,
global_num_experts=self.global_num_experts,
local_num_experts=self.num_experts,
block_size=moe_block_size,
return_scatter_idx=use_scatter,
)
_, sorted_ids, expert_ids, num_tokens_padded = alignment[:4]
scatter_idx = alignment[4] if use_scatter else None
else:
alignment = moe_align_block_size(
topk_ids=topk_ids,
block_size=moe_block_size,
num_experts=self.global_num_experts,
expert_map=expert_map,
ignore_invalid_experts=True,
return_scatter_idx=use_scatter,
)
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
scatter_idx = alignment[3] if len(alignment) == 4 else None
moe_common_kwargs = {
"sorted_ids": sorted_ids,
@@ -21,7 +21,10 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
@@ -844,6 +847,20 @@ def triton_kernel_fused_experts(
return output_tensor
def _tk38_routing_data(
col_sum: torch.Tensor,
order: torch.Tensor,
flat_w: torch.Tensor,
topk: int,
) -> tuple["RoutingData", torch.Tensor, torch.Tensor]:
"""Build 3.8 routing from per-expert counts and the stable expert order."""
nrows = order.shape[0]
ragged = make_ragged_tensor_metadata(col_sum, nrows)
gather_tok = (order // topk).to(torch.int32)
routing = RoutingData(ragged, flat_w[order], gather_tok, topk, nrows)
return routing, gather_tok, gather_tok
def make_routing_data(
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
@@ -851,21 +868,18 @@ def make_routing_data(
) -> tuple["RoutingData", torch.Tensor, torch.Tensor]:
if triton_kernels_version == "3.8":
_, topk = topk_ids.shape
flat_e = topk_ids.reshape(-1)
valid = flat_e >= 0
flat_e = flat_e.to(torch.int64).clamp(min=0)
flat_w = topk_weights.reshape(-1).to(torch.float32) * valid.to(torch.float32)
nrows = flat_e.shape[0]
order = torch.argsort(flat_e, stable=True)
col_sum = torch.zeros(
num_local_experts, dtype=torch.int32, device=flat_e.device
flat_ids = topk_ids.reshape(-1)
# -1 (non-local) slots go to expert 0 with zero weight.
keys = flat_ids.clamp(min=0)
sorted_keys, order = torch.sort(keys, stable=True)
experts = torch.arange(
num_local_experts + 1, dtype=keys.dtype, device=keys.device
)
col_sum.scatter_add_(0, flat_e, torch.ones_like(flat_e, dtype=torch.int32))
ragged = make_ragged_tensor_metadata(col_sum, nrows)
gather_tok = (order // topk).to(torch.int32)
gammas = flat_w[order]
routing = RoutingData(ragged, gammas, gather_tok, topk, nrows)
return routing, gather_tok, gather_tok
edges = torch.searchsorted(sorted_keys, experts, out_int32=True)
col_sum = edges[1:] - edges[:-1]
valid = (flat_ids >= 0).to(torch.float32)
flat_w = topk_weights.reshape(-1).to(torch.float32) * valid
return _tk38_routing_data(col_sum, order, flat_w, topk)
topk_ids = topk_ids.to(torch.int16)
topk_weights = topk_weights.to(torch.bfloat16)
@@ -933,8 +947,14 @@ def routing_data_from_sparse_topk(
remapping) that work is redundant.
"""
if triton_kernels_version == "3.8":
# 3.8 routing is rebuilt from raw ids; no SparseMatrix reuse.
return make_routing_data(sparse_topk.indx, sparse_topk.vals, n_expts)
# topk ids have no -1 slots, so the topk metadata matches make_routing_data.
mask_metadata = sparse_topk.mask_metadata
return _tk38_routing_data(
mask_metadata.col_sum,
mask_metadata.col_sorted_indx,
sparse_topk.vals.reshape(-1).to(torch.float32),
sparse_topk.indx.shape[1],
)
dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx
combine_indx = sparse_topk.mask_metadata.col_sorted_indx
@@ -994,49 +1014,6 @@ def masked_moe_sum(
)
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] int64 local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor, expert_map: torch.Tensor
) -> torch.Tensor:
"""Fused global->local expert-id mapping over a topk_ids tensor, preserving -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW int64 tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
(Distinct from ``deep_gemm_utils.apply_expert_map``, which is a scalar
``@triton.jit`` device helper called from within other kernels.)
"""
out = torch.empty_like(topk_ids, dtype=torch.int64)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](topk_ids, expert_map, out, n, BLOCK=BLOCK)
return out
class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
@@ -3,6 +3,8 @@
import torch
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
@@ -22,6 +24,7 @@ class LoRAExpertsMixin:
state is on lora_context or passed as arguments.
"""
moe_config: FusedMoEConfig
_lora_context: MoELoRAContext | None = None
def set_lora_context(self, ctx: MoELoRAContext) -> None:
@@ -70,8 +73,13 @@ class LoRAExpertsMixin:
# concat swap afterwards.
w13_lora_a_stacked = w13_lora_a_stacked[::-1]
w13_lora_b_stacked = w13_lora_b_stacked[::-1]
return lora_context.punica_wrapper.add_lora_w13(
y,
interleave_w13 = (
lora_context.w13_num_slices == 2
and self.moe_config.activation == MoEActivation.SWIGLUOAI
)
lora_output = torch.zeros_like(y) if interleave_w13 else y
result = lora_context.punica_wrapper.add_lora_w13(
lora_output,
x,
w13_lora_a_stacked,
w13_lora_b_stacked,
@@ -89,9 +97,18 @@ class LoRAExpertsMixin:
lora_context.w13_num_slices,
lora_context.fully_sharded,
lora_context.use_tuned_config,
add_inputs=add_inputs,
add_inputs=add_inputs or interleave_w13,
token_lora_mapping=lora_context.local_token_lora_mapping,
)
if interleave_w13:
# Map gate/up slices onto even/odd output channels without a copy.
gate_up_delta = lora_output.unflatten(-1, (2, -1))
gate_up_output = y.unflatten(-1, (-1, 2)).transpose(-1, -2)
if add_inputs:
gate_up_output.add_(gate_up_delta)
else:
gate_up_output.copy_(gate_up_delta)
return result
def apply_w2_lora(
self,
@@ -218,7 +218,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
global_num_experts=global_num_experts,
expert_map=expert_map,
a1q_scale=None,
a2_scale=None,
a2_scale=a2_scale,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=expert_tokens_meta,
@@ -352,7 +352,11 @@ def rocm_aiter_fused_experts(
intermediate_pad = 0
assert moe_config.hidden_dim_unpadded is not None
assert moe_config.intermediate_size_per_partition_unpadded is not None
hidden_pad = hidden_states.shape[1] - moe_config.hidden_dim_unpadded
hidden_size = hidden_states.shape[1]
if hidden_states.dtype == torch.float4_e2m1fn_x2:
# Pre-quantized MXFP4 from MoRI dispatch, packed two per byte.
hidden_size *= 2
hidden_pad = hidden_size - moe_config.hidden_dim_unpadded
intermediate_pad = (
(
moe_config.intermediate_size_per_partition
@@ -448,7 +452,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
# When paired with MoRI, the prepare/finalize handles FP8
# When paired with MoRI, the prepare/finalize handles FP8 or MXFP4
# quantization during dispatch to reduce network traffic,
# so we should not defer input quantization.
# Otherwise, AITER fused MoE kernels handle input quantization
@@ -482,29 +486,6 @@ class AiterExperts(mk.FusedMoEExpertsModular):
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(None, None),
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
(kFp8StaticTensorSym, kFp8StaticTensorSym),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
(kFp8StaticChannelSym, kFp8DynamicTokenSym),
(kMxfp4Static, None),
(kMxfp4Static, kMxfp4Dynamic),
]
if (weight_key, activation_key) not in SUPPORTED_W_A:
return False
if weight_key == kMxfp4Static:
from vllm.platforms.rocm import on_gfx950, on_gfx1250
if not on_gfx950() or on_gfx1250():
return False
return True
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
@@ -539,7 +520,12 @@ class AiterExperts(mk.FusedMoEExpertsModular):
# Workspaces are managed internally by AITER.
workspace1 = (0,)
workspace2 = (0,)
output = (M, K)
# MoRI dispatches MXFP4 activations packed two per byte, so K is
# half the hidden size there.
packed_fp4_input = (
self.moe_config.use_mori_kernels and self.quant_config.use_mxfp4_w4a4
)
output = (M, K * 2 if packed_fp4_input else K)
return (workspace1, workspace2, output)
def apply(
@@ -597,3 +583,77 @@ class AiterExperts(mk.FusedMoEExpertsModular):
output.set_(result)
else:
output.copy_(result)
class AiterUnquantizedExperts(AiterExperts):
"""Unquantized (bf16/fp16) weights and activations."""
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [(None, None)]
return (weight_key, activation_key) in SUPPORTED_W_A
class AiterFp8BlockExperts(AiterExperts):
"""128x128 block-quantized weight scale, 128-grouped dynamic
activation scale."""
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [(kFp8Static128BlockSym, kFp8Dynamic128Sym)]
return (weight_key, activation_key) in SUPPORTED_W_A
class AiterFp8TensorExperts(AiterExperts):
"""Per-tensor static weight scale, either per-tensor static or
per-tensor dynamic activation scale."""
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8StaticTensorSym, kFp8StaticTensorSym),
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
class AiterFp8ChannelExperts(AiterExperts):
"""Per-output-channel static weight scale, per-token dynamic
activation scale."""
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [(kFp8StaticChannelSym, kFp8DynamicTokenSym)]
return (weight_key, activation_key) in SUPPORTED_W_A
class AiterMxfp4Experts(AiterExperts):
"""MXFP4-quantized weights, with either unquantized (bf16/fp16) or
MXFP4-quantized dynamic activations. gfx950-only (not gfx1250)."""
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp4Static, None),
(kMxfp4Static, kMxfp4Dynamic),
]
if (weight_key, activation_key) not in SUPPORTED_W_A:
return False
from vllm.platforms.rocm import on_gfx950, on_gfx1250
return on_gfx950() and not on_gfx1250()
@@ -315,13 +315,14 @@ class FlashInferMoeEp:
w2_scale=weights.w2_scale,
)
if spec.kernel == "cutedsl":
# In-kernel fc2 reduce keeps its default (off); FlashInfer 0.7.1rc1
# renamed in_kernel_fc2_reduce to enable_in_kernel_fc2_reduce.
megakernel = api.Nvfp4CutedslMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
gate_up_clamp=moe.swiglu_limit,
fast_math=True,
apply_topk_in_fc1=apply_topk_in_fc1,
enable_in_kernel_fc2_reduce=False,
combine_dtype="bf16",
input_norm_const=epilogue.input_norm_const,
fc1_alpha=None,
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm import _custom_ops as ops
from vllm.triton_utils import triton
def fused_globalize_align_block_size(
recv_topk_idx: torch.Tensor,
psum_recv_per_rank: torch.Tensor,
rank_expert_offset: int,
global_num_experts: int,
local_num_experts: int,
block_size: int,
return_scatter_idx: bool = False,
) -> tuple[torch.Tensor, ...]:
"""Fuse the DeepEP-v2 INDEXED decode metadata chain into one launch.
Replaces ``_globalize_recv_topk_idx`` + ``moe_align_block_size`` +
``count_and_sort_expert_tokens``. ``recv_topk_idx`` holds LOCAL expert ids
and is globalized in place; valid-token count is read on-device from
``psum_recv_per_rank[-1]`` (CUDA-graph safe).
Returns the globalized ``recv_topk_idx`` and the ``moe_align_block_size``
outputs ``(sorted_ids, expert_ids, num_tokens_post_pad)``. When requested,
the final output is a masked identity scatter index for valid routes.
"""
assert recv_topk_idx.dtype == torch.int64
_, topk = recv_topk_idx.shape
numel = recv_topk_idx.numel()
device = recv_topk_idx.device
max_num_tokens_padded = numel + local_num_experts * (block_size - 1)
if numel < local_num_experts:
max_num_tokens_padded = min(numel * block_size, max_num_tokens_padded)
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=device)
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device)
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device)
scatter_idx = (
torch.empty((numel, 1), dtype=torch.int32, device=device)
if return_scatter_idx
else None
)
ops.fused_globalize_align_block_size(
recv_topk_idx,
psum_recv_per_rank,
rank_expert_offset,
global_num_experts,
local_num_experts,
block_size,
sorted_ids,
expert_ids,
num_tokens_post_pad,
scatter_idx,
)
outputs = (recv_topk_idx, sorted_ids, expert_ids, num_tokens_post_pad)
return (*outputs, scatter_idx) if scatter_idx is not None else outputs
+25 -1
View File
@@ -23,6 +23,10 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
ExpertMapManager,
)
from vllm.model_executor.layers.fused_moe.expert_substitution import (
SubstitutedRoutedExperts,
make_expert_substitution,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
@@ -253,6 +257,12 @@ def FusedMoEFactory(
fuse_shared_experts,
)
)
expert_substitution = make_expert_substitution(
vllm_config.model_config, layer_name, num_experts, hidden_size, params_dtype
)
if expert_substitution is not None:
# Route over all logical experts but allocate only the retained ones.
global_num_experts -= len(expert_substitution.substituted_expert_ids)
# Initialize EPLB manager (or None?)
eplb_state: EplbLayerState | None = None
@@ -367,6 +377,7 @@ def FusedMoEFactory(
device=vllm_config.device_config.device,
routing_method=router.routing_method_type, # Not ideal
has_hash_routing=hash_indices_table is not None,
require_decomposed_backend=expert_substitution is not None,
swiglu_limit=swiglu_limit,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
@@ -380,7 +391,20 @@ def FusedMoEFactory(
# Create RoutedExperts instance BEFORE create_weights()
# This will hold all expert weight parameters
if routed_experts_cls is None:
if expert_substitution is not None:
if routed_experts_cls not in (None, RoutedExperts) or (
routed_input_transform is not None or routed_output_transform is not None
):
raise NotImplementedError(
"expert substitution requires the default RoutedExperts and no "
"routed input or output transforms"
)
routed_experts_cls = SubstitutedRoutedExperts
routed_experts_args = {
**(routed_experts_args or {}),
"expert_substitution": expert_substitution,
}
elif routed_experts_cls is None:
routed_experts_cls = RoutedExperts
assert params_dtype is not None
@@ -103,6 +103,9 @@ class ExpertTokensMetadata:
expert_num_tokens: torch.Tensor | None
expert_num_tokens_cpu: torch.Tensor | None
psum_recv_per_rank: torch.Tensor | None = None
# local->global expert-id offset, set when globalization is deferred to the
# experts' fused align kernel.
rank_expert_offset: int | None = None
@staticmethod
def make_from_list(
@@ -255,6 +258,15 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def supports_dbo(self) -> bool:
"""Indicates whether or not this class can run under DBO.
Defaults to supports_async(). A class whose synchronous prepare and
finalize do their own DBO compute/comm handoff can return True here
while keeping supports_async() False.
"""
return self.supports_async()
def supports_deferred_moe_finalize(self) -> bool:
"""Whether ``finalize`` can be skipped for a deferring consumer.
@@ -569,6 +581,8 @@ class FusedMoEExperts(ABC):
)
elif moe_config.has_hash_routing and cls.is_monolithic():
return False, _make_reason("hash routing")
elif moe_config.require_decomposed_backend and cls.is_monolithic():
return False, _make_reason("routes adjusted after routing")
elif not cls._supports_routing_method(
moe_config.routing_method, weight_key, activation_key
):
@@ -1046,6 +1060,18 @@ class FusedMoEKernelModularImpl:
):
self.prepare_finalize = prepare_finalize
self.fused_experts = fused_experts
# Experts that globalize recv_topk_idx themselves let a capable prepare
# stage skip its standalone launch.
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingIndexedExperts,
)
if (
current_platform.is_cuda()
and isinstance(fused_experts, HummingIndexedExperts)
and hasattr(prepare_finalize, "defer_globalize")
):
prepare_finalize.defer_globalize = True
self.shared_experts: SharedExperts | None = None
moe_parallel_config = fused_experts.moe_config.moe_parallel_config
self.moe_parallel_config = moe_parallel_config
@@ -1189,7 +1215,7 @@ class FusedMoEKernelModularImpl:
# We shouldn't be running an a2a kernel that doesn't
# support async prepare/finalize
# TODO(lucas): enable in follow-up
assert not dbo_enabled()
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
(
a1q,
@@ -1377,7 +1403,7 @@ class FusedMoEKernelModularImpl:
"""
if not self.prepare_finalize.supports_async():
assert not dbo_enabled()
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
self.prepare_finalize.finalize(
output,
@@ -9,6 +9,7 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -40,6 +41,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import should_auto_disable_deep_gemm
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
@@ -211,10 +213,12 @@ def backend_to_kernel_cls(
elif backend == Fp8MoeBackend.AITER:
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterFp8BlockExperts,
AiterFp8ChannelExperts,
AiterFp8TensorExperts,
)
return [AiterExperts]
return [AiterFp8BlockExperts, AiterFp8TensorExperts, AiterFp8ChannelExperts]
elif backend == Fp8MoeBackend.VLLM_CUTLASS:
from vllm.model_executor.layers.fused_moe.experts.triton_cutlass_moe import (
@@ -490,6 +494,13 @@ def select_fp8_moe_backend(
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
elif (model_config := get_current_vllm_config().model_config) is not None and (
should_auto_disable_deep_gemm(
getattr(model_config.hf_text_config, "model_type", None)
)
):
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.DEEPGEMM)
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.BATCHED_DEEPGEMM)
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
@@ -284,10 +284,10 @@ def backend_to_kernel_cls(
elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterMxfp4Experts,
)
return [AiterExperts]
return [AiterMxfp4Experts]
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
@@ -305,10 +305,10 @@ def backend_to_kernel_cls(
elif backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterMxfp4Experts,
)
return [AiterExperts]
return [AiterMxfp4Experts]
elif backend == Mxfp4MoeBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMxFp4
@@ -875,13 +875,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
triton_uses_128 = (
backend == Mxfp4MoeBackend.TRITON_UNFUSED and get_cdna_version() != 4
)
aiter_w4a4 = backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4
alignment = (
128 if is_situ_or_silu and (aiter_uses_128 or triton_uses_128) else 256
intermediate_alignment = (
128
if aiter_w4a4 or (is_situ_or_silu and (aiter_uses_128 or triton_uses_128))
else 256
)
# W4A4 stage 1 still requires a 256-aligned hidden dimension.
hidden_alignment = 256 if aiter_w4a4 else intermediate_alignment
intermediate_size = round_up(intermediate_size, alignment)
hidden_size = round_up(hidden_size, alignment)
intermediate_size = round_up(intermediate_size, intermediate_alignment)
hidden_size = round_up(hidden_size, hidden_alignment)
elif backend == Mxfp4MoeBackend.CPU:
# CPU AMX kernel uses BLOCK_N=32, align to 32
intermediate_size = round_up(intermediate_size, 32)
@@ -1475,6 +1480,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
activation: MoEActivation | None = None,
use_separated_a4w4: bool = False,
activation_key: QuantKey | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -1536,8 +1542,20 @@ def convert_weight_to_mxfp4_moe_kernel_format(
convert_to_humming_moe_kernel_format,
)
# The env override wins; otherwise use the checkpoint's activation
# scheme instead of Humming's default BF16 activations.
input_schema = None
if activation_key is not None and not envs.VLLM_HUMMING_INPUT_QUANT_CONFIG:
from vllm.model_executor.layers.quantization.utils.humming import (
quant_key_to_input_schema,
)
input_schema = quant_key_to_input_schema(activation_key)
convert_to_humming_moe_kernel_format(
layer, quant_config={"quant_method": "mxfp4"}
layer,
quant_config={"quant_method": "mxfp4"},
input_schema=input_schema,
allow_input_schema_fallback=False,
)
return (
layer.w13_weight,
@@ -2128,6 +2146,59 @@ def make_mxfp4_moe_quant_config(
)
_MXFP4_SCHEME_CANDIDATES: list[tuple[QuantKey, QuantKey | None]] = [
(kMxfp4Static, kMxfp4Dynamic),
(kMxfp4Static, kMxfp8Dynamic),
(kMxfp4Static, kFp8Dynamic128Sym),
(kMxfp4Static, kFp8StaticTensorSym),
(kMxfp4Static, None),
]
def _mxfp4_quant_keys_for_experts(
experts_cls: type[mk.FusedMoEExperts],
mxfp4_backend: Mxfp4MoeBackend,
) -> tuple[QuantKey | None, QuantKey | None]:
for weight_key, activation_key in _MXFP4_SCHEME_CANDIDATES:
if experts_cls._supports_quant_scheme(weight_key, activation_key):
return weight_key, activation_key
return kMxfp4Static, _backend_activation_key(mxfp4_backend)
def _activation_format_for_config(
moe_config: FusedMoEConfig,
) -> mk.FusedMoEActivationFormat:
return (
mk.FusedMoEActivationFormat.BatchedExperts
if moe_config.moe_parallel_config.use_batched_activation_format
else mk.FusedMoEActivationFormat.Standard
)
def _validate_mxfp4_experts_config(
experts_cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
mxfp4_backend: Mxfp4MoeBackend,
) -> None:
"""Fail closed when a caller bypasses the oracle and picks experts directly."""
activation_format = _activation_format_for_config(moe_config)
weight_key, activation_key = _mxfp4_quant_keys_for_experts(
experts_cls, mxfp4_backend
)
supported, reason = experts_cls.is_supported_config(
experts_cls,
moe_config,
weight_key,
activation_key,
activation_format,
)
if not supported:
raise ValueError(
f"{experts_cls.__name__} does not support this MoE deployment"
f"{f' since {reason}' if reason else ''}."
)
def make_mxfp4_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
@@ -2136,6 +2207,8 @@ def make_mxfp4_moe_kernel(
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
_validate_mxfp4_experts_config(experts_cls, moe_config, mxfp4_backend)
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
prepare_finalize = maybe_make_prepare_finalize(
@@ -126,10 +126,10 @@ def backend_to_kernel_cls(
elif backend == UnquantizedMoeBackend.AITER:
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterUnquantizedExperts,
)
return [AiterExperts]
return [AiterUnquantizedExperts]
elif backend == UnquantizedMoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
@@ -303,9 +303,10 @@ def select_unquantized_moe_backend(
)
runner_backend = moe_config.moe_backend
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
# modules_to_not_convert) falls through to auto instead of erroring.
if runner_backend not in ["auto", "humming"]:
# 'humming' and 'marlin' are quantization-only; an unquantized layer (e.g.
# excluded via modules_to_not_convert, or an unquantized MTP draft of a
# quantized model) falls through to auto instead of erroring.
if runner_backend not in ["auto", "humming", "marlin"]:
requested_backend = map_unquantized_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
@@ -127,6 +127,11 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
self.use_cudagraph = use_cudagraph
self.sp_size = sp_size
# Set by FusedMoEKernelModularImpl when the experts globalize
# recv_topk_idx themselves, letting the decode path skip the standalone
# _globalize_recv_topk_idx launch.
self.defer_globalize = False
# DBO microbatching: one handle slot per micro-batch.
self.handles: list[deep_ep.EPHandle | None] = [None, None]
@@ -275,6 +280,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# during profile_run when CUDA graphs are enabled.
expert_tokens_meta = None
globalize_deferred = False
if recv_topk_idx is None:
# do_expand=True (prefill mode): build topk_ids from
# per-expert token counts.
@@ -307,12 +313,15 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# be treated as real routed tokens by experts that build routing
# over *all* rows (e.g. triton MoE backend's make_routing_data),
# polluting the per-expert token lists and corrupting real tokens.
recv_topk_idx = _globalize_recv_topk_idx(
recv_topk_idx,
psum_recv_per_rank,
self.rank_expert_offset,
self.num_experts,
)
if self.defer_globalize and self.use_cudagraph:
globalize_deferred = True
else:
recv_topk_idx = _globalize_recv_topk_idx(
recv_topk_idx,
psum_recv_per_rank,
self.rank_expert_offset,
self.num_experts,
)
# Reshape recv_topk_weights to match recv_topk_idx shape [N, 1]
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
@@ -329,6 +338,9 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
expert_num_tokens_cpu=None,
)
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
if globalize_deferred:
# recv_topk_idx still holds LOCAL ids; experts globalize it.
expert_tokens_meta.rank_expert_offset = self.rank_expert_offset
if _quantize_before_dispatch(quant_config, defer_input_quant):
if quant_config.quant_dtype == "mxfp8" and expert_x_scale is not None:
@@ -21,12 +21,15 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
max_tokens_per_rank: int,
num_dispatchers: int,
use_fp8_dispatch: bool = False,
use_fp4_dispatch: bool = False,
):
super().__init__()
assert not (use_fp8_dispatch and use_fp4_dispatch)
self.mori_op = mori_op
self.num_dispatchers_ = num_dispatchers
self.max_tokens_per_rank = max_tokens_per_rank
self.use_fp8_dispatch = use_fp8_dispatch
self.use_fp4_dispatch = use_fp4_dispatch
self._dispatch_topk_ids: torch.Tensor | None = None
@property
@@ -73,7 +76,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
)
scale = None
# When defer_input_quant is True, the expert kernel handles
# quantization internally, so skip FP8 dispatch quantization.
# quantization internally, so skip dispatch quantization.
if self.use_fp8_dispatch and not defer_input_quant:
from aiter import QuantType, get_hip_quant
@@ -92,6 +95,14 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
)
# mori expects one scale slot per token; broadcast.
scale = scale.expand(a1.shape[0], 1).contiguous()
elif self.use_fp4_dispatch and not defer_input_quant:
from aiter import QuantType, get_hip_quant
# Packed MXFP4 [M, K // 2] with e8m0 scales [M, K // 32]. The
# scales stay unshuffled: AITER fused_moe sorts them itself
# when it receives pre-quantized FP4 input.
quant_func = get_hip_quant(QuantType.per_1x32)
a1, scale = quant_func(a1, shuffle=False)
# mori's combine() reduces over this rank's own [num_tokens, topk]
# routing. The modular kernel rebinds topk_ids to the dispatched ids
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -11,7 +14,13 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.platforms import current_platform
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
from vllm.v1.worker.ubatching import (
dbo_enabled,
dbo_switch_to_compute_sync,
dbo_yield_and_switch_from_compute_to_comm,
)
def _quantize_and_setup_dispatch(
@@ -133,6 +142,34 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
return self._allocate_combine_input(shape, dtype, device)
return None
def supports_dbo(self) -> bool:
# prepare()/finalize() hand the collective to the comm stream
# themselves (see _comm_region), so the synchronous path is DBO-safe.
# supports_async() stays False: it also selects MK-internal
# shared-expert overlap. Only validated on ROCm so far.
return current_platform.is_rocm()
@contextmanager
def _comm_region(self) -> Iterator[None]:
"""Scope the dispatch/combine collective runs in.
Without DBO the collective runs on the calling stream. Under DBO it
runs on the comm stream so the peer ubatch can compute while it is in
flight, using the same compute/comm handoff as DeepEP-HT. Graph
capture keeps every collective on the capture stream, so overlap is
limited to eager steps (prefill).
"""
if not dbo_enabled() or torch.cuda.is_current_stream_capturing():
yield
return
# Release the CPU to the peer ubatch so it can enqueue compute while
# this ubatch's collective runs; the compute stream waits on exit.
dbo_yield_and_switch_from_compute_to_comm()
try:
yield
finally:
dbo_switch_to_compute_sync()
def prepare(
self,
a1: torch.Tensor,
@@ -178,13 +215,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
extra_tensors = []
extra_tensors.append(local_token_lora_mapping)
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=extra_tensors,
)
with self._comm_region():
res = get_ep_group().dispatch(
a1q,
topk_weights,
topk_ids,
is_sequence_parallel=self.is_sequence_parallel,
extra_tensors=extra_tensors,
)
if extra_tensors is None:
assert len(res) == 3
@@ -238,11 +276,12 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
apply_router_weight_on_input=apply_router_weight_on_input,
)
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
with self._comm_region():
get_ep_group().combine_into_output(
out,
output,
is_sequence_parallel=self.is_sequence_parallel,
)
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
@@ -700,12 +700,29 @@ class RoutedExperts(PluggableLayer):
param.data[:, :dim1, :dim2].copy_(loaded_weight)
return True if return_success else None
quant_method_name = self.quant_method.__class__.__name__
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
base_quant_method = self.quant_method
# In case online MOE re-quantization is used, loading is handled
# by the original quant method.
if (
isinstance(base_quant_method, OnlineMoEMethodBase)
and base_quant_method.requantization_source is not None
):
assert isinstance(
base_quant_method.requantization_source, FusedMoEMethodBase
)
base_quant_method = base_quant_method.requantization_source
quant_method_name = base_quant_method.__class__.__name__
global_expert_id = expert_id
expert_id = self._map_global_expert_id_to_local_expert_id(global_expert_id)
use_global_sf = (
getattr(self.quant_method, "use_global_sf", False)
getattr(base_quant_method, "use_global_sf", False)
and "input_scale" in weight_name
)
@@ -842,7 +859,7 @@ class RoutedExperts(PluggableLayer):
if "ModelOpt" in quant_method_name:
# Determine per-tensor weight scale patterns based on variant
# Use the dedicated method instead of brittle string matching
uses_weight_scale_2 = self.quant_method.uses_weight_scale_2_pattern()
uses_weight_scale_2 = base_quant_method.uses_weight_scale_2_pattern()
quant_method = getattr(param, "quant_method", None)
# Call _load_per_tensor_weight_scale() to load per-tensor (scalar)
@@ -921,7 +938,7 @@ class RoutedExperts(PluggableLayer):
FusedMoeWeightScaleSupported.GROUP.value,
FusedMoeWeightScaleSupported.BLOCK.value,
]:
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
scale_refine = getattr(base_quant_method, "weight_scale_refine", None)
if (
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
and scale_refine is not None
@@ -986,12 +1003,20 @@ class RoutedExperts(PluggableLayer):
for expert_name, loaded_weight in weights:
qual_name = f"{self.layer_name}.{expert_name}"
is_fused = loaded_weight.dim() == 3
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
named_expert = (
mapping_by_expert.get(expert_key)
if qual_name.count("experts.") == 1
else None
)
# Encoded per-expert [N, K / block_size, bytes_per_block] weights
# have the same rank as fused [E, N, K] weights. An explicit expert
# name identifies the per-expert layout without treating N as E.
is_fused = loaded_weight.dim() == 3 and named_expert is None
# Fused tensors and ambiguous names keep the full mapping.
candidates = expert_mapping
if not is_fused and qual_name.count("experts.") == 1:
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
candidates = mapping_by_expert.get(expert_key, expert_mapping)
candidates = named_expert or expert_mapping
matched = False
for param_name, weight_name, expert_id, shard_id in candidates:
@@ -31,7 +31,7 @@ def can_use_dsv4_topk(
and correction_bias.is_contiguous()
and topk == _TOPK
and renormalize
and indices_dtype in (torch.int32, torch.uint32, torch.int64)
and indices_dtype in (torch.int32, torch.int64)
)
@@ -41,6 +41,7 @@ if current_platform.is_cuda():
def _dsv4_topk_kernel(
gating_output_ptr,
correction_bias_ptr,
is_padding_ptr,
topk_weights_ptr,
topk_ids_ptr,
routed_scaling_factor,
@@ -49,6 +50,7 @@ if current_platform.is_cuda():
image_sentinel_lo,
NUM_EXPERTS: tl.constexpr,
BLOCK_N: tl.constexpr,
HAS_PADDING: tl.constexpr,
HAS_VL: tl.constexpr,
launch_pdl: tl.constexpr,
):
@@ -106,6 +108,11 @@ if current_platform.is_cuda():
output_mask = topk_offsets < 6
output_offsets = row * 6 + topk_offsets
if HAS_PADDING:
is_padding = tl.load(is_padding_ptr + row)
selected_weights = tl.where(is_padding, 0.0, selected_weights)
selected_ids = tl.where(is_padding, -1, selected_ids)
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
@@ -118,11 +125,20 @@ def dsv4_topk(
correction_bias: torch.Tensor,
indices_dtype: torch.dtype,
routed_scaling_factor: float,
is_padding: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
bias_vl: torch.Tensor | None = None,
image_sentinel_lo: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens, num_experts = gating_output.shape
if is_padding is not None:
assert is_padding.dtype == torch.bool
assert is_padding.shape == (num_tokens,)
assert is_padding.device == gating_output.device
assert is_padding.is_contiguous()
assert indices_dtype in (torch.int32, torch.int64), (
"Padding requires a signed indices dtype for the -1 sentinel."
)
has_vl = bias_vl is not None and image_sentinel_lo > 0
if bias_vl is not None:
assert input_ids is not None, "bias_vl routing requires input_ids"
@@ -136,6 +152,7 @@ def dsv4_topk(
_dsv4_topk_kernel[(num_tokens,)](
gating_output,
correction_bias,
is_padding,
topk_weights,
topk_ids,
routed_scaling_factor,
@@ -144,6 +161,7 @@ def dsv4_topk(
image_sentinel_lo,
NUM_EXPERTS=num_experts,
BLOCK_N=triton.next_power_of_2(num_experts),
HAS_PADDING=is_padding is not None,
HAS_VL=has_vl,
num_warps=1,
launch_pdl=current_platform.is_arch_support_pdl(),
@@ -176,6 +176,7 @@ def fused_topk_bias(
)
output_indices_dtype = torch.int32 if indices_type is None else indices_type
padding_mask = _get_padding_mask(gating_output.shape[0])
if (
scoring_func == "sqrtsoftplus"
and hash_indices_table is None
@@ -193,6 +194,7 @@ def fused_topk_bias(
e_score_correction_bias,
output_indices_dtype,
routed_scaling_factor,
is_padding=padding_mask,
input_ids=input_tokens,
bias_vl=bias_vl,
image_sentinel_lo=image_sentinel_lo,
@@ -4,6 +4,7 @@ import torch
from torch.nn.parameter import Parameter
import vllm._custom_ops as ops
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import (
@@ -22,14 +23,16 @@ class GateLinear(ReplicatedLinear):
"""MoE gate linear layer with multi-tier GEMM dispatch:
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
K divisible by 8)
any shape w/ K divisible by 8)
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
M<=32, model-specific shapes)
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
4. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
M>=2048)
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
6. F.linear via ReplicatedLinear (ultimate fallback)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
7. F.linear via ReplicatedLinear (ultimate fallback)
The ``out_dtype`` attribute is mutable and can be set after init
(e.g. when the required dtype depends on the expert quantization
@@ -37,6 +40,9 @@ class GateLinear(ReplicatedLinear):
A ``quant_config`` that actually quantizes the gate disables every
specialized tier, leaving plain ``ReplicatedLinear`` behavior.
``VLLM_BATCH_INVARIANT`` also disables tiers 1-5: they pick kernels and
split-K from the token count and skip the batch-invariant linear path.
"""
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
@@ -77,8 +83,12 @@ class GateLinear(ReplicatedLinear):
input_size,
output_size,
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
batch_invariant = envs.VLLM_BATCH_INVARIANT
can_use_specialized_kernels = (
current_platform.is_cuda() and (is_hopper or is_blackwell) and not bias
current_platform.is_cuda()
and (is_hopper or is_blackwell)
and not bias
and not batch_invariant
)
# If fp32 compute is required and no specialized kernel is available,
@@ -108,6 +118,7 @@ class GateLinear(ReplicatedLinear):
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
self.allow_fp32_router_gemm = (
self.is_unquantized
and not batch_invariant
and not bias
and self.weight.dtype == torch.float32
and (
@@ -121,6 +132,7 @@ class GateLinear(ReplicatedLinear):
)
self.allow_bf16x3_router_gemm = (
self.is_unquantized
and not batch_invariant
and not bias
and self.weight.dtype == torch.float32
and current_platform.is_cuda()
@@ -137,8 +149,10 @@ class GateLinear(ReplicatedLinear):
# (GB10 / DGX Spark), which this tier still covers. See #49921.
self._router_gemm_no_bias = self.is_unquantized and not bias
self._router_gemm_cublas_capable = (
current_platform.is_cuda() or current_platform.is_rocm()
) and self._router_gemm_no_bias
(current_platform.is_cuda() or current_platform.is_rocm())
and self._router_gemm_no_bias
and not batch_invariant
)
self.allow_cublas_router_gemm = (
self._router_gemm_cublas_capable
and self.weight.dtype == torch.bfloat16
@@ -154,6 +168,7 @@ class GateLinear(ReplicatedLinear):
self._rocm_bf16x3_weight_eligible = (
self._router_gemm_no_bias
and not batch_invariant
and self.weight.dtype == torch.float32
and bf16x3_router_gemm_rocm.platform_supported()
)
@@ -172,20 +187,50 @@ class GateLinear(ReplicatedLinear):
self.quant_method = _GateLinearMethod()
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
# cuteDSL ll_bf16_gemm eligibility. Any dims supported, but SM90+ required bc:
# cuteDSL ll_bf16_gemm/ll_fp32w_gemm eligibility. Any dims supported, but
# SM90+ required because:
# 1. PDL support. Both dot-product and split-K kernels.
# 2. Thread Block Clusters. Split-K kernel for cross-CTA reduction.
self.allow_ll_bf16_gemm = False
self.allow_ll_fp32w_gemm = False
if can_use_specialized_kernels:
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available,
is_available as is_ll_bf16_gemm_available,
)
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
is_available as is_ll_fp32w_gemm_available,
)
self.allow_ll_bf16_gemm = (
self.weight.dtype == torch.bfloat16
is_ll_bf16_gemm_available()
and self.weight.dtype == torch.bfloat16
and self.out_dtype == torch.float32
and is_available()
)
self.allow_ll_fp32w_gemm = (
is_ll_fp32w_gemm_available()
and self.weight.dtype == torch.float32
and self.out_dtype == torch.float32
)
self._register_ll_fp32w_warmup()
def _register_ll_fp32w_warmup(self) -> None:
# Based on multi-tier GEMM dispatch. See above.
# This avoids compiling LL_FP32W_GEMM_KERNEL if not used.
if not self.allow_ll_fp32w_gemm or self.allow_fp32_router_gemm:
return
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
LL_FP32W_GEMM_KERNEL,
)
# FP16 remains runtime-supported; JIT monitoring reports unexpected use.
# The kernel supports it, but there is no FP16 activation use case yet.
LL_FP32W_GEMM_KERNEL.register_warmup(
shapes=((self.weight.shape[1], self.weight.shape[0]),),
m_values=range(1, self.FP32_MAX_TOKENS + 1),
a_dtypes=(torch.bfloat16, torch.float32),
)
def set_out_dtype(self, out_dtype: torch.dtype) -> None:
"""Set output dtype for the router logits after init.
@@ -211,14 +256,23 @@ class GateLinear(ReplicatedLinear):
if self.allow_specialized_router_gemm:
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
is_available,
is_available as is_ll_bf16_gemm_available,
)
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
is_available as is_ll_fp32w_gemm_available,
)
self.allow_ll_bf16_gemm = (
self.weight.dtype == torch.bfloat16
is_ll_bf16_gemm_available()
and self.weight.dtype == torch.bfloat16
and out_dtype == torch.float32
and is_available()
)
self.allow_ll_fp32w_gemm = (
is_ll_fp32w_gemm_available()
and self.weight.dtype == torch.float32
and out_dtype == torch.float32
)
self._register_ll_fp32w_warmup()
def _return(
self, output: torch.Tensor
@@ -228,7 +282,8 @@ class GateLinear(ReplicatedLinear):
def forward(
self, x: torch.Tensor
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, any dims)
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, bf16 in, fp32 out,
# any dims w/ K divisible by 8)
if (
self.allow_ll_bf16_gemm
and x.shape[0] <= self.LL_BF16_MAX_TOKENS
@@ -241,7 +296,7 @@ class GateLinear(ReplicatedLinear):
output = ll_bf16_gemm(x, self.weight)
return self._return(output)
# Tier 4: ROCm bf16x3 router GEMM. Checked before tier 2 because both
# Tier 5: ROCm bf16x3 router GEMM. Checked before tier 2 because both
# tiers claim the same gfx950 shapes and tier 2 returns
# unconditionally; the custom op below picks between them on the
# runtime num_tokens.
@@ -270,7 +325,20 @@ class GateLinear(ReplicatedLinear):
)
return self._return(output)
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
# Tier 3: cuteDSL ll_fp32w_gemm (SM90+, bf16/fp32 in, fp32 out)
if (
self.allow_ll_fp32w_gemm
and x.shape[0] <= 32
and x.dtype in (torch.bfloat16, torch.float16, torch.float32)
):
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
LL_FP32W_GEMM_KERNEL,
)
output = LL_FP32W_GEMM_KERNEL(x, self.weight)
return self._return(output)
# Tier 4: bf16x3 CuteDSL kernel for fp32 router weights
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
bf16x3_router_gemm,
@@ -279,12 +347,12 @@ class GateLinear(ReplicatedLinear):
output = bf16x3_router_gemm(x, self.weight)
return self._return(output)
# Tier 5: cuBLAS bf16→fp32
# Tier 6: cuBLAS bf16→fp32
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
return self._return(output)
# Tier 6: F.linear (ReplicatedLinear)
# Tier 7: F.linear (ReplicatedLinear)
if (
self.out_dtype is not None
and self.is_unquantized
@@ -301,7 +301,10 @@ class MoERunner(MoERunnerInterface):
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
# routed_experts reports names relative to itself, but callers such as
# AutoWeightsLoader qualify what this returns with the runner's prefix.
for name in self.routed_experts.load_weights(weights):
yield f"routed_experts.{name}"
def _select_forward(self) -> Callable:
if current_platform.is_tpu():
@@ -235,6 +235,73 @@ def count_expert_num_tokens(
return expert_num_tokens
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
OUT_INT32: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
if OUT_INT32:
out = tl.where(valid, local, -1)
else:
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor,
expert_map: torch.Tensor,
*,
out_dtype: torch.dtype = torch.int64,
) -> torch.Tensor:
"""Map global expert ids to local ids; non-local or invalid routes become -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
Args:
topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous;
the Triton kernel indexes by flat ``numel()`` offsets.
expert_map: Global-to-local mapping; -1 marks non-local experts.
out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``.
Returns:
Remapped local expert ids with the same shape as ``topk_ids``.
"""
if out_dtype not in (torch.int32, torch.int64):
raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}")
assert topk_ids.is_contiguous()
out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](
topk_ids,
expert_map,
out,
n,
BLOCK=BLOCK,
OUT_INT32=out_dtype == torch.int32,
)
return out
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
"""Shrink the given tensor and apply the given view to it. This is
used to resize the intermediate fused_moe caches.
+8
View File
@@ -12,6 +12,7 @@ from vllm import envs, ir
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.determinism.batch_invariant import rms_norm_batch_invariant
from vllm.model_executor.utils import set_weight_attrs
logger = init_logger(__name__)
@@ -63,6 +64,9 @@ class RMSNorm(CustomOp):
self.weight = torch.ones(hidden_size, dtype=weight_dtype)
if self.has_weight:
self.weight = nn.Parameter(self.weight)
# Dummy weights keep norms neutral: random ones would shrink every
# activation and skew data-dependent kernels (e.g. sparse top-k).
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
# When has_weight=False, pass weight=None so implementations that
# support a weightless path can skip the per-channel multiply.
@@ -154,6 +158,7 @@ class GemmaRMSNorm(CustomOp):
) -> None:
super().__init__()
self.weight = nn.Parameter(torch.zeros(hidden_size))
set_weight_attrs(self.weight, {"dummy_weight_value": 0.0})
self.variance_epsilon = eps
def forward_native(
@@ -239,8 +244,10 @@ class LayerNorm(CustomOp):
self.bias: nn.Parameter | None = None
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
if bias:
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))
set_weight_attrs(self.bias, {"dummy_weight_value": 0.0})
def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
if self.weight is None:
@@ -328,6 +335,7 @@ class RMSNormGated(CustomOp):
self.eps = eps
self.activation = activation
self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
self.register_parameter("bias", None)
self.group_size = group_size
self.norm_before_gate = norm_before_gate
+30 -10
View File
@@ -516,6 +516,20 @@ class ColumnParallelLinear(LinearBase):
self._maybe_allow_fp8_block_shape_mismatch()
self.gather_output = gather_output
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
quant_method = self.quant_method
if (
isinstance(quant_method, OnlineLinearBase)
and quant_method.requantization_source is not None
):
quant_method = quant_method.requantization_source
weight_loader = (
self.weight_loader_v2
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
)
self.quant_method.create_weights(
layer=self,
input_size_per_partition=self.input_size_per_partition,
@@ -523,11 +537,7 @@ class ColumnParallelLinear(LinearBase):
input_size=self.input_size,
output_size=self.output_size,
params_dtype=self.params_dtype,
weight_loader=(
self.weight_loader_v2
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
),
weight_loader=weight_loader,
)
if bias:
@@ -1697,6 +1707,20 @@ class RowParallelLinear(LinearBase):
self.input_is_parallel = input_is_parallel
self.reduce_results = reduce_results
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
quant_method = self.quant_method
if (
isinstance(quant_method, OnlineLinearBase)
and quant_method.requantization_source is not None
):
quant_method = quant_method.requantization_source
weight_loader = (
self.weight_loader_v2
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
)
self.quant_method.create_weights(
layer=self,
input_size_per_partition=self.input_size_per_partition,
@@ -1704,11 +1728,7 @@ class RowParallelLinear(LinearBase):
input_size=self.input_size,
output_size=self.output_size,
params_dtype=self.params_dtype,
weight_loader=(
self.weight_loader_v2
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
),
weight_loader=weight_loader,
)
if not reduce_results and (bias and not skip_bias_add):
raise ValueError(
+27 -2
View File
@@ -2,11 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from collections.abc import Iterable
from math import prod
from math import lcm, prod
import torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backend import AttentionBackend
@@ -62,6 +62,13 @@ class MambaBase(AttentionLayerBase):
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
pass
def get_prefill_checkpoint_alignment(self) -> int | None:
"""Token alignment of the checkpoint offsets this layer can export from
inside a prefill, if any; in align mode, the platform makes Mamba blocks
multiples of it.
"""
return None
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
mamba_block_size = vllm_config.cache_config.mamba_block_size
assert mamba_block_size is not None
@@ -86,3 +93,21 @@ class MambaBase(AttentionLayerBase):
def get_attn_backend(self) -> type[AttentionBackend]:
"""Get the attention backend class for this Mamba layer."""
return get_mamba_attn_backend(self.mamba_type)
def get_mamba_prefill_checkpoint_alignment(vllm_config: VllmConfig) -> int | None:
"""LCM of the Mamba layers' prefill checkpoint alignments.
None unless every Mamba layer declares one: the scheduler takes internal
checkpoints only when every Mamba group can export them.
"""
alignment = 1
for layer in get_layers_from_vllm_config(
vllm_config,
MambaBase, # type: ignore[type-abstract]
).values():
layer_alignment = layer.get_prefill_checkpoint_alignment()
if layer_alignment is None:
return None
alignment = lcm(alignment, layer_alignment)
return alignment
+126 -10
View File
@@ -1,12 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass, replace
import torch
from vllm.config import VllmConfig
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import CommonAttentionMetadata
@@ -71,9 +71,11 @@ def _gather_checkpoint_state_indices(
class MambaPrefillCheckpointMetadata:
checkpoint_offsets: torch.Tensor
state_indices: torch.Tensor
# Host copy of ``checkpoint_offsets``, for backends that lay out the
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
# Host copies of ``checkpoint_offsets`` and of each row's query length, for
# backends that lay out the checkpoint before any tensor exists (Mamba2's
# SSD chunk split, FlashInfer GDN's checkpoint plan).
offsets: list[int] | None = None
query_lens: list[int] | None = None
# Gather indices into the block table, kept so that another KV cache group
# with the same spec can re-derive its own ``state_indices``.
request_rows: torch.Tensor | None = None
@@ -150,20 +152,134 @@ class MambaPrefillCheckpointBuilder:
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
),
offsets=checkpoint_offsets,
query_lens=query_lens,
request_rows=request_rows_tensor,
block_cols=checkpoint_cols_tensor,
)
class MambaPrefillCheckpointExporter(ABC):
"""Export a mid-prefill checkpoint into backend-specific paged states."""
@dataclass(frozen=True)
class MambaPrefillCheckpointExporter:
"""Store each checkpoint's pre-convolution window and recurrent state.
The recurrent states are rows of a buffer the prefill kernel emitted;
``recurrent_rows`` picks each checkpoint row's state out of it, and by
default row ``i`` holds checkpoint ``i``'s.
"""
state_len: int | None = None
@abstractmethod
def export(
self,
checkpoint: MambaPrefillCheckpointMetadata,
*args,
**kwargs,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
recurrent_checkpoint: torch.Tensor,
recurrent_state: torch.Tensor,
cu_seqlens: torch.Tensor,
recurrent_rows: torch.Tensor | None = None,
) -> None:
"""Write checkpoint state into the paged cache."""
raise NotImplementedError
state_len = (
self.state_len if self.state_len is not None else conv_state.shape[-1]
)
width = raw_qkv.shape[-1]
recurrent_row_size = recurrent_checkpoint[0].numel()
block_size = 256
store_cache_checkpoints_kernel[
(
checkpoint.checkpoint_offsets.numel(),
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
)
](
raw_qkv,
conv_state,
recurrent_checkpoint,
recurrent_state,
cu_seqlens,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
recurrent_rows if recurrent_rows is not None else checkpoint.state_indices,
raw_qkv.stride(0),
raw_qkv.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
recurrent_checkpoint.stride(0),
recurrent_state.stride(0),
checkpoint.checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
recurrent_rows is not None,
block_size,
)
@triton.jit
def store_cache_checkpoints_kernel(
x_ptr,
conv_state_ptr,
recurrent_checkpoint_ptr,
recurrent_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
checkpoint_state_indices_ptr,
recurrent_rows_ptr,
x_stride_0: tl.constexpr,
x_stride_1: tl.constexpr,
state_stride_0: tl.constexpr,
state_stride_1: tl.constexpr,
state_stride_2: tl.constexpr,
checkpoint_stride_0: tl.constexpr,
recurrent_state_stride_0: tl.constexpr,
checkpoint_offset_stride: tl.constexpr,
STATE_LEN: tl.constexpr,
WIDTH: tl.constexpr,
RECURRENT_ROW_SIZE: tl.constexpr,
NULL_STATE_IDX: tl.constexpr,
HAS_RECURRENT_ROWS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
seq_idx = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
checkpoint_offset = tl.load(
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
)
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
valid_conv = (
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
)
width_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
token_idx = checkpoint_end - STATE_LEN + history_idx
values = tl.load(
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
mask=valid_conv,
)
tl.store(
conv_state_ptr
+ state_idx * state_stride_0
+ width_idx * state_stride_1
+ history_idx * state_stride_2,
values,
mask=valid_conv,
)
if HAS_RECURRENT_ROWS:
recurrent_row = tl.load(recurrent_rows_ptr + seq_idx).to(tl.int64)
else:
recurrent_row = seq_idx.to(tl.int64)
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
recurrent = tl.load(
recurrent_checkpoint_ptr + recurrent_row * checkpoint_stride_0 + cols,
mask=valid_recurrent,
)
tl.store(
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
recurrent,
mask=valid_recurrent,
)
@@ -4,6 +4,7 @@ import torch
from einops import rearrange
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
VllmConfig,
get_current_vllm_config,
@@ -524,6 +525,7 @@ def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
return weight_loader
@eager_break_during_capture
def olmo_hybrid_gdn_full_forward(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -2,7 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen3-Next/Qwen3.5 model."""
import functools
import itertools
import math
import os
from dataclasses import replace
from typing import Literal
import torch
@@ -15,6 +19,7 @@ from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import (
CacheConfig,
VllmConfig,
get_current_vllm_config,
)
@@ -30,6 +35,12 @@ from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.mamba.abstract import (
get_mamba_prefill_checkpoint_alignment,
)
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader
from vllm.model_executor.layers.mamba.mamba_utils import (
@@ -59,13 +70,19 @@ from vllm.third_party.flash_linear_attention.ops import (
)
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
direct_register_custom_op,
)
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
FLASHINFER_GDN_CHECKPOINT_ALIGNMENT,
GDNAttentionMetadata,
)
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
from vllm.v1.worker.workspace import current_workspace_manager
# Optional ROCm AITER Triton kernels for the GDN decode path.
# Availability is checked centrally via rocm_aiter_ops; the actual function
@@ -90,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _aiter_flydsl_unsupported_reasons(
vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None
) -> list[str]:
"""Every reason the AITER FlyDSL GDN prefill kernels cannot serve this
model, or an empty list if they can."""
reasons = []
if head_k_dim != 128 or head_v_dim != 128:
reasons.append(
f"linear head dims are K={head_k_dim} V={head_v_dim}; "
"FlyDSL requires 128/128"
)
dtype = vllm_config.model_config.dtype
if dtype != torch.bfloat16:
reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16")
return reasons
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
@@ -108,7 +142,7 @@ def _resolve_gdn_prefill_backend(
* Blackwell (SM10.x) with ``head_k_dim == 128``;
AITER FlyDSL GDN prefill kernels are chosen when:
* "aiter_flydsl" is requested; (opt-in only)
* "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950;
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
"""
@@ -136,24 +170,28 @@ def _resolve_gdn_prefill_backend(
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
"newer GPU, and that the installed AITER exports them."
)
if head_k_dim != 128 or head_v_dim != 128:
reasons = _aiter_flydsl_unsupported_reasons(
vllm_config, head_k_dim, head_v_dim
)
if reasons:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
"GDN prefill backend 'aiter_flydsl' was requested but %s. "
"Falling back to Triton/FLA.",
head_k_dim,
head_v_dim,
)
return backend, "triton"
if vllm_config.model_config.dtype != torch.bfloat16:
logger.warning_once(
"GDN prefill backend 'aiter_flydsl' was requested but "
"model dtype is %s; FlyDSL requires bfloat16. "
"Falling back to Triton/FLA.",
vllm_config.model_config.dtype,
"; ".join(reasons),
)
return backend, "triton"
return backend, "aiter_flydsl"
if backend == "auto":
from vllm.platforms.rocm import on_mi3xx
if (
on_mi3xx()
and not _aiter_flydsl_unsupported_reasons(
vllm_config, head_k_dim, head_v_dim
)
and rocm_aiter_ops.is_gdn_flydsl_prefill_available()
):
return backend, "aiter_flydsl"
return backend, "triton"
if backend == "aiter_flydsl":
@@ -241,6 +279,10 @@ def fi_chunk_gated_delta_rule(
output_final_state: bool,
cu_seqlens: torch.Tensor | None = None,
use_qk_l2norm_in_kernel: bool = True,
state_checkpoints: torch.Tensor | None = None,
checkpoint_cu_starts: torch.Tensor | None = None,
checkpoint_every_n_tokens: int = 0,
use_cp: Literal["auto"] | bool = "auto",
):
from flashinfer.gdn_prefill import (
chunk_gated_delta_rule as chunk_gated_delta_rule_fi,
@@ -271,6 +313,10 @@ def fi_chunk_gated_delta_rule(
initial_state=fi_state,
output_final_state=output_final_state,
cu_seqlens=cu_seqlens,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=checkpoint_cu_starts,
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
use_cp=use_cp,
backend="flashinfer",
)
# FlashInfer returns (output, state) when output_final_state=True,
@@ -283,6 +329,20 @@ def fi_chunk_gated_delta_rule(
return result.unsqueeze(0), None
def _checkpoint_grid(cache_config: CacheConfig) -> int:
"""Token grid that FlashInfer GDN checkpoint offsets are kept on.
Prefix-cache boundaries, made multiples of FlashInfer's checkpoint
alignment; the platform already keeps blocks on it.
"""
if cache_config.prefix_match_unit is not None:
return math.lcm(
cache_config.prefix_match_unit, FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
)
assert cache_config.mamba_block_size is not None
return cache_config.mamba_block_size
@CustomOp.register("chunk_gated_delta_rule")
class ChunkGatedDeltaRule(CustomOp):
def __init__(self) -> None:
@@ -622,6 +682,21 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
self.gdn_prefill_backend = self.chunk_gated_delta_rule.gdn_prefill_backend
self._prefill_kernels_warmed_up = False
# FlashInfer's prefill emits mid-prefill states, so an align-mode
# checkpoint needs no split prefill.
self._checkpoint_alignment: int | None = None
if (
vllm_config.cache_config.mamba_cache_mode == "align"
and self.gdn_prefill_backend == "flashinfer"
):
self._checkpoint_alignment = FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
self._vllm_config = vllm_config
self._max_num_batched_tokens = (
vllm_config.scheduler_config.max_num_batched_tokens
)
self._checkpoint_exporter = MambaPrefillCheckpointExporter(
state_len=self.conv_kernel_size - 1
)
self.enable_packed_recurrent_decode = (
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
)
@@ -666,6 +741,46 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def get_prefill_checkpoint_alignment(self) -> int | None:
return self._checkpoint_alignment
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if not self._exports_checkpoints or not isinstance(spec, MambaSpec):
return spec
# FlashInfer emits states at one stride for the whole batch, the gcd of
# the offsets; keeping every offset on the prefix-cache grid keeps that
# stride a multiple of the grid, which bounds the states emitted.
return replace(
spec,
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=_checkpoint_grid(vllm_config.cache_config),
)
@functools.cached_property
def _exports_checkpoints(self) -> bool:
# Known only once every layer is built: a Mamba layer that cannot
# checkpoint (e.g. Qwen4Exp's PLE short conv) keeps the scheduler
# splitting the prefill instead.
return (
self._checkpoint_alignment is not None
and get_mamba_prefill_checkpoint_alignment(self._vllm_config) is not None
)
@functools.cached_property
def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]:
# Offsets on the grid emit at most one state per grid step of a batch.
# Read lazily: the block size is final only after the model is built.
return (
(
cdiv(self._max_num_batched_tokens, _checkpoint_grid(self.cache_config)),
self.num_v_heads // self.tp_size,
self.head_v_dim,
self.head_k_dim,
),
torch.float32,
)
def _fused_gdn_decode_unsupported_reason(
self, vllm_config: VllmConfig
) -> str | None:
@@ -1228,6 +1343,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
if self._prefill_kernels_warmed_up:
return
self._prefill_kernels_warmed_up = True
if self._exports_checkpoints:
# Size the workspace while profiling; it is locked afterwards.
current_workspace_manager().get_simultaneous(self._checkpoint_buffer_spec)
device = qkv_or_qkvz.device
dtype = qkv_or_qkvz.dtype
@@ -1303,6 +1421,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=aiter_prefill_metadata,
)
if self.gdn_prefill_backend == "flashinfer":
self._warmup_flashinfer_prefill_variants(q, k, v, g, beta, state)
except Exception:
logger.warning(
"GDN prefill kernel warmup (T=%d) failed for "
@@ -1337,6 +1457,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
torch.accelerator.empty_cache()
def _warmup_flashinfer_prefill_variants(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
g: torch.Tensor,
beta: torch.Tensor,
state: torch.Tensor,
) -> None:
"""Compile every FlashInfer GDN prefill kernel a batch can dispatch to.
FlashInfer builds separate kernels for its context-parallel route (few
sequences) and its regular route, each with and without checkpoint
output. A single sequence under ``use_cp="auto"`` takes the CP route
wherever the heuristic can pick it; ``use_cp=False`` forces the other.
"""
checkpoint_buffers: list[torch.Tensor | None] = [None]
if self._exports_checkpoints:
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
self._checkpoint_buffer_spec
)
checkpoint_buffers.append(state_checkpoints[:1])
cu_seqlens = torch.tensor([0, q.shape[1]], device=q.device, dtype=torch.int32)
checkpoint_cu_starts = torch.tensor([0, 1], device=q.device)
for use_cp, state_checkpoints in itertools.product(
("auto", False), checkpoint_buffers
):
has_checkpoint = state_checkpoints is not None
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=state,
output_final_state=True,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
use_cp=use_cp,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=checkpoint_cu_starts if has_checkpoint else None,
checkpoint_every_n_tokens=q.shape[1] if has_checkpoint else 0,
)
def _forward_core_rocm(
self,
qkvz: torch.Tensor,
@@ -1508,6 +1672,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
)
# 1.2: Process the remaining part
conv_input_non_spec = mixed_qkv_non_spec
if attn_metadata.num_prefills > 0:
assert mixed_qkv_non_spec is not None
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
@@ -1709,23 +1874,59 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
assert prefill_has_initial_state is not None
initial_state = ssm_state[prefill_state_indices]
initial_state[~prefill_has_initial_state, ...] = 0
(
core_attn_out_non_spec,
last_recurrent_state,
) = self.chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
plan = attn_metadata.flashinfer_checkpoint_plan
if plan is None:
(
core_attn_out_non_spec,
last_recurrent_state,
) = self.chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
chunk_indices=attn_metadata.chunk_indices,
chunk_offsets=attn_metadata.chunk_offsets,
use_qk_l2norm_in_kernel=False,
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
)
else:
# Only FlashInfer's prefill emits checkpoints.
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
self._checkpoint_buffer_spec
)
state_checkpoints = state_checkpoints[: plan.num_checkpoints]
(
core_attn_out_non_spec,
last_recurrent_state,
) = fi_chunk_gated_delta_rule(
q=query_non_spec,
k=key_non_spec,
v=value_non_spec,
g=g_non_spec,
beta=beta_non_spec,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=attn_metadata.prefill_query_start_loc,
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=plan.cu_starts,
checkpoint_every_n_tokens=plan.every_n_tokens,
)
assert attn_metadata.checkpoint is not None
assert non_spec_query_start_loc is not None
self._checkpoint_exporter.export(
attn_metadata.checkpoint,
raw_qkv=conv_input_non_spec,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=non_spec_query_start_loc,
recurrent_rows=plan.rows,
)
# Init cache
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
@@ -1,130 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
return 16 if backend == "flashkda" else None
@dataclass(frozen=True)
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
"""Store FlashKDA recurrent and convolution checkpoint states."""
state_len: int | None = None
def export(
self,
checkpoint: MambaPrefillCheckpointMetadata,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
recurrent_checkpoint: torch.Tensor,
recurrent_state: torch.Tensor,
cu_seqlens: torch.Tensor,
) -> None:
state_len = (
self.state_len if self.state_len is not None else conv_state.shape[-1]
)
width = raw_qkv.shape[-1]
recurrent_row_size = recurrent_checkpoint[0].numel()
block_size = 256
store_cache_checkpoints_kernel[
(
checkpoint.checkpoint_offsets.numel(),
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
)
](
raw_qkv,
conv_state,
recurrent_checkpoint,
recurrent_state,
cu_seqlens,
checkpoint.checkpoint_offsets,
checkpoint.state_indices,
raw_qkv.stride(0),
raw_qkv.stride(1),
conv_state.stride(0),
conv_state.stride(1),
conv_state.stride(2),
recurrent_checkpoint.stride(0),
recurrent_state.stride(0),
checkpoint.checkpoint_offsets.stride(0),
state_len,
width,
recurrent_row_size,
NULL_BLOCK_ID,
block_size,
)
@triton.jit
def store_cache_checkpoints_kernel(
x_ptr,
conv_state_ptr,
recurrent_checkpoint_ptr,
recurrent_state_ptr,
query_start_loc_ptr,
checkpoint_offsets_ptr,
checkpoint_state_indices_ptr,
x_stride_0: tl.constexpr,
x_stride_1: tl.constexpr,
state_stride_0: tl.constexpr,
state_stride_1: tl.constexpr,
state_stride_2: tl.constexpr,
checkpoint_stride_0: tl.constexpr,
recurrent_state_stride_0: tl.constexpr,
checkpoint_offset_stride: tl.constexpr,
STATE_LEN: tl.constexpr,
WIDTH: tl.constexpr,
RECURRENT_ROW_SIZE: tl.constexpr,
NULL_STATE_IDX: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
seq_idx = tl.program_id(0)
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
checkpoint_offset = tl.load(
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
)
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
valid_conv = (
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
)
width_idx = cols // STATE_LEN
history_idx = cols % STATE_LEN
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
token_idx = checkpoint_end - STATE_LEN + history_idx
values = tl.load(
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
mask=valid_conv,
)
tl.store(
conv_state_ptr
+ state_idx * state_stride_0
+ width_idx * state_stride_1
+ history_idx * state_stride_2,
values,
mask=valid_conv,
)
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
recurrent = tl.load(
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
mask=valid_recurrent,
)
tl.store(
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
recurrent,
mask=valid_recurrent,
)
@@ -8,6 +8,7 @@ import torch
import torch.nn.functional as F
from einops import rearrange
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.custom_op import PluggableLayer
@@ -325,6 +326,7 @@ class MiniMaxText01LinearAttention(LinearAttention):
output[:num_actual_tokens], _ = self.out_proj(hidden)
@eager_break_during_capture
def linear_attention(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -7,6 +7,7 @@ import torch
from torch import nn
from torch.nn.parameter import Parameter
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
@@ -465,6 +466,7 @@ def split_batch_to_prefill_and_decode(
)
@eager_break_during_capture
def mamba_mixer(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -8,6 +8,7 @@ from dataclasses import replace
import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.config.mamba import MambaBackendEnum
from vllm.distributed import (
@@ -811,10 +812,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
# - It will read the initial states for every sequence,
# that has "has_initial_states_p" == True,
# from "cache_indices", using "state_indices_tensor_p".
# - It updates the "conv_state" cache in positions pointed
# to by "state_indices_tensor_p".
# In particular, it will always write the state at the
# sequence end.
# - It writes the state at the sequence end to the "conv_state"
# cache in positions pointed to by "state_indices_tensor_p".
x = hidden_states_B_C_p.transpose(
0, 1
) # this is the form that causal-conv see
@@ -891,6 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
varlen_states,
ssm_state,
query_start_loc_p,
state_len=self.conv_kernel_size - 1,
)
ssm_state[state_indices_tensor_p] = final_states
if ring_start is not None and self._updates_replayssm_trackers:
@@ -1214,6 +1214,7 @@ def share_replayssm_ring_trackers(
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
@eager_break_during_capture
def mamba_mixer2(
projected_states: torch.Tensor,
output: torch.Tensor,
@@ -238,13 +238,14 @@ class MambaStateShapeCalculator:
"""Append the physical ReplaySSM ring shapes.
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
B_cache uses the un-extended ``n_groups``.
B_cache extends ``n_groups`` for head shards as in ``mamba2_state_shape``.
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and current verify window together.
ring_buffer_len += 1 + num_speculative_tokens
local_nheads, head_dim, state_size = base_shapes[1]
n_groups = n_groups + cls.extra_groups_for_head_shards(n_groups, tp_world_size)
local_ngroups = divide(n_groups, tp_world_size)
return (
*base_shapes,
@@ -20,17 +20,11 @@ def _causal_conv1d_fwd_kernel( # continuous batching
w_ptr, # (dim, width)
bias_ptr,
initial_states_ptr, # conv_states_ptr
cache_indices_ptr, # (batch, n_blocks + padding) The second dimension contains
# the block indices relevant for each sequence
# plus potential 0-padding at the beginning and at the end
cache_indices_ptr, # (batch,)
has_initial_states_ptr,
query_start_loc_ptr,
batch_ptr,
token_chunk_offset_ptr,
block_idx_first_scheduled_token, # (batch,)
block_idx_last_scheduled_token, # (batch,)
initial_state_idx, # (batch,)
num_computed_tokens, # (batch,)
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
# Matrix dimensions
dim: tl.constexpr,
@@ -46,7 +40,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
stride_cache_indices: tl.constexpr,
stride_o_dim: tl.constexpr,
stride_o_token: tl.int64,
stride_block_m: tl.constexpr, # Stride block to align divided by BLOCK_M
# others
pad_slot_id: tl.constexpr,
null_block_id: tl.constexpr,
@@ -54,7 +47,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
HAS_BIAS: tl.constexpr,
KERNEL_WIDTH: tl.constexpr,
SILU_ACTIVATION: tl.constexpr,
IS_APC_ENABLED: tl.constexpr,
HAS_NULL_BLOCK: tl.constexpr,
NP2_STATELEN: tl.constexpr,
BLOCK_M: tl.constexpr,
@@ -93,40 +85,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
# find the actual sequence length
seqlen = sequence_end_index - sequence_start_index
B_size: tl.constexpr = stride_block_m * BLOCK_M
if IS_APC_ENABLED:
# Handle the case if prefix caching is enabled.
# In particular, if prefix caching is enabled, the program write additional cache states to "cache_indices_ptr"
# Get the length of the completed sequence so far and compute the offset.
current_first_index = tl.load(block_idx_first_scheduled_token + idx_seq)
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
sequence_completed_index = tl.load(num_computed_tokens + idx_seq)
# Compute the offset where the first stride_block_m-aligned first full block is
# Value in "token-space"
sequence_completed_offset_token = sequence_completed_index % B_size
seq_completed_offset = B_size - sequence_completed_offset_token
seq_end_offset = (seqlen - seq_completed_offset) % B_size
last_full_block_token_index = sequence_end_index - seq_end_offset
# If the sequence without the sequence_offset_index is stride_cache_chunk-aligned, then the last full chunk is the second-to-last one
if seq_end_offset == 0:
last_full_block_token_index = last_full_block_token_index - B_size
# Get the number of blocks to be filled for the current sequence
# If n_block_to_fill = 0, then only the state at the sequence end is stored
n_block_to_fill = current_last_index - current_first_index
# Get the index of the init block
conv_state_init_index = tl.load(initial_state_idx + idx_seq)
else:
n_block_to_fill = 0
current_last_index = 0
conv_state_init_index = 0
current_first_index = 0
last_full_block_token_index = 0
token_offset = BLOCK_M * chunk_offset
segment_len = min(BLOCK_M, seqlen - token_offset)
@@ -137,7 +95,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
# cache_idx
conv_states_input_coord = tl.load(
conv_state_indices_ptr + idx_seq * stride_cache_indices + conv_state_init_index
conv_state_indices_ptr + idx_seq * stride_cache_indices
).to(tl.int64)
if HAS_NULL_BLOCK: # noqa
@@ -224,20 +182,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
# Compute the offset where the last block should be written in the conv_states
conv_states_output_coord = tl.load(
conv_state_indices_ptr
+ idx_seq * stride_cache_indices
+ current_last_index
).to(tl.int64)
conv_states_ptrs_target = (
conv_states_ptr
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
+ (idx_feats * stride_conv_state_dim)
)[None, :] + ( # [BLOCK_N,]
idx_tokens_conv * stride_conv_state_tok
)[:, None]
conv_states_base[None, :]
+ (idx_tokens_conv * stride_conv_state_tok)[:, None]
) # [BLOCK_M, BLOCK_N]
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
@@ -346,52 +294,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
conv_states_ptrs = prior_tokens - 3 * stride_x_token # [BLOCK_N]
col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca")
# Store intermediate states aligned with stride_block_m
# The additional states are cached starting from the last stride_block_m.
# For example:
# If n_block_to_fill = 0, then only the state at the sequence end is cached and the process below is not involved.
# If n_block_to_fill > 0, then the states at the sequence end and at the n_block_to_fill-last
# stride_block_m are cached.
# For example chunk_offset = n_block_to_fill stores the state at last_full_block
if (chunk_offset - 1) < n_block_to_fill:
# Store the states at the chunk boundaries from the start of the sequence
idx_tokens_last = (
last_full_block_token_index
- (n_block_to_fill - chunk_offset) * B_size
- state_len
) + tl.arange(0, NP2_STATELEN) # [BLOCK_M]
x_ptrs = (
x_ptr
+ (idx_tokens_last * stride_x_token)[:, None]
+ (idx_feats * stride_x_dim)[None, :]
) # [BLOCK_M,BLOCK_N,]
mask_x = (idx_tokens_last >= 0)[:, None] & (idx_feats < dim)[
None, :
] # token-index # token-index # feature-index
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
# cache_idx
conv_states_output_coord = tl.load(
conv_state_indices_ptr
+ idx_seq * stride_cache_indices
+ current_first_index
+ (chunk_offset - 1)
).to(tl.int64)
conv_states_ptrs_target = (
conv_states_ptr
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
+ (idx_feats * stride_conv_state_dim)
)[None, :] + ( # [BLOCK_N,]
idx_tokens_conv * stride_conv_state_tok
)[:, None]
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
tl.store(conv_states_ptrs_target, loaded_x, mask)
if HAS_BIAS:
bias = bias_ptr + idx_feats
mask_bias = idx_feats < dim
@@ -489,11 +391,6 @@ def causal_conv1d_fn(
activation: str | None = "silu",
pad_slot_id: int = PAD_SLOT_ID,
null_block_id: int = NULL_BLOCK_ID,
block_idx_first_scheduled_token: torch.Tensor | None = None,
block_idx_last_scheduled_token: torch.Tensor | None = None,
initial_state_idx: torch.Tensor | None = None,
num_computed_tokens: torch.Tensor | None = None,
block_size_to_align=0,
metadata=None,
validate_data=False,
):
@@ -536,16 +433,6 @@ def causal_conv1d_fn(
for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
in this case, the kernel will not process entries at
indices 0 and 3
block_idx_first_scheduled_token: (batch,), dtype int32
The pointer into cache_indices, where the first cache block to be filled is located.
block_idx_last_scheduled_token: (batch,), dtype int32
The pointer into cache_indices, where the last cache block to be filled is located.
initial_state_idx: (batch,), dtype int32
The pointer into cache_indices, where the cache block containing the initial state is located.
num_computed_tokens: (batch,), dtype int32
The number of tokens already completed for each sequence
block_size_to_align: int
The block size to align the cached states to
out: same shape as `x`
"""
if isinstance(activation, bool) and activation:
@@ -631,12 +518,6 @@ def causal_conv1d_fn(
assert weight.stride(1) == 1
assert (dim, width) == weight.shape
assert is_channel_last, "Need to run in channel-last layout"
if block_size_to_align is not None and block_size_to_align > 0:
assert (block_size_to_align % BLOCK_M) == 0, (
"The mamba block size needs to be divisible by the BLOCK_M"
)
else:
block_size_to_align = BLOCK_M
if metadata is None:
@@ -720,10 +601,6 @@ def causal_conv1d_fn(
query_start_loc,
batch_ptr,
token_chunk_offset_ptr,
block_idx_first_scheduled_token,
block_idx_last_scheduled_token,
initial_state_idx,
num_computed_tokens,
out,
# Matrix dimensions
dim,
@@ -739,7 +616,6 @@ def causal_conv1d_fn(
stride_cache_indices,
stride_o_dim,
stride_o_token,
block_size_to_align // BLOCK_M,
# others
pad_slot_id,
null_block_id,
@@ -747,7 +623,6 @@ def causal_conv1d_fn(
HAS_BIAS=bias is not None,
KERNEL_WIDTH=width,
SILU_ACTIVATION=activation in ["silu", "swish"],
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
HAS_NULL_BLOCK=null_block_id is not None,
NP2_STATELEN=np2_statelen,
# launch_cooperative_grid=True
@@ -769,8 +644,6 @@ def _causal_conv1d_update_kernel(
conv_state_indices_ptr,
num_accepted_tokens_ptr,
query_start_loc_ptr, # (batch + 1)
block_idx_last_scheduled_token, # (batch,)
initial_state_idx, # (batch,)
o_ptr, # (batch, dim, seqlen)
# Matrix dimensions
batch: int,
@@ -798,7 +671,6 @@ def _causal_conv1d_update_kernel(
KERNEL_WIDTH: tl.constexpr,
SILU_ACTIVATION: tl.constexpr,
IS_VARLEN: tl.constexpr,
IS_APC_ENABLED: tl.constexpr,
IS_SPEC_DECODING: tl.constexpr,
NP2_STATELEN: tl.constexpr,
HAS_NULL_BLOCK: tl.constexpr,
@@ -818,17 +690,9 @@ def _causal_conv1d_update_kernel(
# [BLOCK_N,] elements along the feature-dimension (channel)
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
if IS_APC_ENABLED:
# Get the state from the initial_state_idx
conv_state_init = tl.load(initial_state_idx + idx_seq)
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
else:
conv_state_init = 0
current_last_index = 0
# cache_idx
conv_states_input_coord = tl.load(
conv_state_indices_ptr + idx_seq * stride_state_indices + conv_state_init
conv_state_indices_ptr + idx_seq * stride_state_indices
).to(tl.int64)
if HAS_NULL_BLOCK: # noqa
@@ -941,14 +805,9 @@ def _causal_conv1d_update_kernel(
new_conv_state = tl.where(mask, conv_state, loaded_x)
# Get the state from the initial_state_idx
# cache_idx
conv_states_offset = tl.load(
conv_state_indices_ptr + idx_seq * stride_state_indices + current_last_index
).to(tl.int64)
conv_state_ptrs_target = (
conv_state_ptr
+ (conv_states_offset * stride_conv_state_seq) # Offset from seq
+ (conv_states_input_coord * stride_conv_state_seq) # Offset from seq
+ (idx_feats * stride_conv_state_dim)
)[None, :] + ( # [BLOCK_N,]
idx_tokens * stride_conv_state_tok
@@ -1104,8 +963,6 @@ def causal_conv1d_update(
query_start_loc: torch.Tensor | None = None,
max_query_len: int = -1,
null_block_id: int = NULL_BLOCK_ID,
block_idx_last_scheduled_token: torch.Tensor | None = None,
initial_state_idx: torch.Tensor | None = None,
validate_data=False,
out: torch.Tensor | None = None,
):
@@ -1123,10 +980,6 @@ def causal_conv1d_update(
If not None, the conv_state is a larger tensor along the batch dim,
and we are selecting the batch coords specified by conv_state_indices.
Useful for a continuous batching scenario.
block_idx_last_scheduled_token: (batch,), dtype int32
The pointer into conv_state_indices, where the last cache block to be filled is located.
initial_state_idx: (batch,), dtype int32
The pointer into conv_state_indices, where the cache block containing the initial state is located.
num_accepted_tokens: (batch,), dtype int32
If not None, it indicates the number of accepted tokens for each
sequence in the batch.
@@ -1237,8 +1090,6 @@ def causal_conv1d_update(
conv_state_indices,
num_accepted_tokens,
query_start_loc,
block_idx_last_scheduled_token,
initial_state_idx,
out,
# Matrix dimensions
batch,
@@ -1266,7 +1117,6 @@ def causal_conv1d_update(
KERNEL_WIDTH=width,
SILU_ACTIVATION=activation in ["silu", "swish"],
IS_VARLEN=query_start_loc is not None,
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
IS_SPEC_DECODING=num_accepted_tokens is not None,
NP2_STATELEN=np2_statelen,
HAS_NULL_BLOCK=null_block_id is not None,
@@ -69,6 +69,8 @@ def store_prefill_checkpoint(
chunk_states: torch.Tensor,
ssm_state: torch.Tensor,
query_start_loc: torch.Tensor,
*,
state_len: int,
) -> None:
"""Write each prefill row's mid-prefill checkpoint into its paged block.
@@ -78,14 +80,15 @@ def store_prefill_checkpoint(
checkpoint: Per-row offsets and destination blocks.
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
conv_state: Paged conv state, ``(num_blocks, dim, state_len + num_spec)``.
chunk_states: SSD states at every chunk end.
ssm_state: Paged SSM state.
query_start_loc: Prefill query start locations.
state_len: ``conv_kernel_size - 1``, not the spec-widened conv width.
Readers take the first ``state_len`` slots as the newest inputs.
"""
dim = x.size(0)
state_len = conv_state.size(-1)
ssm_row_size = chunk_states[0].numel()
block_size = 1024
grid = (
@@ -4,6 +4,7 @@
import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.forward_context import ForwardContext, get_forward_context
@@ -360,6 +361,7 @@ class ShortConv(MambaBase, PluggableLayer):
return MambaAttentionBackendEnum.SHORT_CONV
@eager_break_during_capture
def short_conv(
hidden_states: torch.Tensor,
output: torch.Tensor,
@@ -4,6 +4,7 @@
import array
import contextlib
import ctypes
import struct
import sys
import threading
@@ -16,6 +17,7 @@ except ImportError:
from cuda import cudart
_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment
_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t)
# ---------------------------------------------------------------------------
@@ -30,6 +32,20 @@ def _check(error):
raise RuntimeError(f"CUDA runtime error: {error}")
# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so
# copy the raw handle bytes through getPtr() instead. This is safe because the
# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]).
def _ipc_handle_to_bytes(handle) -> bytes:
return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE)
def _ipc_handle_from_bytes(raw: bytes):
assert len(raw) == _IPC_HANDLE_SIZE, len(raw)
handle = cudart.cudaIpcMemHandle_t()
ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE)
return handle
def _cuda_malloc(size: int):
aligned = ((size + _ALIGN - 1) >> 21) << 21
err, ptr = cudart.cudaMalloc(aligned)
@@ -84,15 +100,16 @@ class IpcBuffer:
all_handles: list[bytes | None] = [None] * world_size
torch.distributed.all_gather_object(
all_handles, bytes(local_handle.reserved), group=process_group
all_handles, _ipc_handle_to_bytes(local_handle), group=process_group
)
for r in range(world_size):
if r == rank:
self.peer_ptrs[r] = self.local_ptr
else:
handle = cudart.cudaIpcMemHandle_t()
handle.reserved = all_handles[r]
raw = all_handles[r]
assert raw is not None
handle = _ipc_handle_from_bytes(raw)
err, ptr = cudart.cudaIpcOpenMemHandle(
handle, cudart.cudaIpcMemLazyEnablePeerAccess
)
@@ -198,6 +198,11 @@ class BOSEOSFilter(Pooler):
for i, (prompt_len, token_ids) in enumerate(
zip(pooling_metadata.prompt_lens, prompt_token_ids)
):
# Late chunking returns one row per chunk, with special tokens
# already included in the means. Skip BOS/EOS row removal for these
# requests; ordinary token outputs still use the filtering below.
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
continue
pooled_data = pooled_outputs[i]
assert (
isinstance(pooled_data, torch.Tensor)
@@ -17,6 +17,23 @@ from .methods import TokenPoolingMethodOutputItem
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
"""Mean-pool complete contextual states, accumulating in FP32."""
chunk_size = min(chunk_size, len(data))
num_full, remainder = divmod(len(data), chunk_size)
full_end = num_full * chunk_size
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
if not remainder:
return full_chunks.mean(dim=1, dtype=torch.float32)
return torch.cat(
[
full_chunks.mean(dim=1, dtype=torch.float32),
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
]
)
class TokenPoolerHead(nn.Module, ABC):
@abstractmethod
def get_supported_tasks(self) -> Set[PoolingTask]:
@@ -80,6 +97,12 @@ class TokenEmbeddingPoolerHead(TokenPoolerHead):
if pooled_data is None:
return None
if (late_chunking := pooling_param.late_chunking_params) is not None:
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
pooled_data = (
means.to(pooled_data.dtype) if self.head_dtype is None else means
)
if self.head_dtype is not None:
pooled_data = pooled_data.to(self.head_dtype)
# pooled_data shape: [n_tokens, hidden_size]
@@ -84,11 +84,14 @@ class QuantizeMethodBase(ABC):
"""
return
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
def dequantize_weight(
self, layer: nn.Module
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Materialize a serialized quantized weight for requantization."""
raise NotImplementedError(
f"The quantization method {type(self)} does not implement "
"dequantize_weight. Please open an issue."
"dequantize_weight, so re-quantization from this checkpoint is "
"not supported. Please open an issue."
)
@@ -302,6 +305,9 @@ def resolve_quant_method(
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
@@ -328,11 +334,6 @@ def resolve_quant_method(
# The checkpoint quant method is applied as there is no online override.
return base_quant_method
if isinstance(layer, RoutedExperts):
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
online_quant_method = quant_config.online_quantization_config.get_quant_method(
layer, prefix
)
@@ -341,7 +342,7 @@ def resolve_quant_method(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
assert isinstance(online_quant_method, OnlineLinearBase)
assert isinstance(online_quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
online_quant_method.set_requantization_source(base_quant_method)
# The online method dequantizes the checkpoint method before requantizing.
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsW4A4Mxfp4,
CompressedTensorsW4A8Fp8,
CompressedTensorsW4A8Int,
CompressedTensorsW4A8Mxfp4,
CompressedTensorsW8A8Fp8,
CompressedTensorsW8A8Int8,
CompressedTensorsW8A8Mxfp8,
@@ -63,6 +64,11 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
create_fp8_quant_key,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -449,6 +455,39 @@ class CompressedTensorsConfig(QuantizationConfig):
and is_symmetric
)
@classmethod
def _is_mxfp4_w4a8_fp8(
cls, weight_quant: QuantizationArgs, input_quant: QuantizationArgs
) -> bool:
if not cls._is_mxfp4(weight_quant) or input_quant is None:
return False
is_float_type = input_quant.type == QuantizationType.FLOAT
is_8_bits = input_quant.num_bits == 8
is_static_weight = not weight_quant.dynamic
is_dynamic_input = bool(input_quant.dynamic)
is_token_or_group = input_quant.strategy in (
QuantizationStrategy.TOKEN,
QuantizationStrategy.GROUP,
)
return (
is_float_type
and is_8_bits
and is_static_weight
and is_dynamic_input
and is_token_or_group
and input_quant.symmetric
)
@staticmethod
def _fp8_dynamic_input_key(input_quant: QuantizationArgs) -> QuantKey:
if input_quant.strategy == QuantizationStrategy.GROUP:
group_shape = GroupShape(1, input_quant.group_size)
else:
group_shape = GroupShape.PER_TOKEN
return create_fp8_quant_key(static=False, group_shape=group_shape)
@staticmethod
def _is_mxfp8(quant_args: QuantizationArgs) -> bool:
if quant_args is None:
@@ -764,6 +803,11 @@ class CompressedTensorsConfig(QuantizationConfig):
)
return CompressedTensorsW4A4Fp4()
if self._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
return CompressedTensorsW4A8Mxfp4(
activation_quant_key=self._fp8_dynamic_input_key(input_quant)
)
if self._is_mxfp4(weight_quant):
return CompressedTensorsW4A4Mxfp4()
@@ -34,8 +34,12 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
# make sure quantization config for Linear can target it
quant_config._add_fused_moe_to_target_scheme_map()
unfused_names = [
layer_name + proj_name
for proj_name in [".0.gate_proj", ".0.up_proj", ".0.down_proj"]
f"{layer_name}.0.{proj_name}"
for proj_name in (
layer.ckpt_gate_proj_name,
layer.ckpt_up_proj_name,
layer.ckpt_down_proj_name,
)
]
# TODO: refactor this to use expert_mapping and check all layer numbers
all_scheme_dicts = [
@@ -59,6 +63,16 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
input_quant = scheme_dict.get("input_activations")
format = scheme_dict.get("format")
if quant_config._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
from .compressed_tensors_moe_w4a8_mxfp4 import (
CompressedTensorsW4A8Mxfp4MoEMethod,
)
return CompressedTensorsW4A8Mxfp4MoEMethod(
layer.moe_config,
activation_key=quant_config._fp8_dynamic_input_key(input_quant),
)
if quant_config._is_mxfp4(weight_quant):
from .compressed_tensors_moe_w4a4_mxfp4 import (
CompressedTensorsW4A4Mxfp4MoEMethod,
@@ -26,6 +26,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
CompressedTensorsMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_nvfp4_moe_weights,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kNvfp4Dynamic,
kNvfp4Static,
@@ -185,6 +188,22 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
)
set_weight_attrs(w2_input_scale, extra_weight_attrs)
def dequantize_weight(
self, layer: torch.nn.Module
) -> tuple[torch.Tensor, torch.Tensor]:
"""Materialize serialized NVFP4 expert weights for requantization."""
return dequantize_nvfp4_moe_weights(
layer.w13_weight_packed,
layer.w13_weight_scale,
layer.w13_weight_global_scale,
layer.w2_weight_packed,
layer.w2_weight_scale,
layer.w2_weight_global_scale,
layer.params_dtype,
group_size=self.group_size,
invert_global_scales=True,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
# NOTE(rob): wN_weight_packed -> wN_weight is because ModularKernelMethod
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
Only the checkpoint naming and the declared activation scheme differ from
the native MXFP4 path, so backend selection, weight conversion and
execution are inherited.
"""
# The packed -> unpacked rename means a processed state dict no longer
# matches the parameters registered by create_weights.
supports_pre_processed_weights = False
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
super().create_weights(layer, *args, **kwargs)
for name in _PACKED_WEIGHTS:
param = getattr(layer, name)
delattr(layer, name)
layer.register_parameter(f"{name}_packed", param)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
for name in _PACKED_WEIGHTS:
param = getattr(layer, f"{name}_packed")
delattr(layer, f"{name}_packed")
layer.register_parameter(name, param)
super().process_weights_after_loading(layer)
@@ -6,6 +6,7 @@ from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
@@ -23,6 +24,7 @@ __all__ = [
"CompressedTensorsW8A8Int8",
"CompressedTensorsW8A8Fp8",
"CompressedTensorsW4A4Mxfp4",
"CompressedTensorsW4A8Mxfp4",
"CompressedTensorsW4A4Fp4",
"CompressedTensorsW4A8Int",
"CompressedTensorsW4A8Fp8",
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
)
from vllm.model_executor.parameter import (
@@ -35,10 +36,10 @@ class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
Otherwise: W4A16 weight-only via Marlin.
"""
def __init__(self):
def __init__(self, activation_quant_key: QuantKey = kMxfp4Dynamic):
self.group_size = 32
self.kernel = init_mxfp4_linear_kernel(
activation_quant_key=kMxfp4Dynamic,
activation_quant_key=activation_quant_key,
)
@classmethod
@@ -0,0 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxfp4 import ( # noqa: E501
CompressedTensorsW4A4Mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
logger = init_logger(__name__)
__all__ = ["CompressedTensorsW4A8Mxfp4"]
class CompressedTensorsW4A8Mxfp4(CompressedTensorsW4A4Mxfp4):
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
The weight layout matches W4A4; only the activation quantization differs.
The checkpoint's FP8 activations are used when a kernel supports them,
otherwise this falls back to the W4A4 activation format.
"""
def __init__(self, activation_quant_key: QuantKey):
try:
super().__init__(activation_quant_key=activation_quant_key)
except ValueError as e:
logger.warning_once(
"No MXFP4 linear kernel supports %s activations on this "
"platform; falling back to MXFP4 activations. %s",
activation_quant_key,
e,
)
super().__init__()
@@ -6,11 +6,15 @@ from collections.abc import Callable
from fractions import Fraction
import torch
from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
from vllm import envs
from vllm.distributed.utils import verify_group_size_divides_partition
from vllm.logger import init_logger
from vllm.model_executor.determinism.batch_invariant import matmul_batch_invariant
from vllm.model_executor.kernels.linear import (
MarlinLinearKernel,
MPLinearKernel,
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
@@ -28,6 +32,7 @@ from vllm.model_executor.parameter import (
PackedColumnParameter,
PackedvLLMParameter,
)
from vllm.model_executor.utils import replace_parameter
from vllm.scalar_type import scalar_types
logger = init_logger(__name__)
@@ -71,6 +76,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
self.symmetric = symmetric
self.group_size = -1 if group_size is None else group_size
self.layer_name = layer_name
# Marlin picks its tiling and K-split from the number of tokens, so its
# results depend on the batch size. In batch-invariant mode keep the
# checkpoint layout, unpack the weight and use the batch-invariant matmul.
self.use_batch_invariant = envs.VLLM_BATCH_INVARIANT
self.kernel: MPLinearKernel | None = None
if self.group_size == -1 and self.strategy != "channel":
raise ValueError(
@@ -132,11 +142,16 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
zero_points=not self.symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
if self.use_batch_invariant:
kernel_type = None
backend_name = "dequantize + batch-invariant matmul"
else:
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
backend_name = kernel_type.__name__
if kernel_type.__name__ not in self._kernel_backends_being_used:
logger.info("Using %s for CompressedTensorsWNA16", kernel_type.__name__)
self._kernel_backends_being_used.add(kernel_type.__name__)
if backend_name not in self._kernel_backends_being_used:
logger.info("Using %s for CompressedTensorsWNA16", backend_name)
self._kernel_backends_being_used.add(backend_name)
if kernel_type is MarlinLinearKernel:
input_dtype = get_marlin_input_dtype(self.layer_name)
@@ -227,19 +242,49 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
if not self.symmetric:
layer.register_parameter("weight_zero_point", qzeros)
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
if kernel_type is not None:
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
# Checkpoints are serialized in compressed-tensors format, which is
# different from the format the kernel may want. Handle repacking here.
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if self.use_batch_invariant:
# Keep the checkpoint layout: Marlin's reduction depends on M.
for name in ("weight_packed", "weight_scale", "weight_zero_point"):
if hasattr(layer, name):
replace_parameter(layer, name, getattr(layer, name).data)
return
assert self.kernel is not None
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
if self.use_batch_invariant:
shape = torch.Size(
(layer.output_size_per_partition, layer.input_size_per_partition)
)
weight = unpack_from_int32(layer.weight_packed, self.num_bits, shape)
weight = weight.to(layer.weight_scale.dtype).reshape(
shape[0], layer.weight_scale.shape[1], -1
)
if not self.symmetric:
zero_point = unpack_from_int32(
layer.weight_zero_point,
self.num_bits,
layer.weight_scale.shape,
packed_dim=0,
)
weight = weight - zero_point.unsqueeze(-1)
weight = (weight * layer.weight_scale.unsqueeze(-1)).reshape(shape)
output = matmul_batch_invariant(x, weight.t().to(x.dtype))
if bias is not None:
output.add_(bias)
return output
assert self.kernel is not None
return self.kernel.apply_weights(layer, x, bias)
@@ -150,16 +150,16 @@ def scaled_mm_kernel(
scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1))
accumulator = scale_b.T * accumulator.to(tl.float32)
# Convert to output format.
c = accumulator.to(c_ptr.type.element_ty)
# Add bias, it's already in output format, so add it after conversion.
# Add bias in accumulator dtype for less rounding error.
if bias_ptr:
offsets_bias = offsets_bn
bias_ptrs = bias_ptr + offsets_bias
bias_mask = offsets_bias < N
bias = tl.load(bias_ptrs, bias_mask)
c += bias
accumulator += bias
# Convert to output format.
c = accumulator.to(c_ptr.type.element_ty)
# Save output
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
@@ -224,6 +224,8 @@ def triton_scaled_mm(
tile_shape = (64, 128, 128)
else:
tile_shape = (128, 128, 128)
else:
tile_shape = (block_size_m, block_size_n, block_size_k)
block_size_m, block_size_n, block_size_k = tile_shape
@@ -9,10 +9,10 @@ import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.model_executor.kernels.linear import (
fp8_scaled_mm_is_batch_invariant,
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention import Attention
@@ -432,7 +432,7 @@ class Fp8LinearMethod(LinearMethodBase):
bias,
)
else:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
return self.fp8_linear.apply_weights(layer, x, bias)
# per-tensor/channel: dequant to BF16 and run GEMM
@@ -584,7 +584,9 @@ class HummingLinearMethod(LinearMethodBase):
param_dtype=layer.param_dtype,
allow_fallback=self.quant_config.allow_input_schema_fallback,
)
self.layer_config = _hm.prepare_layer_config(
source_tensors = dict(layer.named_parameters())
self.layer_config, tensors = _hm.prepare_and_transform_humming_tensors(
tensors=source_tensors,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=self.weight_schema,
@@ -595,8 +597,6 @@ class HummingLinearMethod(LinearMethodBase):
torch_dtype=layer.param_dtype,
device=layer.weight.device,
)
source_tensors = dict(layer.named_parameters())
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
name = self.input_schema.static_tensor_scale_name
if name is not None:
tensors[name] = source_tensors[name]
@@ -610,7 +610,7 @@ class HummingLinearMethod(LinearMethodBase):
setattr(layer, name, param)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = _hm.create_humming_locks(layer.weight.device)
def apply(
self,
@@ -244,10 +244,12 @@ class QuantFP8(CustomOp):
x_grouped = x.view(-1, num_groups, self.group_size)
absmax = x_grouped.abs().max(dim=-1, keepdim=True)[0].float()
scales_raw = absmax / _FP8_MAX
if self.use_ue8m0:
scales_raw = torch.exp2(torch.ceil(torch.log2(scales_raw)))
scales = (scales_raw).clamp(min=_FP8_MIN_SCALING_FACTOR)
scales_raw = (absmax / _FP8_MAX).clamp(min=_FP8_MIN_SCALING_FACTOR)
scales = (
torch.exp2(torch.ceil(torch.log2(scales_raw)))
if self.use_ue8m0
else scales_raw
)
x_scaled = x_grouped / scales
x_quant = x_scaled.clamp(_FP8_MIN, _FP8_MAX).to(_FP8_DTYPE)
@@ -81,6 +81,9 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_VALUE_DTYPE,
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_nvfp4_moe_weights,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
QuantKey,
@@ -99,7 +102,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
requantize_with_max_scale,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
@@ -111,6 +117,7 @@ from vllm.model_executor.utils import (
replace_parameter,
set_weight_attrs,
)
from vllm.utils.b12x import B12X_BLOCK_CODECS
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
@@ -885,6 +892,22 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
"""FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales."""
return True
def dequantize_weight(
self, layer: torch.nn.Module
) -> tuple[torch.Tensor, torch.Tensor]:
"""Materialize serialized ModelOpt NVFP4 expert weights."""
return dequantize_nvfp4_moe_weights(
layer.w13_weight,
layer.w13_weight_scale,
layer.w13_weight_scale_2,
layer.w2_weight,
layer.w2_weight_scale,
layer.w2_weight_scale_2,
layer.params_dtype,
group_size=self.quant_config.group_size,
invert_global_scales=False,
)
def create_weights(
self,
layer: RoutedExperts,
@@ -1559,6 +1582,19 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
super().__init__(exclude_modules)
self.kv_cache_quant_method = kv_cache_quant_method
self.quantized_layers = quantized_layers
for prefix, recipe in quantized_layers.items():
codec = recipe.get("quant_algo", "").lower()
if codec in B12X_BLOCK_CODECS and any(
recipe.get(key) != value
for key, value in {
"group_size": B12X_BLOCK_CODECS[codec][0],
"block_payload_bytes": B12X_BLOCK_CODECS[codec][1],
"packing": "ggml",
}.items()
):
raise ValueError(
f"unsupported {codec.upper()} block contract for {prefix}: {recipe}"
)
self.fp8_config = fp8_config
self.nvfp4_config = nvfp4_config
self.w4a16_nvfp4_config = w4a16_nvfp4_config
@@ -1811,6 +1847,22 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
quant_algo = self._resolve_quant_algo(prefix)
if quant_algo is not None and quant_algo.lower() in B12X_BLOCK_CODECS:
from .modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
ModelOptBlockQuantMoEMethod,
)
if isinstance(layer, RoutedExperts):
return ModelOptBlockQuantMoEMethod(
layer.moe_config, codec=quant_algo.lower()
)
if isinstance(layer, (LinearBase, VocabParallelEmbedding)):
return ModelOptBlockQuantLinearMethod(codec=quant_algo.lower())
raise ValueError(
f"{quant_algo} requires a linear, embedding or routed experts"
)
if isinstance(layer, (LinearBase, ParallelLMHead)):
# Per-prefix algo -> its sub-config, then the generic linear method.
# resolve() reads group_size off whichever sub-config it is handed.
@@ -0,0 +1,435 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ModelOpt safetensors block codecs with BF16 activations."""
import torch
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.b12x import B12xExperts
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
FusedMoEQuantDesc,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.linear import (
LinearMethodBase,
register_weight_loader_v2_supported_method,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.parameter import ModelWeightParameter
from vllm.utils.b12x import B12X_BLOCK_CODECS, B12xWarmupUnit, get_b12x_blockscaled
@torch.library.custom_op(
"vllm::modelopt_block_embedding_chunked", mutates_args=("out",)
)
def _modelopt_block_embedding_chunked(
weight: torch.Tensor, ids: torch.Tensor, out: torch.Tensor, plan_handle: int
) -> None:
from b12x.preparation.types import plan_from_handle
from b12x.sequence import embedding
plan = plan_from_handle(plan_handle)
capacity = plan.query.max_rows
for start in range(0, ids.numel(), capacity):
embedding.run(
weight,
ids[start : start + capacity],
out=out[start : start + capacity],
plan=plan,
)
@_modelopt_block_embedding_chunked.register_fake
def _modelopt_block_embedding_chunked_fake(
weight: torch.Tensor, ids: torch.Tensor, out: torch.Tensor, plan_handle: int
) -> None:
return None
class ModelOptBlockQuantMoEMethod(FusedMoEMethodBase):
"""Load native blocks and retain compact b12x prepared expert storage."""
def __init__(self, moe_config: FusedMoEConfig, codec: str = "iq2_xs"):
super().__init__(moe_config)
self.codec = codec.lower()
self.block_size, self.block_bytes = B12X_BLOCK_CODECS[self.codec]
if moe_config.moe_backend not in ("auto", "b12x"):
raise ValueError(
"Block-quantized routed experts require the b12x MoE backend"
)
if not B12xExperts._supports_current_device():
raise ValueError("Block-quantized routed experts require b12x on SM12x")
if not B12xExperts._supports_parallel_config(moe_config.moe_parallel_config):
raise ValueError(
"Block-quantized supports tensor parallelism without EP or EPLB"
)
if moe_config.in_dtype != torch.bfloat16:
raise ValueError("Block-quantized routed experts require BF16 activations")
if (
moe_config.activation
not in (MoEActivation.SILU, MoEActivation.RELU2_NO_MUL)
or moe_config.has_bias
):
raise ValueError(
"Block-quantized routed experts require bias-free SiLU or ReLU2"
)
self.gated = moe_config.activation == MoEActivation.SILU
if any(
value is not None
for value in (
moe_config.swiglu_limit,
moe_config.swiglu_alpha,
moe_config.swiglu_beta,
)
):
raise ValueError(
"Block-quantized routed experts require standard SiLU parameters"
)
def maybe_roundup_sizes(
self,
hidden_size,
intermediate_size_per_partition,
act_dtype,
moe_parallel_config,
):
alignment = max(128, self.block_size)
if hidden_size % alignment or intermediate_size_per_partition % alignment:
raise ValueError(
f"{self.codec.upper()} hidden and per-rank intermediate sizes "
f"must align to {alignment}"
)
return hidden_size, intermediate_size_per_partition
def create_weights(
self,
layer,
num_experts,
hidden_size,
intermediate_size_per_partition,
params_dtype,
**extra_weight_attrs,
):
self.maybe_roundup_sizes(
hidden_size,
intermediate_size_per_partition,
params_dtype,
self.moe.moe_parallel_config,
)
for name, shape in (
(
"w13_weight",
(
num_experts,
(2 if self.gated else 1) * intermediate_size_per_partition,
hidden_size // self.block_size,
self.block_bytes,
),
),
(
"w2_weight",
(
num_experts,
hidden_size,
intermediate_size_per_partition // self.block_size,
self.block_bytes,
),
),
):
layer.register_parameter(
name,
ModelWeightParameter(
data=torch.empty(shape, dtype=torch.uint8),
input_dim=2,
output_dim=1,
weight_loader=self.weight_loader,
),
)
def weight_loader(
self,
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success=False,
):
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"invalid Block-quantized expert projection: {shard_id}")
if not self.gated and shard_id == "w3":
raise ValueError("non-gated Block-quantized experts have no w3 projection")
local_i = self.moe.intermediate_size_per_partition
global_i = local_i * self.moe.moe_parallel_config.tp_size
hidden = self.moe.hidden_dim
expected = (
(hidden, global_i // self.block_size, self.block_bytes)
if shard_id == "w2"
else (global_i, hidden // self.block_size, self.block_bytes)
)
if loaded_weight.dtype != torch.uint8 or tuple(loaded_weight.shape) != expected:
raise ValueError(
f"invalid packed tensor {weight_name}: expected uint8{expected}, "
f"got {loaded_weight.dtype}{tuple(loaded_weight.shape)}"
)
if not 0 <= expert_id < param.shape[0]:
raise ValueError(f"invalid Block-quantized expert index: {expert_id}")
destination = param.data[expert_id]
if shard_id == "w2":
source = loaded_weight.narrow(
1,
self.moe.tp_rank * (local_i // self.block_size),
local_i // self.block_size,
)
else:
source = loaded_weight.narrow(0, self.moe.tp_rank * local_i, local_i)
destination = destination.narrow(
0, 0 if shard_id == "w1" else local_i, local_i
)
destination.copy_(source)
return True if return_success else None
def get_fused_moe_quant_config(self, layer):
return FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(),
_a2=FusedMoEQuantDesc(),
_w1=FusedMoEQuantDesc(dtype=self.codec),
_w2=FusedMoEQuantDesc(dtype=self.codec),
)
def process_weights_after_loading(self, layer):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
prepare_finalize = maybe_make_prepare_finalize(
moe=self.moe,
quant_config=self.moe_quant_config,
routing_tables=layer._expert_routing_tables(),
allow_new_interface=True,
)
assert prepare_finalize is not None
experts = B12xExperts(self.moe, self.moe_quant_config)
self.moe_kernel = mk.FusedMoEKernel(prepare_finalize, experts)
experts.process_weights_after_loading(layer)
def apply(
self,
layer,
x,
topk_weights,
topk_ids,
shared_experts,
shared_experts_input,
):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
ModelOptBlockQuantMoEMethod.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
@register_weight_loader_v2_supported_method
class ModelOptBlockQuantLinearMethod(LinearMethodBase):
"""Load native block payloads and execute the prepared b12x dense API."""
def __init__(self, codec: str = "iq2_xs"):
self.codec = codec.lower()
self.block_size, self.block_bytes = B12X_BLOCK_CODECS[self.codec]
self.is_embedding = False
from vllm.model_executor.kernels.linear import _get_linear_backend
from vllm.utils.b12x import get_b12x_blockscaled
if _get_linear_backend(quantization=self.codec) not in ("auto", "b12x"):
raise ValueError(
"Block-quantized dense weights require the b12x linear backend"
)
api = get_b12x_blockscaled()
if (
api is None
or not hasattr(api, "BlockQuantLinearWeight")
or not api.is_supported()
):
raise ValueError("Block-quantized dense weights require b12x on SM12x")
def create_weights(
self,
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
):
n = sum(output_partition_sizes)
self.is_embedding = isinstance(
layer, VocabParallelEmbedding
) and not isinstance(layer, ParallelLMHead)
if self.is_embedding and self.codec != "q8_0":
raise ValueError("packed embedding currently requires Q8_0")
if (
params_dtype != torch.bfloat16
or input_size_per_partition % self.block_size
or n % 8
):
raise ValueError(
f"{self.codec.upper()} dense weights require BF16, "
f"K{self.block_size} and N8"
)
layer.register_parameter(
"weight",
ModelWeightParameter(
data=torch.empty(
(n, input_size_per_partition // self.block_size, self.block_bytes),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs["weight_loader"],
),
)
def process_weights_after_loading(self, layer):
from b12x.sequence import embedding
from vllm.model_executor.utils import replace_parameter
api = get_b12x_blockscaled()
assert api is not None
config = get_current_vllm_config()
scheduler = config.scheduler_config
capture_sizes = config.compilation_config.cudagraph_capture_sizes or []
capacity = max(
scheduler.max_num_batched_tokens,
scheduler.max_num_scheduled_tokens or 0,
*capture_sizes,
)
weight = layer.weight.data
if self.is_embedding:
bases = weight[..., :2].contiguous().view(torch.float16)
if not torch.isfinite(bases).all().item():
raise ValueError("Q8_0 embedding scales must be finite")
layer.b12x_embedding_plans = {}
width = weight.shape[1] * self.block_size
if config.model_config is not None:
capacity = max(capacity, config.model_config.max_model_len)
for dtype in (torch.int32, torch.int64):
plan = embedding.plan(
embedding.EmbeddingQuery(
max_rows=capacity,
table_rows=weight.shape[0],
width=width,
row_stride=weight.stride(0),
weight_dtype="uint8",
id_dtype=str(dtype).removeprefix("torch."),
),
device=weight.device,
)
layer.b12x_embedding_plans[dtype] = plan
else:
packed = api.pack_weight(weight, recipe=self.codec)
query = api.BlockscaledQuery(
recipe=self.codec,
num_tokens=capacity,
in_features=packed.in_features,
padded_in_features=packed.in_features,
out_features=packed.out_features,
activation_mode="a16",
)
plan = api.plan_regimes(
query,
exact_m=tuple(sorted({m for m in capture_sizes if 0 < m < capacity})),
)
layer.b12x_block_weight = packed
layer.b12x_block_plan = plan
if not self.is_embedding:
replace_parameter(
layer, "weight", torch.empty(0, dtype=torch.uint8, device=weight.device)
)
layer.b12x_warmup_provider = self
def get_b12x_warmup_unit(
self,
layer: torch.nn.Module,
token_counts: tuple[int, ...],
output_dtype: torch.dtype,
) -> B12xWarmupUnit:
if self.is_embedding:
plans = tuple(layer.b12x_embedding_plans.values())
else:
plans = (layer.b12x_block_plan,)
def compile() -> None:
if self.is_embedding:
for dtype, plan in layer.b12x_embedding_plans.items():
ids = torch.zeros(
plan.query.max_rows, dtype=dtype, device=layer.weight.device
)
self.embedding(layer, ids)
else:
packed = layer.b12x_block_weight
for tokens in sorted(set(token_counts) | set(plans[0].token_counts)):
source = torch.zeros(
(tokens, packed.in_features),
dtype=output_dtype,
device=packed.values.device,
)
self.apply(layer, source)
return B12xWarmupUnit(
name=(
self.codec.upper() + (" embedding" if self.is_embedding else " linear")
),
# Every layer owns a plan that must be prepared before capture.
key=(type(self), *(plan.handle for plan in plans)),
compile=compile,
)
def apply(self, layer, x, bias=None):
if x.dtype != torch.bfloat16:
raise ValueError(
"Block-quantized dense execution requires BF16 activations"
)
api = get_b12x_blockscaled()
assert api is not None
output = api.mm(
x.reshape(-1, x.shape[-1]).contiguous(),
layer.b12x_block_weight,
plan=layer.b12x_block_plan,
bias=bias,
)
return output.view(*x.shape[:-1], layer.b12x_block_weight.out_features)
def embedding(self, layer, input_):
plan = layer.b12x_embedding_plans[input_.dtype]
width = layer.weight.shape[1] * self.block_size
out = torch.empty(
(*input_.shape, width), dtype=torch.bfloat16, device=layer.weight.device
)
ids, rows = input_.reshape(-1), out.view(-1, width)
_modelopt_block_embedding_chunked(layer.weight, ids, rows, plan.handle)
return out
@@ -38,7 +38,10 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
is_layer_skipped,
)
from vllm.model_executor.utils import (
is_weights_pre_processed,
replace_parameter,
@@ -541,9 +544,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
supports_pre_processed_weights = True
def __init__(self, moe: FusedMoEConfig):
def __init__(self, moe: FusedMoEConfig, activation_key: QuantKey | None = None):
super().__init__(moe)
self.weight_dtype = "mxfp4"
# Checkpoint-declared activation quantization. Only backends that pick
# their activation format from the layer (Humming) consume it.
self.activation_key = activation_key
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
@@ -822,6 +828,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
_cache_permute_indices=self._cache_permute_indices,
activation=self.moe.activation,
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
activation_key=self.activation_key,
)
)
@@ -15,9 +15,11 @@ if TYPE_CHECKING:
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.kernels.linear import (
fp8_scaled_mm_is_batch_invariant,
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -32,10 +34,12 @@ from vllm.model_executor.layers.fusion.quant_activation import (
from vllm.model_executor.layers.linear import (
LinearMethodBase,
)
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.online.requantization import (
OnlineRequantizationMixin,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
@@ -116,7 +120,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class OnlineLinearBase(LinearMethodBase):
class OnlineLinearBase(OnlineRequantizationMixin, LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -124,15 +128,17 @@ class OnlineLinearBase(LinearMethodBase):
activation_quant_key: QuantKey | None
def __init__(self):
super().__init__()
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
self.requantization_source: QuantizeMethodBase | None = None
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Configure serialized-weight conversion before online quantization."""
self.requantization_source = source_method
self.uses_meta_device = False
def get_weight_for_quantization(self, layer: torch.nn.Module) -> torch.Tensor:
if self.requantization_source is None:
return layer.weight
weight = self.requantization_source.dequantize_weight(layer)
if not isinstance(weight, torch.Tensor):
raise TypeError("Linear requantization requires a single weight tensor.")
return weight
def create_weights(
self,
@@ -144,22 +150,15 @@ class OnlineLinearBase(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if self.requantization_source is not None:
existing_parameter_names = set(layer._parameters)
self.requantization_source.create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
self.requantization_source_parameters = {
name: parameter
for name, parameter in layer._parameters.items()
if name not in existing_parameter_names and parameter is not None
}
if self.create_requantization_source_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
):
return
output_size_per_partition = sum(output_partition_sizes)
@@ -185,19 +184,6 @@ class OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
"""Release checkpoint parameters after successful requantization."""
for name, source_parameter in self.requantization_source_parameters.items():
if layer._parameters.get(name) is source_parameter:
delattr(layer, name)
self.requantization_source_parameters.clear()
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
"""Return checkpoint weights materialized for online quantization."""
if self.requantization_source is None:
return layer.weight
return self.requantization_source.dequantize_weight(layer)
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
@@ -266,6 +252,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
self.fp8_linear.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -278,7 +265,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
) -> torch.Tensor:
# if batch invariant mode is enabled, use BF16 dequant
if envs.VLLM_BATCH_INVARIANT:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
return self.fp8_linear.apply_weights(layer, x, bias)
weight_fp8 = layer.weight.to(torch.bfloat16)
@@ -366,6 +353,7 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
del layer.weight_scale
self.fp8_linear.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -466,7 +454,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
# if batch invariant mode is enabled dequant
if (
envs.VLLM_BATCH_INVARIANT
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
and not fp8_scaled_mm_is_batch_invariant(self.fp8_linear)
and not isinstance(x, QuantizedActivation)
):
weight_dequant = (
@@ -622,27 +610,29 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
# If checkpoint is fp16, quantize in place.
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
layer.w13_input_scale = None
layer.w2_input_scale = None
moe_tp_size = self.moe.tp_size
w13_amax = weight_amax(layer.w13_weight.flatten(1), dim=-1)
w13_amax = weight_amax(w13_weight.flatten(1), dim=-1)
w13_amax = amax_for_moe_weight_quant(w13_amax, moe_tp_size)
w13_scale = _fp8_scale(w13_amax)
w2_amax = weight_amax(layer.w2_weight.flatten(1), dim=-1)
w2_amax = weight_amax(w2_weight.flatten(1), dim=-1)
w2_amax = amax_for_moe_weight_quant(w2_amax, moe_tp_size)
w2_scale = _fp8_scale(w2_amax)
for expert in range(layer.local_num_experts):
w13[expert, :, :], _ = ops.scaled_fp8_quant(
layer.w13_weight[expert, :, :], scale=w13_scale[expert]
w13_weight[expert, :, :], scale=w13_scale[expert]
)
w2[expert, :, :], _ = ops.scaled_fp8_quant(
layer.w2_weight[expert, :, :], scale=w2_scale[expert]
w2_weight[expert, :, :], scale=w2_scale[expert]
)
# Shuffle weights to runtime format and setup kernel.
@@ -655,6 +645,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
w13_input_scale=layer.w13_input_scale,
w2_input_scale=layer.w2_input_scale,
)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -698,11 +689,12 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._zero_padding(layer)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
self._zero_padding(layer, w13_weight, w2_weight)
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
block_size = self.weight_block_size
assert block_size is not None
@@ -711,8 +703,8 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
# Create block-shaped scales (computed here rather than in
# create_weights because online quant doesn't need them until now).
num_experts = layer.local_num_experts
_, w13_out, w13_in = layer.w13_weight.shape
_, w2_out, w2_in = layer.w2_weight.shape
_, w13_out, w13_in = w13_weight.shape
_, w2_out, w2_in = w2_weight.shape
w13_scale = torch.ones(
num_experts,
@@ -731,18 +723,22 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
for expert in range(num_experts):
w13[expert], w13_scale[expert] = per_block_cast_to_fp8(
layer.w13_weight[expert],
w13_weight[expert],
block_size=block_size,
use_ue8m0=False,
)
w2[expert], w2_scale[expert] = per_block_cast_to_fp8(
layer.w2_weight[expert],
w2_weight[expert],
block_size=block_size,
use_ue8m0=False,
)
layer.weight_block_size = block_size
# Activation quantization is dynamic (including in re-quantization)
layer.w13_input_scale = None
layer.w2_input_scale = None
# Shuffle weights to runtime format and setup kernel.
self._setup_kernel(
layer,
@@ -750,9 +746,10 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
w2,
w13_scale,
w2_scale,
layer.w13_input_scale,
layer.w2_input_scale,
w13_input_scale=None,
w2_input_scale=None,
)
self.release_requantization_source_weights(layer)
# Prevent duplicate processing (e.g., during weight reload)
layer._already_called_process_weights_after_loading = True
@@ -804,32 +801,32 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
# Scale's leading dim is taken from the fp8 weight tensor by
# construction, so it cannot drift from the weight's expert count
# under EP / padded MoE.
n_w13 = layer.w13_weight.shape[1]
n_w13 = w13_weight.shape[1]
w13_scale = torch.ones(
w13.shape[0], n_w13, 1, device=w13.device, dtype=torch.float32
)
layer.w13_input_scale = None
layer.w2_input_scale = None
w2_amax = weight_amax(layer.w2_weight, dim=-1, keepdim=True)
w2_amax = weight_amax(w2_weight, dim=-1, keepdim=True)
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
w2_scale = _fp8_channel_scale(w2_amax)
for expert in range(layer.local_num_experts):
w13[expert], w13_scale[expert] = ops.scaled_fp8_quant(
layer.w13_weight[expert],
w13_weight[expert],
scale=None,
use_per_token_if_dynamic=True,
)
w2[expert] = _fp8_quant_per_channel(
layer.w2_weight[expert], w2_scale[expert]
)
w2[expert] = _fp8_quant_per_channel(w2_weight[expert], w2_scale[expert])
self._setup_kernel(
layer,
@@ -840,5 +837,6 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
w13_input_scale=None,
w2_input_scale=None,
)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -52,42 +52,51 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._quantize_weights(layer)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
self._quantize_weights(layer, w13_weight, w2_weight)
self._setup_kernel(layer)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
def _quantize_weights(self, layer: Module) -> None:
def _quantize_weights(
self,
layer: Module,
w13_weight: torch.Tensor | None = None,
w2_weight: torch.Tensor | None = None,
) -> None:
w13_weight = layer.w13_weight if w13_weight is None else w13_weight
w2_weight = layer.w2_weight if w2_weight is None else w2_weight
vmax = torch.iinfo(torch.int8).max
w13 = torch.empty_like(layer.w13_weight, dtype=torch.int8)
w2 = torch.empty_like(layer.w2_weight, dtype=torch.int8)
w13 = torch.empty_like(w13_weight, dtype=torch.int8)
w2 = torch.empty_like(w2_weight, dtype=torch.int8)
w13_scale = torch.zeros(
layer.num_experts,
layer.w13_weight.shape[1],
w13_weight.shape[1],
device=w13.device,
dtype=torch.float32,
)
w2_scale = torch.zeros(
layer.num_experts,
layer.w2_weight.shape[1],
w2_weight.shape[1],
device=w2.device,
dtype=torch.float32,
)
w2_amax = weight_amax(layer.w2_weight, dim=-1)
w2_amax = weight_amax(w2_weight, dim=-1)
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
for expert in range(layer.local_num_experts):
# w13: per-row quantization over hidden_size dim
w = layer.w13_weight[expert, :, :]
w = w13_weight[expert, :, :]
scales = w.abs().amax(dim=1) / vmax
q = w.div(scales.unsqueeze(1)).round().clamp(-vmax, vmax)
w13[expert, :, :] = q.to(torch.int8)
w13_scale[expert, :] = scales
# w2: per-row quantization over intermediate_size dim
w = layer.w2_weight[expert, :, :]
w = w2_weight[expert, :, :]
scales = w2_amax[expert] / vmax
q = w.div(scales.unsqueeze(1)).round().clamp(-vmax, vmax)
w2[expert, :, :] = q.to(torch.int8)
@@ -10,26 +10,26 @@ from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.online.requantization import (
OnlineRequantizationMixin,
)
from vllm.model_executor.model_loader.reload.layerwise import (
initialize_online_processing,
)
from vllm.model_executor.utils import set_weight_attrs
class OnlineMoEMethodBase(FusedMoEMethodBase):
class OnlineMoEMethodBase(OnlineRequantizationMixin, FusedMoEMethodBase):
"""Base for MoE methods that load full-precision weights on meta device
and quantize them after loading via the QeRL layerwise processing system.
"""
uses_meta_device: bool = True
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Reject requantization from a checkpoint-quantized MoE method."""
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
def create_weights(
self,
@@ -40,10 +40,22 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
# Target conversion needs this metadata even when the source creates weights.
layer.num_experts = num_experts
layer.orig_dtype = params_dtype
layer.weight_block_size = None
if self.requantization_source is not None:
self.create_requantization_source_weights(
layer,
num_experts,
layer.moe_config.hidden_dim_unpadded,
layer.moe_config.intermediate_size_per_partition_unpadded,
params_dtype,
**extra_weight_attrs,
)
return
# Fused gate_up_proj (column parallel) — full precision on meta device
w13_weight = torch.nn.Parameter(
torch.empty(
@@ -103,22 +115,95 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
initialize_online_processing(layer)
def _zero_padding(self, layer: torch.nn.Module) -> None:
def get_weights_for_quantization(
self, layer: torch.nn.Module
) -> tuple[torch.Tensor, torch.Tensor]:
if self.requantization_source is None:
return layer.w13_weight, layer.w2_weight
weights = self.requantization_source.dequantize_weight(layer)
if not isinstance(weights, tuple):
raise TypeError("MoE requantization requires w13 and w2 weight tensors.")
w13, w2 = weights
config = layer.moe_config
hidden_size = config.hidden_dim_unpadded
intermediate_size = config.intermediate_size_per_partition_unpadded
target_hidden_size = config.hidden_dim
target_intermediate_size = config.intermediate_size_per_partition
num_experts = layer.local_num_experts
num_shards = self.moe.w13_num_shards
if w13.shape != (num_experts, num_shards * intermediate_size, hidden_size):
raise ValueError("Requantization requires unpadded source w13 weights.")
if w2.shape != (num_experts, hidden_size, intermediate_size):
raise ValueError("Requantization requires unpadded source w2 weights.")
if (
target_hidden_size < hidden_size
or target_intermediate_size < intermediate_size
):
raise ValueError("Requantization target dimensions cannot shrink weights.")
# No padding required.
if (hidden_size, intermediate_size) == (
target_hidden_size,
target_intermediate_size,
):
return w13, w2
if any(
getattr(layer, name, None) is not None for name in ("w13_bias", "w2_bias")
):
raise NotImplementedError(
"Padded requantization with biases is unsupported."
)
def pad_moe_weights(
w13: torch.Tensor, w2: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
padded_w13 = w13.new_zeros(
(num_experts, num_shards * target_intermediate_size, target_hidden_size)
)
padded_w2 = w2.new_zeros(
(num_experts, target_hidden_size, target_intermediate_size)
)
for shard in range(num_shards):
source_start = shard * intermediate_size
target_start = shard * target_intermediate_size
padded_w13[
:, target_start : target_start + intermediate_size, :hidden_size
] = w13[:, source_start : source_start + intermediate_size, :]
padded_w2[:, :hidden_size, :intermediate_size] = w2
return padded_w13, padded_w2
return pad_moe_weights(w13, w2)
def _zero_padding(
self,
layer: torch.nn.Module,
w13_weight: torch.Tensor | None = None,
w2_weight: torch.Tensor | None = None,
) -> None:
w13_weight = layer.w13_weight if w13_weight is None else w13_weight
w2_weight = layer.w2_weight if w2_weight is None else w2_weight
hidden_size = layer.moe_config.hidden_dim_unpadded
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
w13_shard = w13_weight.shape[1] // self.moe.w13_num_shards
if w13_shard > intermediate_size:
for shard in range(self.moe.w13_num_shards):
start = shard * w13_shard + intermediate_size
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
if layer.w13_weight.shape[2] > hidden_size:
layer.w13_weight[:, :, hidden_size:] = 0
w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
if w13_weight.shape[2] > hidden_size:
w13_weight[:, :, hidden_size:] = 0
if layer.w2_weight.shape[1] > hidden_size:
layer.w2_weight[:, hidden_size:, :] = 0
if layer.w2_weight.shape[2] > intermediate_size:
layer.w2_weight[:, :, intermediate_size:] = 0
if w2_weight.shape[1] > hidden_size:
w2_weight[:, hidden_size:, :] = 0
if w2_weight.shape[2] > intermediate_size:
w2_weight[:, :, intermediate_size:] = 0
if getattr(layer, "w13_bias", None) is not None:
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
@@ -127,6 +127,7 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
replace_parameter(layer, "weight_scale", weight_scale.data)
self.kernel.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -295,7 +296,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._zero_padding(layer)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
self._zero_padding(layer, w13_weight, w2_weight)
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
layer._already_called_process_weights_after_loading = True
@@ -304,8 +306,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
layer.w13_input_scale = None
layer.w2_input_scale = None
w13, w13_scale = _quantize_mxfp4_moe_weight(layer.w13_weight)
w2, w2_scale = _quantize_mxfp4_moe_weight(layer.w2_weight)
w13, w13_scale = _quantize_mxfp4_moe_weight(w13_weight)
w2, w2_scale = _quantize_mxfp4_moe_weight(w2_weight)
self._setup_kernel(
layer,
@@ -316,5 +318,6 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -34,7 +34,6 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
@@ -86,6 +85,7 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
replace_parameter(layer, "weight_scale", weight_scale.data)
self.kernel.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -237,14 +237,12 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
fp8_dtype = current_platform.fp8_dtype()
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
layer.w13_input_scale = None
layer.w2_input_scale = None
w13, w13_scale = self._quantize_mxfp8_moe_weight(layer.w13_weight)
w2, w2_scale = self._quantize_mxfp8_moe_weight(layer.w2_weight)
w13, w13_scale = self._quantize_mxfp8_moe_weight(w13_weight)
w2, w2_scale = self._quantize_mxfp8_moe_weight(w2_weight)
self._setup_kernel(
layer,
@@ -255,5 +253,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
layer.w13_input_scale,
layer.w2_input_scale,
)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -116,19 +116,26 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._quantize_weights(layer)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
self._quantize_weights(layer, w13_weight, w2_weight)
self._setup_kernel(layer)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
def _quantize_weights(self, layer: Module) -> None:
def _quantize_weights(
self,
layer: Module,
w13_weight: torch.Tensor | None = None,
w2_weight: torch.Tensor | None = None,
) -> None:
w13_weight = layer.w13_weight if w13_weight is None else w13_weight
w2_weight = layer.w2_weight if w2_weight is None else w2_weight
moe_tp_size = self.moe.tp_size
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(
layer.w13_weight, moe_tp_size
)
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(
layer.w2_weight, moe_tp_size
w13_weight, moe_tp_size
)
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(w2_weight, moe_tp_size)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w13_weight_scale", w13_scale)
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
class OnlineRequantizationMixin:
"""Shared lifecycle for converting serialized weights before quantization."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.requantization_source: QuantizeMethodBase | None = None
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Configure serialized-weight conversion before online quantization."""
self.requantization_source = source_method
self.uses_meta_device = False
def create_requantization_source_weights(
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
) -> bool:
"""Delegate weight creation to the source and track its parameters."""
if self.requantization_source is None:
return False
existing_parameter_names = set(layer._parameters)
self.requantization_source.create_weights(
layer, *weight_args, **extra_weight_attrs
)
self.requantization_source_parameters = {
name: parameter
for name, parameter in layer._parameters.items()
if name not in existing_parameter_names and parameter is not None
}
return True
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
"""Release serialized parameters after successful requantization."""
for name, source_parameter in self.requantization_source_parameters.items():
if layer._parameters.get(name) is source_parameter:
delattr(layer, name)
self.requantization_source_parameters.clear()
@@ -69,6 +69,9 @@ from vllm.model_executor.layers.quantization.quark.utils import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
validate_fp8_block_shape_moe,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_nvfp4_moe_weights,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
@@ -2110,6 +2113,22 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
layer.register_parameter("w2_input_scale_2", w2_input_scale_2)
set_weight_attrs(w2_input_scale_2, extra_weight_attrs)
def dequantize_weight(
self, layer: torch.nn.Module
) -> tuple[torch.Tensor, torch.Tensor]:
"""Dequantize Quark NVFP4 expert weights for requantization."""
return dequantize_nvfp4_moe_weights(
layer.w13_weight,
layer.w13_weight_scale,
layer.w13_weight_scale_2,
layer.w2_weight,
layer.w2_weight_scale,
layer.w2_weight_scale_2,
layer.params_dtype,
group_size=self.group_size,
invert_global_scales=False,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
@@ -1041,7 +1041,10 @@ def w8a8_triton_block_scaled_mm(
C_shape = A.shape[:-1] + (N,)
C = A.new_empty(C_shape, dtype=output_dtype)
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
if envs.VLLM_BATCH_INVARIANT:
configs = None # use the default config
else:
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
if configs:
# Get the optimal config if there is one
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
@@ -59,13 +59,13 @@ def prepare_humming_linear_layer_config(
input_quant_config: dict | None = None,
*,
input_schema: "BaseInputSchema | None" = None,
allow_input_schema_fallback: bool = True,
) -> "LayerConfig":
from vllm.utils.humming import (
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
prepare_layer_config,
transform_humming_tensors,
prepare_and_transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
@@ -120,6 +120,7 @@ def prepare_humming_linear_layer_config(
weight_schema=weight_schema,
input_schema=input_schema,
param_dtype=layer.params_dtype,
allow_fallback=allow_input_schema_fallback,
)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
@@ -127,7 +128,8 @@ def prepare_humming_linear_layer_config(
input_tensors.pop(name, None)
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
config, tensors = prepare_and_transform_humming_tensors(
tensors=tensors,
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
@@ -138,7 +140,6 @@ def prepare_humming_linear_layer_config(
torch_dtype=layer.params_dtype,
device=tensors["weight"].device,
)
tensors = transform_humming_tensors(config, tensors)
tensors.update(input_tensors)
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
@@ -397,11 +397,15 @@ def _prepare_and_transform_sublayer(
) -> "LayerConfig":
"""Prepare Humming configuration and transform one sublayer's tensors."""
from vllm.utils.humming import (
prepare_layer_config,
transform_humming_tensors,
prepare_and_transform_humming_tensors,
)
config = prepare_layer_config(
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
source_tensors.pop(name, None)
config, tensors = prepare_and_transform_humming_tensors(
tensors=source_tensors,
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
@@ -413,11 +417,6 @@ def _prepare_and_transform_sublayer(
torch_dtype=param_dtype,
device=getattr(layer, f"{sublayer_name}_weight").device,
)
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
source_tensors.pop(name, None)
tensors = transform_humming_tensors(config, source_tensors)
for name in ("input_scale", "input_scale_2"):
if name in source_tensors:
tensors[name] = source_tensors[name]
@@ -322,22 +322,6 @@ def check_and_fallback_input_schema(
input_group_size = input_schema.input_scale_group_size
input_scale_dtype = input_schema.input_scale_dtype
def is_compatible(candidate: "HummingInputSchema") -> bool:
check_weight_schema = weight_schema
if (
candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
and weight_schema.b_dtype == humming_dtypes.float4e2m1
and weight_group_size == 16
and weight_schema.bs_dtype
in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
):
# Humming rejects folded NVFP4 scales despite A16 software dequant
# supporting them. Relax only this check, preserving kernel metadata.
check_weight_schema = dataclasses.replace(
weight_schema, bs_dtype=humming_dtypes.float8e4m3
)
return candidate.is_compatible_with(check_weight_schema, param_dtype)
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
if dtype is None or dtype.num_bits == 16:
return False
@@ -349,7 +333,7 @@ def check_and_fallback_input_schema(
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
if is_compatible(input_schema):
if input_schema.is_compatible_with(weight_schema, param_dtype):
if not allow_fallback:
if is_deprecated(a_dtype):
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
@@ -406,7 +390,7 @@ def check_and_fallback_input_schema(
input_scale_dtype=scale_dtype,
input_quant_mode=quant_mode,
)
if is_compatible(candidate):
if candidate.is_compatible_with(weight_schema, param_dtype):
return candidate
raise ValueError(

Some files were not shown because too many files have changed in this diff Show More