Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -6,6 +6,7 @@ from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.determinism.batch_invariant_configs import (
|
||||
@@ -59,6 +60,7 @@ def matmul_kernel_persistent(
|
||||
stride_bn,
|
||||
stride_cm,
|
||||
stride_cn,
|
||||
stride_bias,
|
||||
BLOCK_SIZE_M: tl.constexpr, #
|
||||
BLOCK_SIZE_N: tl.constexpr, #
|
||||
BLOCK_SIZE_K: tl.constexpr, #
|
||||
@@ -128,7 +130,7 @@ def matmul_kernel_persistent(
|
||||
c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
|
||||
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
|
||||
if HAS_BIAS:
|
||||
bias_ptrs = bias_ptr + offs_cn
|
||||
bias_ptrs = bias_ptr + offs_cn * stride_bias
|
||||
bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
|
||||
accumulator += bias
|
||||
c = accumulator.to(c_ptr.dtype.element_ty)
|
||||
@@ -305,6 +307,8 @@ def matmul_persistent(
|
||||
),
|
||||
)
|
||||
|
||||
if not _BLOCK_SIZES_RESOLVED:
|
||||
_resolve_block_sizes()
|
||||
configs = {
|
||||
torch.bfloat16: {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
@@ -345,6 +349,7 @@ def matmul_persistent(
|
||||
b.stride(1), #
|
||||
c.stride(0),
|
||||
c.stride(1), #
|
||||
bias.stride(0) if bias is not None else 1,
|
||||
NUM_SMS=NUM_SMS, #
|
||||
A_LARGE=a.numel() > 2**31,
|
||||
B_LARGE=b.numel() > 2**31,
|
||||
@@ -489,16 +494,19 @@ def bmm_kernel(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _log_softmax_kernel(
|
||||
def _softmax_kernel(
|
||||
input_ptr,
|
||||
output_ptr,
|
||||
input_row_stride,
|
||||
input_col_stride,
|
||||
output_row_stride,
|
||||
n_cols,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LOG: tl.constexpr = False,
|
||||
):
|
||||
"""Compute log_softmax along the last dimension of a 2D tensor.
|
||||
Each block handles one row of the input tensor.
|
||||
"""Compute softmax, or log_softmax when ``LOG``, along the last dimension of a
|
||||
2D tensor. Each block handles one row of the input tensor, in a fixed block
|
||||
order, so the result never depends on the row count.
|
||||
"""
|
||||
# Get the row index for this block
|
||||
row_idx = tl.program_id(0).to(tl.int64)
|
||||
@@ -514,7 +522,9 @@ def _log_softmax_kernel(
|
||||
mask = col_idx < n_cols
|
||||
|
||||
# Load values
|
||||
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=-float("inf"))
|
||||
vals = tl.load(
|
||||
row_start_ptr + col_idx * input_col_stride, mask=mask, other=-float("inf")
|
||||
)
|
||||
|
||||
# Update maximum
|
||||
max_val = tl.max(tl.maximum(vals, max_val))
|
||||
@@ -526,25 +536,26 @@ def _log_softmax_kernel(
|
||||
mask = col_idx < n_cols
|
||||
|
||||
# Load values
|
||||
vals = tl.load(row_start_ptr + col_idx, mask=mask, other=0.0)
|
||||
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask, other=0.0)
|
||||
|
||||
# Compute exp(x - max_val) and accumulate
|
||||
exp_vals = tl.exp(vals - max_val)
|
||||
sum_exp += tl.sum(tl.where(mask, exp_vals, 0.0))
|
||||
|
||||
# Compute log(sum_exp)
|
||||
log_sum_exp = tl.log(sum_exp)
|
||||
log_sum_exp = tl.log(sum_exp) if LOG else 0.0
|
||||
|
||||
# Step 3: Compute final log_softmax values: x - max_val - log_sum_exp
|
||||
# Step 3: normalise
|
||||
for col_offset in range(0, n_cols, BLOCK_SIZE):
|
||||
col_idx = col_offset + tl.arange(0, BLOCK_SIZE)
|
||||
mask = col_idx < n_cols
|
||||
|
||||
# Load values
|
||||
vals = tl.load(row_start_ptr + col_idx, mask=mask)
|
||||
vals = tl.load(row_start_ptr + col_idx * input_col_stride, mask=mask)
|
||||
|
||||
# Compute log_softmax
|
||||
output = vals - max_val - log_sum_exp
|
||||
if LOG:
|
||||
output = vals - max_val - log_sum_exp
|
||||
else:
|
||||
output = tl.exp(vals - max_val) / sum_exp
|
||||
|
||||
# Store results
|
||||
tl.store(output_row_start_ptr + col_idx, output, mask=mask)
|
||||
@@ -570,30 +581,55 @@ def log_softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
# Flatten all dimensions except the last one
|
||||
original_shape = input.shape
|
||||
input_2d = input.reshape(-1, input.shape[-1])
|
||||
input_2d = input_2d.contiguous()
|
||||
|
||||
n_rows, n_cols = input_2d.shape
|
||||
|
||||
# Allocate output tensor
|
||||
output = torch.empty_like(input_2d)
|
||||
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
|
||||
|
||||
# Choose block size based on the number of columns
|
||||
BLOCK_SIZE = 1024
|
||||
|
||||
# Launch kernel with one block per row
|
||||
grid = (n_rows,)
|
||||
_log_softmax_kernel[grid](
|
||||
_softmax_kernel[grid](
|
||||
input_2d,
|
||||
output,
|
||||
input_2d.stride(0),
|
||||
input_2d.stride(1),
|
||||
output.stride(0),
|
||||
n_cols,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
LOG=True,
|
||||
)
|
||||
# Reshape output back to original shape
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
def softmax(input: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
"""Compute softmax along the last dimension using a Triton kernel."""
|
||||
if dim != -1 and dim != input.ndim - 1:
|
||||
raise ValueError(
|
||||
"This implementation only supports softmax along the last dimension"
|
||||
)
|
||||
|
||||
original_shape = input.shape
|
||||
input_2d = input.reshape(-1, input.shape[-1])
|
||||
n_rows, n_cols = input_2d.shape
|
||||
output = torch.empty((n_rows, n_cols), dtype=input_2d.dtype, device=input_2d.device)
|
||||
|
||||
_softmax_kernel[(n_rows,)](
|
||||
input_2d,
|
||||
output,
|
||||
input_2d.stride(0),
|
||||
input_2d.stride(1),
|
||||
output.stride(0),
|
||||
n_cols,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
return output.reshape(original_shape)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def mean_kernel(
|
||||
input_ptr,
|
||||
@@ -735,6 +771,9 @@ def mean_dim(
|
||||
|
||||
def mm_batch_invariant(a, b, bias=None):
|
||||
if a.device.type == "xpu":
|
||||
# The descriptor kernel reads the bias at unit stride.
|
||||
if bias is not None:
|
||||
bias = bias.contiguous()
|
||||
return matmul_descriptor_persistent(a, b, bias=bias)
|
||||
return matmul_persistent(a, b, bias=bias)
|
||||
|
||||
@@ -820,6 +859,8 @@ def bmm_batch_invariant(a, b, *, out=None):
|
||||
assert out.dtype == dtype and out.device == a.device, "out tensor mismatch"
|
||||
c = out
|
||||
|
||||
if not _BLOCK_SIZES_RESOLVED:
|
||||
_resolve_block_sizes()
|
||||
configs = {
|
||||
torch.bfloat16: {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
@@ -877,8 +918,50 @@ def bmm_batch_invariant(a, b, *, out=None):
|
||||
return c
|
||||
|
||||
|
||||
def addmm_batch_invariant(bias, a, b):
|
||||
return mm_batch_invariant(a, b, bias=bias)
|
||||
def _addmm_fused_bias(bias, beta, alpha, n):
|
||||
# The kernel adds bias into its fp32 accumulator indexed by output column,
|
||||
# so it can only absorb a row vector of width n, unscaled by alpha.
|
||||
if beta == 0 or alpha != 1:
|
||||
return None
|
||||
if bias.dim() != 1 or bias.shape[0] != n:
|
||||
return None
|
||||
# fp32 keeps the scaling exact; the kernel casts the bias to fp32 anyway.
|
||||
return bias if beta == 1 else bias.float() * beta
|
||||
|
||||
|
||||
def _addmm_impl(bias, a, b, beta, alpha, out):
|
||||
fused_bias = _addmm_fused_bias(bias, beta, alpha, b.shape[1])
|
||||
result = mm_batch_invariant(a, b, bias=fused_bias)
|
||||
if fused_bias is None:
|
||||
if alpha != 1:
|
||||
result = result * alpha
|
||||
if beta != 0:
|
||||
result = result + (bias if beta == 1 else beta * bias)
|
||||
if out is None:
|
||||
return result
|
||||
return out.copy_(result)
|
||||
|
||||
|
||||
def addmm_batch_invariant(bias, a, b, *, beta=1, alpha=1):
|
||||
return _addmm_impl(bias, a, b, beta, alpha, None)
|
||||
|
||||
|
||||
# Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``, which
|
||||
# dispatches the ``.out`` overload rather than the default one. Registering only
|
||||
# the default leaves the compiled path on the vendor GEMM, so a compiled
|
||||
# ``torch.addmm`` is batch variant while the eager one is not.
|
||||
|
||||
|
||||
def mm_out_batch_invariant(a, b, *, out):
|
||||
return matmul_batch_invariant(a, b, out=out)
|
||||
|
||||
|
||||
def addmm_out_batch_invariant(bias, a, b, *, beta=1, alpha=1, out):
|
||||
return _addmm_impl(bias, a, b, beta, alpha, out)
|
||||
|
||||
|
||||
def bmm_out_batch_invariant(a, b, *, out):
|
||||
return bmm_batch_invariant(a, b, out=out)
|
||||
|
||||
|
||||
def _log_softmax_batch_invariant(input, dim, _half_to_float):
|
||||
@@ -887,10 +970,16 @@ def _log_softmax_batch_invariant(input, dim, _half_to_float):
|
||||
return log_softmax(input, dim=dim)
|
||||
|
||||
|
||||
def softmax_batch_invariant(input, dim, dtype=None):
|
||||
# Compute softmax in a deterministic way
|
||||
# First subtract max for numerical stability (standard practice)
|
||||
def softmax_batch_invariant(input, dim, half_to_float=False):
|
||||
if half_to_float:
|
||||
input = input.float()
|
||||
if dim == -1 or dim == input.ndim - 1:
|
||||
return softmax(input, dim=-1)
|
||||
|
||||
# Off the last dimension this falls back to torch ops, whose reduction split
|
||||
# depends on the tensor shape.
|
||||
input_max = torch.amax(input, dim=dim, keepdim=True)
|
||||
# First subtract max for numerical stability (standard practice)
|
||||
input = input - input_max
|
||||
exp_x = torch.exp(input)
|
||||
sum_exp_x = torch.sum(exp_x, dim=dim, keepdim=True)
|
||||
@@ -1035,11 +1124,150 @@ def rms_norm_batch_invariant(
|
||||
|
||||
|
||||
def linear_batch_invariant(input, weight, bias=None):
|
||||
output = matmul_batch_invariant(input, weight.t())
|
||||
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output
|
||||
bias = bias.expand(weight.shape[0])
|
||||
output = mm_batch_invariant(
|
||||
input.reshape(-1, input.shape[-1]), weight.t(), bias=bias
|
||||
)
|
||||
return output.view(*input.shape[:-1], output.shape[-1])
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fixed_order_sum_kernel(
|
||||
src_ptr, # (WORLD_SIZE, numel) gathered contributions, rank-major
|
||||
out_ptr, # (numel,)
|
||||
numel,
|
||||
stride_rank,
|
||||
WORLD_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
LARGE: tl.constexpr,
|
||||
):
|
||||
"""Sum ``WORLD_SIZE`` contributions in ascending rank order, fp32 accumulator.
|
||||
|
||||
An output element sums exactly ``WORLD_SIZE`` values in a compile-time
|
||||
constant order and rounds once, so the result depends only on the rank
|
||||
ordering and never on how many elements are being reduced.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
if LARGE:
|
||||
pid = pid.to(tl.int64)
|
||||
stride_rank = stride_rank.to(tl.int64)
|
||||
offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offs < numel
|
||||
|
||||
acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
|
||||
for r in tl.static_range(WORLD_SIZE):
|
||||
vals = tl.load(src_ptr + r * stride_rank + offs, mask=mask, other=0.0)
|
||||
acc += vals.to(tl.float32)
|
||||
|
||||
tl.store(out_ptr + offs, acc.to(out_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
def all_reduce_batch_invariant(
|
||||
input_: torch.Tensor, group: "torch.distributed.ProcessGroup | None" = None
|
||||
) -> torch.Tensor:
|
||||
"""Sum all-reduce whose result does not depend on the message size.
|
||||
|
||||
Library all-reduces (NCCL/RCCL) pick their algorithm, channel count and chunk
|
||||
boundaries from the message size, so the order in which a given element's
|
||||
contributions are summed changes with the number of tokens in the batch.
|
||||
|
||||
Instead, all-gather the contributions -- pure data movement, so bitwise
|
||||
reproducible at any size -- and reduce them with ``_fixed_order_sum_kernel``.
|
||||
"""
|
||||
world_size = dist.get_world_size(group)
|
||||
if world_size == 1:
|
||||
return input_
|
||||
|
||||
x = input_.contiguous()
|
||||
numel = x.numel()
|
||||
gathered = torch.empty((world_size, numel), dtype=x.dtype, device=x.device)
|
||||
dist.all_gather_into_tensor(gathered, x.view(-1), group=group)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (triton.cdiv(numel, BLOCK_SIZE),)
|
||||
_fixed_order_sum_kernel[grid](
|
||||
gathered,
|
||||
out.view(-1),
|
||||
numel,
|
||||
numel,
|
||||
WORLD_SIZE=world_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
LARGE=gathered.numel() > 2**31,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def reduce_scatter_batch_invariant(
|
||||
input_: torch.Tensor,
|
||||
group: "torch.distributed.ProcessGroup | None" = None,
|
||||
sizes: list[int] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Sum reduce-scatter over dim 0 whose result does not depend on the size.
|
||||
|
||||
``ncclReduceScatter`` picks its chunking from the message size just as
|
||||
``ncclAllReduce`` does. Route around it the same way
|
||||
``all_reduce_batch_invariant`` does: an all-to-all sends rank ``d`` exactly
|
||||
the rows it owns from every rank -- pure data movement, so bitwise
|
||||
reproducible however the library chunks it -- and lands them rank-major,
|
||||
which is the layout ``_fixed_order_sum_kernel`` already consumes.
|
||||
|
||||
``sizes`` (the reduce-scatterv case) changes only *which* rows a rank
|
||||
receives, never how the received contributions are summed, so variable
|
||||
shard sizes are invariant for the same reason.
|
||||
|
||||
Args:
|
||||
input_: Contiguous full-size input; dim 0 is the scattered axis.
|
||||
group: Process group, defaults to the world group.
|
||||
sizes: Per-rank row counts. Uniform split when ``None``.
|
||||
|
||||
Returns:
|
||||
This rank's shard of the sum.
|
||||
|
||||
"""
|
||||
world_size = dist.get_world_size(group)
|
||||
if world_size == 1:
|
||||
return input_
|
||||
|
||||
x = input_.contiguous()
|
||||
if sizes is None:
|
||||
assert x.shape[0] % world_size == 0
|
||||
sizes = [x.shape[0] // world_size] * world_size
|
||||
else:
|
||||
assert len(sizes) == world_size
|
||||
assert x.shape[0] == sum(sizes)
|
||||
|
||||
rows = sizes[dist.get_rank(group)]
|
||||
out = torch.empty((rows, *x.shape[1:]), dtype=x.dtype, device=x.device)
|
||||
recv = torch.empty(
|
||||
(world_size * rows, *x.shape[1:]), dtype=x.dtype, device=x.device
|
||||
)
|
||||
# Every rank takes part even when its own shard is empty: it still has rows
|
||||
# to send to the others.
|
||||
dist.all_to_all_single(
|
||||
recv,
|
||||
x,
|
||||
output_split_sizes=[rows] * world_size,
|
||||
input_split_sizes=sizes,
|
||||
group=group,
|
||||
)
|
||||
|
||||
numel = out.numel()
|
||||
if numel == 0:
|
||||
return out
|
||||
BLOCK_SIZE = 1024
|
||||
grid = (triton.cdiv(numel, BLOCK_SIZE),)
|
||||
_fixed_order_sum_kernel[grid](
|
||||
recv.view(-1),
|
||||
out.view(-1),
|
||||
numel,
|
||||
numel,
|
||||
WORLD_SIZE=world_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
LARGE=recv.numel() > 2**31,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
_batch_invariant_MODE = False
|
||||
@@ -1047,9 +1275,33 @@ _batch_invariant_LIB = None
|
||||
_fp16_block_size_n = 256
|
||||
_fp32_block_size_n = 128
|
||||
_fp32_num_stages = 3
|
||||
_BLOCK_SIZES_RESOLVED = False
|
||||
_NUM_SMS: int = 0
|
||||
|
||||
|
||||
def _resolve_block_sizes():
|
||||
"""Size the tiles to the shared memory, to avoid triton OutOfResources.
|
||||
|
||||
Runs when the mode is enabled, or on first use by a caller that never
|
||||
enables it, such as a kernel unit test.
|
||||
"""
|
||||
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
|
||||
global _BLOCK_SIZES_RESOLVED
|
||||
_BLOCK_SIZES_RESOLVED = True
|
||||
if not current_platform.is_cuda_alike():
|
||||
return
|
||||
if get_max_shared_memory_bytes() > 106496:
|
||||
_fp16_block_size_n = 256
|
||||
_fp32_block_size_n = 128
|
||||
_fp32_num_stages = 3
|
||||
else:
|
||||
_fp16_block_size_n = 128
|
||||
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
|
||||
# Only N=1 OORs; keep 128/stages=3 for every other N.
|
||||
_fp32_block_size_n = 32
|
||||
_fp32_num_stages = 2
|
||||
|
||||
|
||||
def enable_batch_invariant_mode():
|
||||
global _batch_invariant_MODE, _batch_invariant_LIB
|
||||
global _fp16_block_size_n, _fp32_block_size_n, _fp32_num_stages
|
||||
@@ -1062,12 +1314,17 @@ def enable_batch_invariant_mode():
|
||||
_batch_invariant_LIB = torch.library.Library("aten", "IMPL")
|
||||
|
||||
key = current_platform.dispatch_key
|
||||
if current_platform.is_cuda():
|
||||
if current_platform.is_device_capability_family(80):
|
||||
# SM80 (Ampere) cannot rely on cuBLASLt-only determinism; install the
|
||||
# triton persistent matmul overrides for mm/addmm/matmul/linear.
|
||||
if current_platform.is_cuda_alike():
|
||||
if current_platform.is_rocm() or current_platform.is_device_capability_family(
|
||||
80
|
||||
):
|
||||
# Neither cuBLASLt on SM80 (Ampere) nor hipBLASLt can be made batch
|
||||
# invariant; install the triton persistent matmul overrides for
|
||||
# mm/addmm/matmul/linear.
|
||||
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
|
||||
else:
|
||||
@@ -1077,18 +1334,7 @@ def enable_batch_invariant_mode():
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
|
||||
os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1"
|
||||
|
||||
# Query the shared memory size and set block size
|
||||
# accordingly to avoid triton OutOfResources.
|
||||
if get_max_shared_memory_bytes() > 106496:
|
||||
_fp16_block_size_n = 256
|
||||
_fp32_block_size_n = 128
|
||||
_fp32_num_stages = 3
|
||||
else:
|
||||
_fp16_block_size_n = 128
|
||||
# SM89 N=1 fp32 128/stages=3 compiles to 131072B (>101376).
|
||||
# Only N=1 OORs; keep 128/stages=3 for every other N.
|
||||
_fp32_block_size_n = 32
|
||||
_fp32_num_stages = 2
|
||||
_resolve_block_sizes()
|
||||
elif current_platform.is_xpu():
|
||||
# Tensor descriptors need a global-memory allocator; must be set outside
|
||||
# torch.compile regions (triton.set_allocator modifies global state).
|
||||
@@ -1100,6 +1346,8 @@ def enable_batch_invariant_mode():
|
||||
_NUM_SMS = num_compute_units(0)
|
||||
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::mm.out", mm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::addmm.out", addmm_out_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
|
||||
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
|
||||
|
||||
@@ -1123,7 +1371,9 @@ def enable_batch_invariant_mode():
|
||||
_batch_invariant_LIB.impl(
|
||||
"aten::bmm", bmm_batch_invariant, key, allow_override=True
|
||||
)
|
||||
torch.bmm = bmm_batch_invariant
|
||||
_batch_invariant_LIB.impl(
|
||||
"aten::bmm.out", bmm_out_batch_invariant, key, allow_override=True
|
||||
)
|
||||
|
||||
reduced_precision_val = (
|
||||
(False, False) if is_torch_equal_or_newer("2.10.0") else False
|
||||
@@ -1145,28 +1395,32 @@ def override_envs_for_invariance():
|
||||
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
|
||||
# unlike the pins below, it also applies to communicators shared with other
|
||||
# processes.
|
||||
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
|
||||
pin_nccl_env(
|
||||
{
|
||||
"NCCL_COLLNET_ENABLE": "0",
|
||||
"NCCL_NVLS_ENABLE": "0",
|
||||
"NCCL_P2P_NET_DISABLE": "1",
|
||||
"NCCL_MIN_NCHANNELS": "1",
|
||||
"NCCL_MAX_NCHANNELS": "1",
|
||||
"NCCL_PROTO": "Simple",
|
||||
# NCCL >= 2.31 zero-fills the algorithm table of every collective
|
||||
# when NCCL_ALGO is set and re-enables only the named ones; together
|
||||
# with the NCCL_PROTO above, collectives not named here end up with
|
||||
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
|
||||
# Tree for all collectives, then pin AllReduce to Tree.
|
||||
"NCCL_ALGO": "ring,tree;allreduce:tree",
|
||||
"NCCL_NTHREADS": "1",
|
||||
"NCCL_SOCKET_NTHREADS": "1",
|
||||
}
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
# The skinny GEMMs pick their split of K from the batch size.
|
||||
os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
|
||||
else:
|
||||
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
|
||||
# unlike the pins below, it also applies to communicators shared with other
|
||||
# processes.
|
||||
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
|
||||
pin_nccl_env(
|
||||
{
|
||||
"NCCL_COLLNET_ENABLE": "0",
|
||||
"NCCL_NVLS_ENABLE": "0",
|
||||
"NCCL_P2P_NET_DISABLE": "1",
|
||||
"NCCL_MIN_NCHANNELS": "1",
|
||||
"NCCL_MAX_NCHANNELS": "1",
|
||||
"NCCL_PROTO": "Simple",
|
||||
# NCCL >= 2.31 zero-fills the algorithm table of every collective
|
||||
# when NCCL_ALGO is set and re-enables only the named ones; together
|
||||
# with the NCCL_PROTO above, collectives not named here end up with
|
||||
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
|
||||
# Tree for all collectives, then pin AllReduce to Tree.
|
||||
"NCCL_ALGO": "ring,tree;allreduce:tree",
|
||||
"NCCL_NTHREADS": "1",
|
||||
"NCCL_SOCKET_NTHREADS": "1",
|
||||
}
|
||||
)
|
||||
|
||||
# torch.compile settings
|
||||
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
|
||||
|
||||
@@ -3,7 +3,14 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select
|
||||
from vllm.model_executor.layers.indexer_topk import (
|
||||
deep_select_topk,
|
||||
get_deep_select_stride_requirement,
|
||||
is_deep_select_supported,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -17,38 +24,47 @@ def _block_scores_kernel(
|
||||
starts,
|
||||
ends,
|
||||
scores,
|
||||
valid_blocks,
|
||||
stride_row,
|
||||
stride_col,
|
||||
stride_start,
|
||||
stride_end,
|
||||
stride_scores,
|
||||
width,
|
||||
nblocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_STARTS: tl.constexpr,
|
||||
ROW_REPEAT: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
FULL_WIDTH: tl.constexpr,
|
||||
):
|
||||
# Writes each row's valid block count; FULL_WIDTH also scores the -inf tail.
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
blocks = tl.program_id(1) * TILE + tl.arange(0, TILE)
|
||||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||||
end = tl.load(ends + row // ROW_REPEAT * stride_end)
|
||||
col_end = tl.minimum(end, width)
|
||||
num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE)
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(valid_blocks + row, num_valid)
|
||||
num_blocks = nblocks if FULL_WIDTH else num_valid
|
||||
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
|
||||
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
|
||||
values = tl.load(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
(blocks[:, None] < nblocks)
|
||||
& (offsets[None, :] < BLOCK_SIZE)
|
||||
& (cols < end)
|
||||
& (cols < width),
|
||||
other=-float("inf"),
|
||||
)
|
||||
reduced = tl.reduce(values, 1, _max_with_nan)
|
||||
reduced = tl.where(
|
||||
(end > start) & (blocks == (end - start - 1) // BLOCK_SIZE),
|
||||
float("inf"),
|
||||
reduced,
|
||||
)
|
||||
tl.store(scores + row * nblocks + blocks, reduced, blocks < nblocks)
|
||||
num_tiles = tl.cdiv(num_blocks, TILE)
|
||||
for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)):
|
||||
blocks = tile_id * TILE + tl.arange(0, TILE)
|
||||
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
|
||||
values = tl.load(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
(offsets[None, :] < BLOCK_SIZE) & (cols < col_end),
|
||||
other=-float("inf"),
|
||||
).to(tl.float32)
|
||||
reduced = tl.reduce(values, 1, _max_with_nan)
|
||||
# Pin the newest block. DeepSelect leaves NaN rows partially written,
|
||||
# so also pin NaN blocks (torch.topk ranks NaN first too).
|
||||
pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE)
|
||||
if not FULL_WIDTH:
|
||||
pinned |= reduced != reduced
|
||||
reduced = tl.where(pinned, float("inf"), reduced)
|
||||
tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["k"])
|
||||
@@ -160,22 +176,42 @@ def select_candidate_blocks(
|
||||
out.fill_(-1)
|
||||
return
|
||||
nblocks = triton.cdiv(width, block_size)
|
||||
scores = logits.new_empty((rows, nblocks))
|
||||
_block_scores_kernel[(rows, triton.cdiv(nblocks, 128))](
|
||||
# Logits are max_model_len wide (CUDA-graph static) but rows are shorter:
|
||||
# DeepSelect stops at each row's valid block count. Order does not matter.
|
||||
use_deep_select = has_deep_select() and nblocks >= topk_blocks
|
||||
if use_deep_select:
|
||||
in_align, out_align = get_deep_select_stride_requirement()
|
||||
padded = round_up(nblocks, in_align // logits.element_size())
|
||||
scores = logits.new_empty((rows, padded))[:, :nblocks]
|
||||
use_deep_select = (
|
||||
out.stride(1) == 1
|
||||
and out.stride(0) * out.element_size() % out_align == 0
|
||||
and is_deep_select_supported(scores, topk_blocks)
|
||||
)
|
||||
else:
|
||||
scores = logits.new_empty((rows, nblocks))
|
||||
valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device)
|
||||
_block_scores_kernel[(rows, 32)](
|
||||
logits,
|
||||
row_ks,
|
||||
row_ke,
|
||||
scores,
|
||||
valid_blocks,
|
||||
*logits.stride(),
|
||||
row_ks.stride(0) if row_ks is not None else 0,
|
||||
row_ke.stride(0),
|
||||
scores.stride(0),
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
128,
|
||||
TILE=128,
|
||||
FULL_WIDTH=not use_deep_select,
|
||||
)
|
||||
if use_deep_select:
|
||||
deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out)
|
||||
return
|
||||
# Keep the existing top-k tie behavior.
|
||||
top = scores.topk(min(topk_blocks, nblocks), dim=-1)
|
||||
_store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))](
|
||||
|
||||
@@ -123,6 +123,9 @@ from vllm.model_executor.kernels.linear.mxfp8.humming import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
MarlinMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
|
||||
TorchMxFp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
@@ -135,6 +138,7 @@ from vllm.model_executor.kernels.linear.nvfp4 import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.b12x import (
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.cutlass import (
|
||||
CutlassNvFp4LinearKernel,
|
||||
@@ -218,6 +222,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -253,6 +258,20 @@ def _get_linear_backend(*, quantization: str) -> str:
|
||||
return config.kernel_config.linear_backend
|
||||
|
||||
|
||||
def _batch_invariant_fp8_kernel(kernel, config, quantization: str):
|
||||
can_implement, reason = is_supported_and_can_implement_kernel(kernel, config, None)
|
||||
if not can_implement:
|
||||
raise ValueError(f"Batch invariance on ROCm: {reason}")
|
||||
linear_backend = _get_linear_backend(quantization=quantization)
|
||||
if linear_backend not in ("auto", "triton"):
|
||||
logger.warning_once(
|
||||
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
|
||||
"using the Triton backend for deterministic execution.",
|
||||
linear_backend,
|
||||
)
|
||||
return kernel
|
||||
|
||||
|
||||
# Kernel classes covered by each --linear-backend value.
|
||||
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"b12x": {
|
||||
@@ -260,6 +279,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
B12xMxFp4LinearKernel,
|
||||
B12xMxfp8LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
},
|
||||
"cutlass": {
|
||||
@@ -321,6 +341,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
TorchMxFp8LinearKernel,
|
||||
TorchNvFp4LinearKernel,
|
||||
},
|
||||
"aiter": {
|
||||
@@ -541,6 +562,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
EmulationMxfp8LinearKernel,
|
||||
HummingMxfp8LinearKernel,
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
TorchMxFp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
|
||||
@@ -550,6 +572,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp8LinearKernel,
|
||||
TorchMxFp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
}
|
||||
@@ -567,6 +590,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
FlashInferCudnnNvFp4LinearKernel,
|
||||
FbgemmNvFp4LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
TorchNvFp4LinearKernel,
|
||||
EmulationNvFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
@@ -723,6 +747,13 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
if activation_quant_key.scale.group_shape.is_per_group():
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
|
||||
force_kernel = _batch_invariant_fp8_kernel(
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
scaled_mm_linear_kernel_config,
|
||||
"fp8_block_w8a8",
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config=scaled_mm_linear_kernel_config,
|
||||
possible_kernels=_POSSIBLE_FP8_BLOCK_KERNELS, # type: ignore[misc]
|
||||
@@ -755,6 +786,13 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
else:
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
|
||||
force_kernel = _batch_invariant_fp8_kernel(
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
scaled_mm_linear_kernel_config,
|
||||
"fp8_w8a8",
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config=scaled_mm_linear_kernel_config,
|
||||
possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type]
|
||||
@@ -780,6 +818,19 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
|
||||
def fp8_scaled_mm_is_batch_invariant(
|
||||
kernel: FP8ScaledMMLinearKernel | Fp8BlockScaledMMLinearKernel,
|
||||
) -> bool:
|
||||
"""Whether a row's output from the kernel is independent of the other rows."""
|
||||
if not isinstance(
|
||||
kernel, (CutlassFP8ScaledMMLinearKernel, TritonFP8ScaledMMLinearKernel)
|
||||
):
|
||||
return False
|
||||
# A dynamic per-tensor activation scale is an amax over every row.
|
||||
scale = kernel.config.activation_quant_key.scale
|
||||
return scale.static or not scale.group_shape.is_per_tensor()
|
||||
|
||||
|
||||
def init_int8_linear_kernel(
|
||||
is_channelwise: bool,
|
||||
is_static_input_scheme: bool,
|
||||
@@ -1085,6 +1136,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
current platform."""
|
||||
config = NvFp4LinearLayerConfig()
|
||||
a16_kernels = (
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
MarlinNvFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
@@ -1250,6 +1302,7 @@ def register_linear_kernel(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"fp8_scaled_mm_is_batch_invariant",
|
||||
"init_fp8_linear_kernel",
|
||||
"init_int8_linear_kernel",
|
||||
"init_nvfp4_linear_kernel",
|
||||
@@ -1298,6 +1351,7 @@ __all__ = [
|
||||
"B12xMxfp8LinearKernel",
|
||||
"B12xMxFp4LinearKernel",
|
||||
"B12xNvFp4LinearKernel",
|
||||
"B12xNvFp4W4A16LinearKernel",
|
||||
"init_mxfp4_linear_kernel",
|
||||
"MxFp4LinearKernel",
|
||||
"MxFp4LinearLayerConfig",
|
||||
@@ -1315,6 +1369,7 @@ __all__ = [
|
||||
"FlashInferTrtllmMxfp8LinearKernel",
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"XPUMxFp8LinearKernel",
|
||||
"TorchMxFp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
"CutlassNvFp4LinearKernel",
|
||||
"EmulationNvFp4LinearKernel",
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import const_expr
|
||||
|
||||
|
||||
class LLFp32WDotprod:
|
||||
"""FP32-weight router GEMM kernel based on CTA-local dot products.
|
||||
|
||||
This kernel computes C[M, N] = A[M, K] @ B[N, K]^T for bf16/fp16/fp32
|
||||
activations, fp32 weights, and fp32 output. It launches one CTA per
|
||||
small output-column tile, distributes K across CTA threads with vectorized
|
||||
loads, accumulates one fp32 dot product per token/expert, and reduces
|
||||
through warp shuffles plus shared memory.
|
||||
|
||||
:param k: Compile-time K dimension specialized into the generated kernel.
|
||||
:type k: int
|
||||
:param bs: Threads per CTA and K-stripe width used by the reduction.
|
||||
:type bs: int
|
||||
:param main_vec_width: Elements loaded per thread in the main loop.
|
||||
:type main_vec_width: int
|
||||
:param tail_vec_width: Elements loaded per thread in the vector tail.
|
||||
:type tail_vec_width: int
|
||||
:param use_pdl: Whether to launch with Programmatic Dependent Launch.
|
||||
:type use_pdl: bool
|
||||
|
||||
:note: Supported A/B data types:
|
||||
- BFloat16/Float32
|
||||
- Float16/Float32
|
||||
- Float32/Float32
|
||||
:note: Supported accumulator data types:
|
||||
- Float32
|
||||
:note: Supported C data types:
|
||||
- Float32
|
||||
:note: Constraints:
|
||||
- Inputs must be contiguous row-major 2D tensors.
|
||||
|
||||
:note: K is handled as vectorized main/tail loops plus scalar remainder.
|
||||
|
||||
:compile-key: ``(M, K, bs, epb, act_dtype)`` selects the token count,
|
||||
hidden size, CTA thread/K-stripe width, experts per CTA, and activation
|
||||
input type.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
m: int,
|
||||
k: int,
|
||||
bs: int = 128,
|
||||
main_vec_width: int = 8,
|
||||
tail_vec_width: int = 4,
|
||||
token_groups: int = 1,
|
||||
epb: int = 1,
|
||||
use_pdl: bool = False,
|
||||
):
|
||||
"""Initialize the dot-product kernel configuration.
|
||||
|
||||
This configuration fixes the CTA thread count, reduction warp count,
|
||||
vector widths, and K-loop decomposition used by the generated kernel.
|
||||
|
||||
:param m: Token count M selected by the compile key.
|
||||
:type m: int
|
||||
:param k: Hidden size K used to specialize the K-loop decomposition.
|
||||
:type k: int
|
||||
:param bs: Threads per CTA and K-stripe width for the reduction.
|
||||
:type bs: int
|
||||
:param main_vec_width: Elements loaded per thread in the main loop.
|
||||
:type main_vec_width: int
|
||||
:param tail_vec_width: Elements loaded per thread in the vector tail.
|
||||
:type tail_vec_width: int
|
||||
:param use_pdl: Whether to launch with Programmatic Dependent Launch.
|
||||
:type use_pdl: bool
|
||||
"""
|
||||
self.m = m
|
||||
self.m_per_group = m // token_groups
|
||||
self.bs = bs
|
||||
self.main_vec_width = main_vec_width
|
||||
self.tail_vec_width = tail_vec_width
|
||||
self.token_groups = token_groups
|
||||
self.epb = epb
|
||||
self.use_pdl = use_pdl
|
||||
self.num_warps = bs // cute.arch.WARP_SIZE
|
||||
self._init_k_tiles(k)
|
||||
|
||||
def _vectorized_elems(self, k_extent: int, vec_width: int) -> int:
|
||||
vector_tile = vec_width * self.bs
|
||||
return (k_extent // vector_tile) * vector_tile
|
||||
|
||||
def _init_k_tiles(self, k: int) -> None:
|
||||
"""Split K into vector loops, scalar rounds, and ragged tail."""
|
||||
self.k_main_elems = self._vectorized_elems(k, self.main_vec_width)
|
||||
self.k_after_main = k - self.k_main_elems
|
||||
self.k_tail_elems = self._vectorized_elems(
|
||||
self.k_after_main, self.tail_vec_width
|
||||
)
|
||||
self.k_done_all = self.k_main_elems + self.k_tail_elems
|
||||
self.scalar_rem = k - self.k_done_all
|
||||
self.ks_full = self.scalar_rem // self.bs
|
||||
self.ks_part = self.scalar_rem % self.bs
|
||||
self.k_scalar_full = self.ks_full * self.bs
|
||||
self.k_part_offset = self.k_done_all + self.k_scalar_full
|
||||
self.main_tiles = self.k_main_elems // (self.main_vec_width * self.bs)
|
||||
self.tail_tiles = self.k_tail_elems // (self.tail_vec_width * self.bs)
|
||||
|
||||
@cute.jit
|
||||
def _vector_dotprod(
|
||||
self,
|
||||
acc: cute.Tensor,
|
||||
tA: cute.Tensor,
|
||||
tB: cute.Tensor,
|
||||
M: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
num_tiles: cutlass.Constexpr,
|
||||
):
|
||||
for tile in cutlass.range_constexpr(num_tiles):
|
||||
bt0 = tB[0, None, tile]
|
||||
br0 = cute.make_rmem_tensor_like(bt0)
|
||||
cute.autovec_copy(bt0, br0)
|
||||
br0_f32 = br0.load().to(cutlass.Float32)
|
||||
|
||||
if const_expr(epb == 2):
|
||||
bt1 = tB[1, None, tile]
|
||||
br1 = cute.make_rmem_tensor_like(bt1)
|
||||
cute.autovec_copy(bt1, br1)
|
||||
br1_f32 = br1.load().to(cutlass.Float32)
|
||||
|
||||
for m in cutlass.range_constexpr(M):
|
||||
at = tA[m, None, tile]
|
||||
ar = cute.make_rmem_tensor_like(at)
|
||||
cute.autovec_copy(at, ar)
|
||||
vec_width: cutlass.Constexpr = cute.size(ar)
|
||||
for v in cutlass.range_constexpr(vec_width):
|
||||
av = ar[v].to(cutlass.Float32)
|
||||
acc[m, 0] = acc[m, 0] + av * br0_f32[v]
|
||||
if const_expr(epb == 2):
|
||||
acc[m, 1] = acc[m, 1] + av * br1_f32[v]
|
||||
|
||||
def _make_thread_vector_slice(
|
||||
self,
|
||||
gA_vec: cute.Tensor,
|
||||
gB_vec: cute.Tensor,
|
||||
tidx: cutlass.Int32,
|
||||
bs: cutlass.Constexpr,
|
||||
):
|
||||
# A: (M, K_TILE, K_LANE, K_VEC); B: (epb, K_TILE, K_LANE, K_VEC).
|
||||
tA = cute.logical_divide(gA_vec, (None, (None, bs)))
|
||||
tB = cute.logical_divide(gB_vec, (None, (None, bs)))
|
||||
return tA[None, (None, (tidx, None))], tB[None, (None, (tidx, None))]
|
||||
|
||||
def _make_m_slice(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
m_offset: cutlass.Int32,
|
||||
m_extent: cutlass.Constexpr,
|
||||
):
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((m_offset, 0), gX),
|
||||
(m_extent, cute.size(gX, mode=[1])),
|
||||
(0, 0),
|
||||
)
|
||||
|
||||
def _make_n_tile(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
n_tile_idx: cutlass.Int32,
|
||||
n_extent: cutlass.Constexpr,
|
||||
):
|
||||
return cute.local_tile(
|
||||
gX,
|
||||
(n_extent, cute.size(gX, mode=[1])),
|
||||
(n_tile_idx, 0),
|
||||
)
|
||||
|
||||
def _make_sm_m_slice(
|
||||
self,
|
||||
sm: cute.Tensor,
|
||||
m_offset: cutlass.Int32,
|
||||
m_extent: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
):
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((m_offset, 0, 0), sm),
|
||||
(m_extent, epb, num_warps),
|
||||
(0, 0, 0),
|
||||
)
|
||||
|
||||
def _make_k_slice(
|
||||
self,
|
||||
gX: cute.Tensor,
|
||||
k_offset: cutlass.Constexpr,
|
||||
k_extent: cutlass.Constexpr,
|
||||
):
|
||||
k_layout_extent: cutlass.Constexpr = (
|
||||
1 if const_expr(k_extent == 0) else k_extent
|
||||
)
|
||||
|
||||
if const_expr(k_offset == 0):
|
||||
return cute.local_tile(
|
||||
gX, (cute.size(gX, mode=[0]), k_layout_extent), (0, 0)
|
||||
)
|
||||
return cute.local_tile(
|
||||
cute.domain_offset((0, k_offset), gX),
|
||||
(cute.size(gX, mode=[0]), k_layout_extent),
|
||||
(0, 0),
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _compute_token_group(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
sm: cute.Tensor,
|
||||
n_tile_idx: cutlass.Int32,
|
||||
tidx: cutlass.Int32,
|
||||
local_wid: cutlass.Int32,
|
||||
m_per_group: cutlass.Constexpr,
|
||||
m_offset: cutlass.Int32,
|
||||
main_vec_width: cutlass.Constexpr,
|
||||
tail_vec_width: cutlass.Constexpr,
|
||||
bs: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
k_main_elems: cutlass.Constexpr,
|
||||
k_tail_elems: cutlass.Constexpr,
|
||||
k_done_all: cutlass.Constexpr,
|
||||
ks_full: cutlass.Constexpr,
|
||||
ks_part: cutlass.Constexpr,
|
||||
k_scalar_full: cutlass.Constexpr,
|
||||
k_part_offset: cutlass.Constexpr,
|
||||
main_tiles: cutlass.Constexpr,
|
||||
tail_tiles: cutlass.Constexpr,
|
||||
):
|
||||
# Each CTA reuses A for epb adjacent experts.
|
||||
acc = cute.make_rmem_tensor((m_per_group, epb), cutlass.Float32)
|
||||
acc.fill(0.0)
|
||||
|
||||
gA_group = self._make_m_slice(gA, m_offset, m_per_group)
|
||||
gB_epb = self._make_n_tile(gB, n_tile_idx, epb)
|
||||
sm_group = self._make_sm_m_slice(sm, m_offset, m_per_group, epb, num_warps)
|
||||
|
||||
if const_expr(k_main_elems > 0):
|
||||
gA_main = self._make_k_slice(gA_group, 0, k_main_elems)
|
||||
gB_main = self._make_k_slice(gB_epb, 0, k_main_elems)
|
||||
gA_vec = cute.logical_divide(gA_main, (None, main_vec_width))
|
||||
gB_vec = cute.logical_divide(gB_main, (None, main_vec_width))
|
||||
tA, tB = self._make_thread_vector_slice(gA_vec, gB_vec, tidx, bs)
|
||||
self._vector_dotprod(acc, tA, tB, m_per_group, epb, main_tiles)
|
||||
|
||||
if const_expr(k_tail_elems > 0):
|
||||
gA_tail = self._make_k_slice(gA_group, k_main_elems, k_tail_elems)
|
||||
gB_tail = self._make_k_slice(gB_epb, k_main_elems, k_tail_elems)
|
||||
gA_tail_vec = cute.logical_divide(gA_tail, (None, tail_vec_width))
|
||||
gB_tail_vec = cute.logical_divide(gB_tail, (None, tail_vec_width))
|
||||
tA_t, tB_t = self._make_thread_vector_slice(
|
||||
gA_tail_vec, gB_tail_vec, tidx, bs
|
||||
)
|
||||
self._vector_dotprod(acc, tA_t, tB_t, m_per_group, epb, tail_tiles)
|
||||
|
||||
if const_expr(ks_full > 0):
|
||||
gA_scalar = self._make_k_slice(gA_group, k_done_all, k_scalar_full)
|
||||
gB_scalar = self._make_k_slice(gB_epb, k_done_all, k_scalar_full)
|
||||
gA_scalar_vec = cute.logical_divide(gA_scalar, (None, 1))
|
||||
gB_scalar_vec = cute.logical_divide(gB_scalar, (None, 1))
|
||||
tA_s, tB_s = self._make_thread_vector_slice(
|
||||
gA_scalar_vec, gB_scalar_vec, tidx, bs
|
||||
)
|
||||
self._vector_dotprod(acc, tA_s, tB_s, m_per_group, epb, ks_full)
|
||||
|
||||
if const_expr(ks_part > 0):
|
||||
gA_part = self._make_k_slice(gA_group, k_part_offset, ks_part)
|
||||
gB_part = self._make_k_slice(gB_epb, k_part_offset, ks_part)
|
||||
if tidx < ks_part:
|
||||
for local_m in cutlass.range_constexpr(m_per_group):
|
||||
av = gA_part[local_m, tidx].to(cutlass.Float32)
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
bv = gB_part[e, tidx].to(cutlass.Float32)
|
||||
acc[local_m, e] = acc[local_m, e] + av * bv
|
||||
|
||||
for local_m in cutlass.range_constexpr(m_per_group):
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
acc[local_m, e] = cute.arch.warp_reduction_sum(acc[local_m, e])
|
||||
|
||||
with cute.arch.elect_one():
|
||||
for local_m in cutlass.range_constexpr(m_per_group):
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
sm_group[local_m, e, local_wid] = acc[local_m, e]
|
||||
|
||||
@cute.jit
|
||||
def _store_outputs(
|
||||
self,
|
||||
gC: cute.Tensor,
|
||||
sm: cute.Tensor,
|
||||
n_tile_idx: cutlass.Int32,
|
||||
N_dim: cutlass.Int32,
|
||||
tidx: cutlass.Int32,
|
||||
group_idx: cutlass.Int32,
|
||||
M: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
):
|
||||
n_tile_layout = cute.make_layout(
|
||||
(cute.ceil_div(N_dim, epb), epb), stride=(epb, 1)
|
||||
)
|
||||
n_coords = cute.make_tensor(0, n_tile_layout)
|
||||
|
||||
if group_idx == 0 and tidx < M:
|
||||
for e in cutlass.range_constexpr(epb):
|
||||
n = n_coords[n_tile_idx, e]
|
||||
if n < N_dim:
|
||||
partials = sm[tidx, e, None].load()
|
||||
gC[tidx, n] = partials.reduce(
|
||||
cute.ReductionOp.ADD,
|
||||
init_val=cutlass.Float32(0.0),
|
||||
reduction_profile=0,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
N_dim: cutlass.Int32,
|
||||
stream: CUstream,
|
||||
):
|
||||
"""Execute the dot-product GEMM operation in steps:
|
||||
- Launch one CTA per small output-column tile with ``bs`` threads.
|
||||
- Keep one FP32 accumulator per token/expert in each thread.
|
||||
- Traverse K with vectorized main/tail, scalar, and ragged-tail loops
|
||||
from the precomputed K decomposition.
|
||||
- Reduce each token accumulator first within the warp, then across
|
||||
warps through shared memory, and store ``C[:, n]``.
|
||||
|
||||
:param gA: Input tensor A with shape ``[M, K]``.
|
||||
:type gA: cute.Tensor
|
||||
:param gB: Input tensor B with shape ``[N, K]``.
|
||||
:type gB: cute.Tensor
|
||||
:param gC: Output tensor C with shape ``[M, N]``.
|
||||
:type gC: cute.Tensor
|
||||
:param N_dim: Output column count used for the launch grid.
|
||||
:type N_dim: cutlass.Int32
|
||||
:param stream: CUDA stream for asynchronous execution.
|
||||
:type stream: CUstream
|
||||
"""
|
||||
self.kernel(
|
||||
gA,
|
||||
gB,
|
||||
gC,
|
||||
N_dim,
|
||||
self.m,
|
||||
self.m_per_group,
|
||||
self.main_vec_width,
|
||||
self.tail_vec_width,
|
||||
self.bs,
|
||||
self.num_warps,
|
||||
self.token_groups,
|
||||
self.epb,
|
||||
self.k_main_elems,
|
||||
self.k_tail_elems,
|
||||
self.k_done_all,
|
||||
self.ks_full,
|
||||
self.ks_part,
|
||||
self.k_scalar_full,
|
||||
self.k_part_offset,
|
||||
self.main_tiles,
|
||||
self.tail_tiles,
|
||||
).launch(
|
||||
grid=[cute.ceil_div(N_dim, self.epb), 1, 1],
|
||||
block=[self.bs, self.token_groups, 1],
|
||||
smem=self.m * self.epb * 4 * self.num_warps,
|
||||
stream=stream,
|
||||
use_pdl=self.use_pdl,
|
||||
min_blocks_per_mp=1,
|
||||
)
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
gA: cute.Tensor,
|
||||
gB: cute.Tensor,
|
||||
gC: cute.Tensor,
|
||||
N_dim: cutlass.Int32,
|
||||
M: cutlass.Constexpr,
|
||||
m_per_group: cutlass.Constexpr,
|
||||
main_vec_width: cutlass.Constexpr,
|
||||
tail_vec_width: cutlass.Constexpr,
|
||||
bs: cutlass.Constexpr,
|
||||
num_warps: cutlass.Constexpr,
|
||||
token_groups: cutlass.Constexpr,
|
||||
epb: cutlass.Constexpr,
|
||||
k_main_elems: cutlass.Constexpr,
|
||||
k_tail_elems: cutlass.Constexpr,
|
||||
k_done_all: cutlass.Constexpr,
|
||||
ks_full: cutlass.Constexpr,
|
||||
ks_part: cutlass.Constexpr,
|
||||
k_scalar_full: cutlass.Constexpr,
|
||||
k_part_offset: cutlass.Constexpr,
|
||||
main_tiles: cutlass.Constexpr,
|
||||
tail_tiles: cutlass.Constexpr,
|
||||
):
|
||||
tidx, group_idx, _ = cute.arch.thread_idx()
|
||||
n_tile_idx, _, _ = cute.arch.block_idx()
|
||||
|
||||
local_wid = tidx // cute.arch.WARP_SIZE
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_wait()
|
||||
|
||||
smem_red_layout = cute.make_layout(
|
||||
(M, epb, num_warps), stride=(epb * num_warps, num_warps, 1)
|
||||
)
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
sm = smem.allocate_tensor(cutlass.Float32, smem_red_layout, byte_alignment=16)
|
||||
|
||||
if const_expr(token_groups == 1):
|
||||
m_offset: cutlass.Constexpr = 0
|
||||
else:
|
||||
m_offset = group_idx * m_per_group
|
||||
self._compute_token_group(
|
||||
gA,
|
||||
gB,
|
||||
sm,
|
||||
n_tile_idx,
|
||||
tidx,
|
||||
local_wid,
|
||||
m_per_group,
|
||||
m_offset,
|
||||
main_vec_width,
|
||||
tail_vec_width,
|
||||
bs,
|
||||
epb,
|
||||
num_warps,
|
||||
k_main_elems,
|
||||
k_tail_elems,
|
||||
k_done_all,
|
||||
ks_full,
|
||||
ks_part,
|
||||
k_scalar_full,
|
||||
k_part_offset,
|
||||
main_tiles,
|
||||
tail_tiles,
|
||||
)
|
||||
|
||||
cute.arch.sync_threads()
|
||||
self._store_outputs(gC, sm, n_tile_idx, N_dim, tidx, group_idx, M, epb)
|
||||
|
||||
if const_expr(self.use_pdl):
|
||||
cute.arch.griddepcontrol_launch_dependents()
|
||||
|
||||
|
||||
def make_host_fp32w(
|
||||
k_val: int, bs: int = 128, token_groups: int = 1, epb: int = 1, m: int = 1
|
||||
):
|
||||
return LLFp32WDotprod(m=m, k=k_val, bs=bs, token_groups=token_groups, epb=epb)
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_cutedsl_available: bool | None = None
|
||||
_cute_ctx: tuple[Any, Any] | None = None
|
||||
|
||||
|
||||
def is_cutedsl_available(kernel_name: str) -> bool:
|
||||
global _cutedsl_available
|
||||
if _cutedsl_available is not None:
|
||||
return _cutedsl_available
|
||||
try:
|
||||
import cutlass # noqa: F401
|
||||
import cutlass.cute # noqa: F401
|
||||
|
||||
_cutedsl_available = True
|
||||
except ImportError:
|
||||
_cutedsl_available = False
|
||||
logger.info(
|
||||
"cuteDSL (CUTLASS Python) not available, %s disabled",
|
||||
kernel_name,
|
||||
)
|
||||
return _cutedsl_available
|
||||
|
||||
|
||||
def cute_context():
|
||||
global _cute_ctx
|
||||
if _cute_ctx is not None:
|
||||
return _cute_ctx
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
|
||||
_cute_ctx = (cute, CUstream)
|
||||
return _cute_ctx
|
||||
|
||||
|
||||
def current_cuda_stream():
|
||||
_, CUstream = cute_context()
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
|
||||
return CUstream(current_stream().cuda_stream)
|
||||
|
||||
|
||||
def use_pdl() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
|
||||
def cutlass_dtype(dtype: torch.dtype):
|
||||
from cutlass import BFloat16, Float16, Float32
|
||||
|
||||
if dtype == torch.bfloat16:
|
||||
return BFloat16
|
||||
if dtype == torch.float16:
|
||||
return Float16
|
||||
if dtype == torch.float32:
|
||||
return Float32
|
||||
raise ValueError(f"unsupported router GEMM dtype: {dtype}")
|
||||
|
||||
|
||||
def make_fake_gemm_tensors(
|
||||
*,
|
||||
M,
|
||||
K,
|
||||
N,
|
||||
a_dtype: torch.dtype,
|
||||
b_dtype: torch.dtype,
|
||||
divisibility: int,
|
||||
):
|
||||
from cutlass import Float32
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
hidden_states = make_fake_tensor(
|
||||
cutlass_dtype(a_dtype), (M, K), divisibility=divisibility
|
||||
)
|
||||
router_weight = make_fake_tensor(
|
||||
cutlass_dtype(b_dtype), (N, K), divisibility=divisibility
|
||||
)
|
||||
output = make_fake_tensor(Float32, (M, N), divisibility=1)
|
||||
return hidden_states, router_weight, output
|
||||
|
||||
|
||||
def validate_common_gemm_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
*,
|
||||
op_name: str,
|
||||
k_multiple: int | None = None,
|
||||
) -> None:
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError("hidden_states and router_weight must have device_type=cuda")
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be on the same CUDA device"
|
||||
)
|
||||
if output_dtype != torch.float32:
|
||||
raise ValueError(f"{op_name} only supports output_dtype=torch.float32")
|
||||
if hidden_states.shape[1] != router_weight.shape[1]:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have matching K dimensions"
|
||||
)
|
||||
if k_multiple is not None and hidden_states.shape[1] % k_multiple != 0:
|
||||
raise ValueError(f"{op_name} requires K to be divisible by {k_multiple}")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be contiguous row-major inputs"
|
||||
)
|
||||
@@ -11,24 +11,20 @@ from typing import Any, Literal
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from ._ll_router_common import (
|
||||
current_cuda_stream,
|
||||
cute_context,
|
||||
is_cutedsl_available,
|
||||
make_fake_gemm_tensors,
|
||||
use_pdl,
|
||||
validate_common_gemm_inputs,
|
||||
)
|
||||
|
||||
_cutedsl_available: bool | None = None
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
global _cutedsl_available
|
||||
if _cutedsl_available is not None:
|
||||
return _cutedsl_available
|
||||
try:
|
||||
import cutlass # noqa: F401
|
||||
import cutlass.cute # noqa: F401
|
||||
|
||||
_cutedsl_available = True
|
||||
except ImportError:
|
||||
_cutedsl_available = False
|
||||
logger.info("cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled")
|
||||
return _cutedsl_available
|
||||
return is_cutedsl_available("ll_bf16_gemm")
|
||||
|
||||
|
||||
# Default configs
|
||||
@@ -96,33 +92,6 @@ def _arch_tuned_configs() -> tuple[
|
||||
return {}, {}
|
||||
|
||||
|
||||
_cute_ctx = None
|
||||
|
||||
|
||||
def _cute():
|
||||
global _cute_ctx
|
||||
if _cute_ctx is not None:
|
||||
return _cute_ctx
|
||||
import cutlass.cute as cute
|
||||
from cuda.bindings.driver import CUstream
|
||||
|
||||
_cute_ctx = (cute, CUstream)
|
||||
return _cute_ctx
|
||||
|
||||
|
||||
def _stream():
|
||||
_, CUstream = _cute()
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
|
||||
return CUstream(current_stream().cuda_stream)
|
||||
|
||||
|
||||
def _use_pdl() -> bool:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.is_arch_support_pdl()
|
||||
|
||||
|
||||
class LLBf16Gemm:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompileKey:
|
||||
@@ -165,16 +134,17 @@ class LLBf16Gemm:
|
||||
|
||||
@staticmethod
|
||||
def _fake_gemm_tensors(*, M, K, N, divisibility: int):
|
||||
from cutlass import BFloat16, Float32
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
hidden_states = make_fake_tensor(BFloat16, (M, K), divisibility=divisibility)
|
||||
router_weight = make_fake_tensor(BFloat16, (N, K), divisibility=divisibility)
|
||||
output = make_fake_tensor(Float32, (M, N), divisibility=1)
|
||||
return hidden_states, router_weight, output
|
||||
return make_fake_gemm_tensors(
|
||||
M=M,
|
||||
K=K,
|
||||
N=N,
|
||||
a_dtype=torch.bfloat16,
|
||||
b_dtype=torch.bfloat16,
|
||||
divisibility=divisibility,
|
||||
)
|
||||
|
||||
def _compile_splitk(self, compile_key: CompileKey) -> None:
|
||||
cute, _ = _cute()
|
||||
cute, _ = cute_context()
|
||||
from ._ll_bf16_splitk import LLBf16SplitK
|
||||
|
||||
hidden_states, router_weight, output = self._fake_gemm_tensors(
|
||||
@@ -189,14 +159,14 @@ class LLBf16Gemm:
|
||||
num_stages=compile_key.num_stages,
|
||||
num_dma_warps=4,
|
||||
split_k=compile_key.split_k,
|
||||
use_pdl=_use_pdl(),
|
||||
use_pdl=use_pdl(),
|
||||
)
|
||||
compiled = cute.compile(
|
||||
gemm,
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output,
|
||||
_stream(),
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
self._splitk_cache[(compile_key.split_k, compile_key.num_stages)] = compiled
|
||||
@@ -207,7 +177,7 @@ class LLBf16Gemm:
|
||||
)
|
||||
|
||||
def _compile_dotprod(self, compile_key: CompileKey) -> None:
|
||||
cute, _ = _cute()
|
||||
cute, _ = cute_context()
|
||||
from ._ll_bf16_dotprod import LLBf16Dotprod
|
||||
|
||||
N = cute.sym_int()
|
||||
@@ -221,7 +191,7 @@ class LLBf16Gemm:
|
||||
gemm = LLBf16Dotprod(
|
||||
k=compile_key.K,
|
||||
bs=compile_key.bs,
|
||||
use_pdl=_use_pdl(),
|
||||
use_pdl=use_pdl(),
|
||||
prefetch_pdl_weights=self._prefetch_pdl_weights,
|
||||
)
|
||||
compiled = cute.compile(
|
||||
@@ -232,7 +202,7 @@ class LLBf16Gemm:
|
||||
compile_key.M,
|
||||
compile_key.K,
|
||||
1, # runtime N placeholder for fake-tensor compile
|
||||
_stream(),
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi --ptxas-options -maxrregcount=64",
|
||||
)
|
||||
self._compiled_cache[(compile_key.M, compile_key.K, compile_key.bs)] = compiled
|
||||
@@ -269,34 +239,19 @@ class LLBf16Gemm:
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> None:
|
||||
if hidden_states.dim() != 2 or router_weight.dim() != 2:
|
||||
raise ValueError("hidden_states and router_weight must be 2D tensors")
|
||||
if (
|
||||
hidden_states.dtype != torch.bfloat16
|
||||
or router_weight.dtype != torch.bfloat16
|
||||
):
|
||||
raise ValueError("hidden_states and router_weight must have dtype=bfloat16")
|
||||
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have device_type=cuda"
|
||||
)
|
||||
if hidden_states.device != router_weight.device:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be on the same CUDA device"
|
||||
)
|
||||
if output_dtype != torch.float32:
|
||||
raise ValueError("ll_bf16_gemm only supports output_dtype=torch.float32")
|
||||
if hidden_states.shape[1] != router_weight.shape[1]:
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must have matching K dimensions"
|
||||
)
|
||||
# Kernels use vectorized bf16 loads and require 16-byte row alignment.
|
||||
if hidden_states.shape[1] % 8 != 0:
|
||||
raise ValueError("ll_bf16_gemm requires K to be divisible by 8")
|
||||
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
|
||||
raise ValueError(
|
||||
"hidden_states and router_weight must be contiguous row-major inputs"
|
||||
)
|
||||
validate_common_gemm_inputs(
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output_dtype,
|
||||
op_name="ll_bf16_gemm",
|
||||
k_multiple=8,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -320,7 +275,7 @@ class LLBf16Gemm:
|
||||
self.compile(compile_key)
|
||||
kernel = self._compiled_cache[dotprod_cache_key]
|
||||
|
||||
stream = _stream()
|
||||
stream = current_cuda_stream()
|
||||
output = torch.empty(M, N, dtype=output_dtype, device=hidden_states.device)
|
||||
if compile_key.backend == "splitk":
|
||||
kernel(hidden_states, router_weight, output, stream, 1.0)
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, zip_inputs
|
||||
|
||||
from ._ll_router_common import (
|
||||
current_cuda_stream,
|
||||
cute_context,
|
||||
is_cutedsl_available,
|
||||
make_fake_gemm_tensors,
|
||||
use_pdl,
|
||||
validate_common_gemm_inputs,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
return is_cutedsl_available("ll_fp32w_gemm")
|
||||
|
||||
|
||||
_DEFAULT_DOTPROD_CONFIG = (192, 1, 1)
|
||||
_DEFAULT_DOTPROD_GROUPED_CONFIG = (192, 2, 1)
|
||||
|
||||
|
||||
_TUNED_DOTPROD_CONFIGS: dict[tuple[int, int], dict[int, tuple[int, int, int]]] = {
|
||||
(6144, 128): {
|
||||
**{
|
||||
M: (384, 1, 1)
|
||||
for M in (
|
||||
*range(1, 6),
|
||||
*range(9, 14, 2),
|
||||
17,
|
||||
*range(23, 32, 2),
|
||||
)
|
||||
},
|
||||
**{M: (256, 1, 1) for M in (7, 15, *range(19, 22, 2))},
|
||||
**{M: (384, 2, 1) for M in range(6, 12, 2)},
|
||||
**{M: (192, 2, 1) for M in (*range(12, 26, 2), 32)},
|
||||
**{M: (128, 2, 1) for M in range(26, 32, 2)},
|
||||
},
|
||||
(6144, 256): {
|
||||
**{M: (384, 1, 1) for M in range(1, 4)},
|
||||
**{M: (128, 1, 1) for M in (*range(4, 7, 2), 11)},
|
||||
**{M: (128, 1, 2) for M in range(5, 11, 2)},
|
||||
**{M: (128, 2, 2) for M in range(8, 33, 2)},
|
||||
**{M: (256, 1, 2) for M in range(13, 33, 2)},
|
||||
},
|
||||
}
|
||||
_SUPPORTED_ACTIVATION_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
|
||||
|
||||
|
||||
class LLFp32WGemm(VllmJitKernel["LLFp32WGemm.CompileKey"]):
|
||||
# Dot-prod is specialized on M/K, tile shape, and activation dtype.
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CompileKey:
|
||||
m: int
|
||||
k: int
|
||||
bs: int
|
||||
a_dtype: torch.dtype
|
||||
token_groups: int = 1
|
||||
epb: int = 1
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
from ._ll_fp32w_dotprod import LLFp32WDotprod
|
||||
|
||||
return LLFp32WDotprod(
|
||||
m=compile_key.m,
|
||||
k=compile_key.k,
|
||||
bs=compile_key.bs,
|
||||
token_groups=compile_key.token_groups,
|
||||
epb=compile_key.epb,
|
||||
use_pdl=use_pdl(),
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
M: int,
|
||||
K: int,
|
||||
N: int,
|
||||
a_dtype: torch.dtype,
|
||||
) -> CompileKey:
|
||||
default_config = (
|
||||
_DEFAULT_DOTPROD_GROUPED_CONFIG
|
||||
if N <= 128 and M >= 12 and M % 2 == 0
|
||||
else _DEFAULT_DOTPROD_CONFIG
|
||||
)
|
||||
tuned_configs = _TUNED_DOTPROD_CONFIGS.get((K, N))
|
||||
config = (
|
||||
tuned_configs.get(M, default_config) if tuned_configs else default_config
|
||||
)
|
||||
return self.CompileKey(
|
||||
m=M,
|
||||
k=K,
|
||||
bs=config[0],
|
||||
a_dtype=a_dtype,
|
||||
token_groups=config[1],
|
||||
epb=config[2],
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
shapes: Iterable[tuple[int, int]],
|
||||
m_values: Iterable[int],
|
||||
a_dtypes: Iterable[torch.dtype] = (torch.bfloat16,),
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
zip_inputs(*(dict(K=K, N=N) for K, N in shapes)),
|
||||
M=tuple(m_values),
|
||||
a_dtype=tuple(a_dtypes),
|
||||
)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
if compile_key in self._compiled_cache:
|
||||
return
|
||||
if compile_key.token_groups not in (1, 2):
|
||||
raise ValueError("ll_fp32w_gemm supports token_groups 1 or 2")
|
||||
if compile_key.epb not in (1, 2):
|
||||
raise ValueError("ll_fp32w_gemm supports epb 1 or 2")
|
||||
if compile_key.m % compile_key.token_groups != 0:
|
||||
raise ValueError("M must be divisible by token_groups")
|
||||
|
||||
cute, _ = cute_context()
|
||||
N = cute.sym_int()
|
||||
stride_divisibility = math.gcd(8, compile_key.k)
|
||||
hidden_states, router_weight, output = make_fake_gemm_tensors(
|
||||
M=compile_key.m,
|
||||
K=compile_key.k,
|
||||
N=N,
|
||||
a_dtype=compile_key.a_dtype,
|
||||
b_dtype=torch.float32,
|
||||
divisibility=stride_divisibility,
|
||||
)
|
||||
compiled = cute.compile(
|
||||
self.kernel(compile_key),
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output,
|
||||
1, # runtime N placeholder for fake-tensor compile
|
||||
current_cuda_stream(),
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
self._compiled_cache[compile_key] = compiled
|
||||
|
||||
@staticmethod
|
||||
def _validate_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
router_weight: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
) -> None:
|
||||
if hidden_states.dtype not in _SUPPORTED_ACTIVATION_DTYPES:
|
||||
raise ValueError("hidden_states must have dtype bf16, fp16, or fp32")
|
||||
if router_weight.dtype != torch.float32:
|
||||
raise ValueError("router_weight must have dtype=float32")
|
||||
validate_common_gemm_inputs(
|
||||
hidden_states,
|
||||
router_weight,
|
||||
output_dtype,
|
||||
op_name="ll_fp32w_gemm",
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor, # [M, K] bf16/fp16/fp32
|
||||
router_weight: torch.Tensor, # [N, K] fp32
|
||||
output_dtype: torch.dtype = torch.float32,
|
||||
) -> torch.Tensor: # [M, N] fp32
|
||||
self._validate_inputs(hidden_states, router_weight, output_dtype)
|
||||
|
||||
M, K = hidden_states.shape
|
||||
N = router_weight.shape[0]
|
||||
compile_key = self.dispatch(M=M, K=K, N=N, a_dtype=hidden_states.dtype)
|
||||
compiled = self._get_or_compile(
|
||||
compile_key,
|
||||
runtime_context={"M": M, "K": K, "N": N},
|
||||
)
|
||||
|
||||
stream = current_cuda_stream()
|
||||
output = torch.empty(M, N, dtype=output_dtype, device=hidden_states.device)
|
||||
compiled(hidden_states, router_weight, output, N, stream)
|
||||
return output
|
||||
|
||||
|
||||
LL_FP32W_GEMM_KERNEL = LLFp32WGemm()
|
||||
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
@@ -71,7 +72,7 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
input_schema=input_schema,
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -57,9 +57,21 @@ def _on_gfx1151() -> bool:
|
||||
return on_gfx1151()
|
||||
|
||||
|
||||
def _on_gfx115x() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx115x
|
||||
|
||||
return on_gfx115x()
|
||||
|
||||
|
||||
# Maximum activation rows supported by HIP skinny (Python M maps to C++ N_in).
|
||||
# When K*M exceeds regular LDS capacity, the C++ medium kernel caches the
|
||||
# prefix in LDS and reads the remaining activation elements from global memory.
|
||||
# On gfx115x the K*M term is dropped entirely: deep-K shapes stage every
|
||||
# activation row in LDS one K chunk at a time, and only when N is not a
|
||||
# multiple of the kernel's row tile does the medium kernel fall back to reading
|
||||
# the rows that do not fit from global memory.
|
||||
MAX_SKINNY_BATCH_SIZE = 5
|
||||
# 64 KiB per-workgroup LDS limit expressed in fp16 elements.
|
||||
# (AMD RDNA has 128 KiB total LDS per CU, but 64 KiB per workgroup.)
|
||||
@@ -86,6 +98,7 @@ def _triton_w4a16_skinny_fmt_kernel(
|
||||
K,
|
||||
K8, # K // 8
|
||||
num_groups, # K // group_size
|
||||
stride_sn, # row stride of scales_ptr and zp_ptr, in groups
|
||||
stride_bn, # b_ptr row stride; >= K8, the rows may be padded
|
||||
stride_am, # a_ptr row stride in elements; >= K, rows may be padded
|
||||
# Quantization parameters
|
||||
@@ -146,12 +159,12 @@ def _triton_w4a16_skinny_fmt_kernel(
|
||||
b = (b >> shifts_full) & 0xF
|
||||
|
||||
group_idx = (k_start * BLOCK_K) // group_size
|
||||
scale_ptrs = scales_ptr + offs_n * num_groups + group_idx
|
||||
scale_ptrs = scales_ptr + offs_n * stride_sn + group_idx
|
||||
scale_mask = offs_n < N
|
||||
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
|
||||
|
||||
if HAS_ZP:
|
||||
zp_ptrs = zp_ptr + (offs_n // 8) * num_groups + group_idx
|
||||
zp_ptrs = zp_ptr + (offs_n // 8) * stride_sn + group_idx
|
||||
zp_word = tl.load(zp_ptrs, mask=scale_mask, other=0)
|
||||
zp_raw = (zp_word >> (4 * (offs_n % 8))) & 0xF
|
||||
b_fp = (b - zp_raw[:, None]).to(scales.dtype) * scales[:, None]
|
||||
@@ -216,7 +229,8 @@ def triton_w4a16_skinny_fmt_gemm(
|
||||
"""
|
||||
assert a.stride(1) == 1, "Activation rows must be contiguous"
|
||||
assert b_q.stride(1) == 1, "Weight rows must be contiguous"
|
||||
assert scales.is_contiguous(), "Scales must be contiguous"
|
||||
assert scales.stride(1) == 1, "Scale rows must be contiguous"
|
||||
stride_sn = scales.stride(0)
|
||||
|
||||
M, K = a.shape
|
||||
N = b_q.shape[0]
|
||||
@@ -230,7 +244,10 @@ def triton_w4a16_skinny_fmt_gemm(
|
||||
f"scales shape mismatch: {scales.shape} vs ({N}, {num_groups})"
|
||||
)
|
||||
if zp is not None:
|
||||
assert zp.is_contiguous(), "Zero-points must be contiguous"
|
||||
assert zp.stride(1) == 1, "Zero-point rows must be contiguous"
|
||||
assert zp.stride(0) == stride_sn, (
|
||||
f"zp row stride {zp.stride(0)} must match scales {stride_sn}"
|
||||
)
|
||||
assert N % 8 == 0, f"N must be divisible by 8 for packed zp, got {N}"
|
||||
assert zp.shape == (N // 8, num_groups), (
|
||||
f"zp shape mismatch: {zp.shape} vs ({N // 8}, {num_groups})"
|
||||
@@ -337,6 +354,7 @@ def triton_w4a16_skinny_fmt_gemm(
|
||||
K,
|
||||
K8,
|
||||
num_groups,
|
||||
stride_sn,
|
||||
stride_bn,
|
||||
stride_am,
|
||||
group_size=group_size,
|
||||
@@ -466,7 +484,9 @@ def _rdna_hybrid_w4a16_apply_impl(
|
||||
K = x_2d.shape[1]
|
||||
N = w_q.shape[0]
|
||||
|
||||
if M <= MAX_SKINNY_BATCH_SIZE and K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS:
|
||||
if M <= MAX_SKINNY_BATCH_SIZE and (
|
||||
K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS or _on_gfx115x()
|
||||
):
|
||||
# record_function is not torch.compile-safe; use nullcontext when
|
||||
# compiling to keep the op traceable.
|
||||
ctx = (
|
||||
|
||||
@@ -186,7 +186,7 @@ class XPUW4A8IntLinearKernel(MPLinearKernel):
|
||||
|
||||
# Free the original unpacked int8 weight (still registered as "weight")
|
||||
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
|
||||
layer.register_parameter("weight", None)
|
||||
delattr(layer, "weight")
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -62,7 +62,12 @@ if is_aiter_found_and_supported():
|
||||
N = weight.shape[0]
|
||||
K = weight.shape[1]
|
||||
if rocm_use_aiter_fp4_asm_gemm:
|
||||
if M <= 64 and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K):
|
||||
# The preshuffle kernel's tuned configs split K in some M buckets.
|
||||
if (
|
||||
M <= 64
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
|
||||
):
|
||||
if x_scales is None:
|
||||
# use hip quant kernel for performance
|
||||
if M >= 32:
|
||||
@@ -116,7 +121,16 @@ if is_aiter_found_and_supported():
|
||||
x_q.shape[0], weight.shape[0], device=x_q.device, dtype=out_dtype
|
||||
)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
|
||||
config = None
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# The configs split K in some M buckets (up to M <= 256 for
|
||||
# tuned shapes). Keep the tile, drop the split.
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import _get_config
|
||||
|
||||
config, _ = _get_config(M, N, K)
|
||||
config = dict(config, NUM_KSPLIT=1)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y, config)
|
||||
return y
|
||||
|
||||
def gemm_with_dynamic_quant_fake(
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
@@ -55,12 +56,20 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
input_schema = quant_key_to_input_schema(self.config.activation_quant_key)
|
||||
act_key = self.config.activation_quant_key
|
||||
input_schema = quant_key_to_input_schema(act_key)
|
||||
# Keep checkpoint-declared FP8 activations instead of falling back to
|
||||
# A16 for group-scaled weights.
|
||||
self.layer_config = prepare_humming_linear_layer_config(
|
||||
layer, quant_config, input_schema=input_schema
|
||||
layer,
|
||||
quant_config,
|
||||
input_schema=input_schema,
|
||||
allow_input_schema_fallback=(
|
||||
act_key is None or act_key.dtype != current_platform.fp8_dtype()
|
||||
),
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp6E2M3Static
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
|
||||
@@ -66,7 +67,7 @@ class HummingMxFp6LinearKernel(MxFp6LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
@@ -57,7 +58,7 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
mxfp8_e4m3_quantize,
|
||||
swizzle_mxfp8_scale,
|
||||
xpu_mxfp8_quantize,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
_E8M0 = torch.float8_e8m0fnu
|
||||
|
||||
|
||||
class TorchMxFp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM using the native ``torch._scaled_mm`` dispatch.
|
||||
|
||||
Supported on XPU (oneDNN) and CUDA (cuBLASLt, SM90+).
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if current_platform.is_xpu():
|
||||
return True, None
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(90):
|
||||
return True, None
|
||||
return False, "requires XPU or CUDA (>=sm_90) for native torch._scaled_mm"
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# weight: [N, K] fp8_e4m3; weight_scale: [N, K//32] (E8M0-as-uint8).
|
||||
weight = layer.weight.data # [N, K]
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
|
||||
if current_platform.is_xpu():
|
||||
# oneDNN: un-swizzled scales; store column-major weight (B operand).
|
||||
replace_parameter(layer, "weight", weight.t())
|
||||
weight_scale = weight_scale.t().contiguous().view(_E8M0)
|
||||
else:
|
||||
# CUDA: L4-padded SWIZZLE_32_4_4 scales; keep [N, K] weight and
|
||||
# transpose to column-major at apply time.
|
||||
replace_parameter(layer, "weight", weight.contiguous())
|
||||
weight_scale = swizzle_mxfp8_scale(weight_scale, M=N, K=K).view(_E8M0)
|
||||
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
input_shape = x.shape
|
||||
|
||||
if current_platform.is_xpu():
|
||||
# weight stored column-major [K, N].
|
||||
K, N = weight.shape
|
||||
mat_b = weight
|
||||
input_2d = x.reshape(-1, K)
|
||||
# xpu_mxfp8_quantize already returns float8_e8m0fnu, [M, K//32].
|
||||
x_fp8, x_scale = xpu_mxfp8_quantize(input_2d)
|
||||
else:
|
||||
# weight stored [N, K]; transpose to column-major for _scaled_mm.
|
||||
N, K = weight.shape
|
||||
mat_b = weight.t()
|
||||
input_2d = x.reshape(-1, K)
|
||||
x_fp8, x_scale = mxfp8_e4m3_quantize(input_2d, is_sf_swizzled_layout=True)
|
||||
x_scale = x_scale.view(_E8M0)
|
||||
|
||||
out = torch._scaled_mm(
|
||||
x_fp8,
|
||||
mat_b,
|
||||
scale_a=x_scale,
|
||||
scale_b=weight_scale,
|
||||
bias=bias,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
return out.reshape(*input_shape[:-1], N)
|
||||
@@ -12,6 +12,7 @@ instead, since ``can_implement`` does not filter on K.
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
@@ -246,7 +247,10 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
block_scale = _as_block32_scale(weight_scale)
|
||||
# rocm_mxfp8_block32_gemm's split-K varies with M.
|
||||
block_scale = (
|
||||
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
|
||||
)
|
||||
if block_scale is not None:
|
||||
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
|
||||
# back from the loader expanded per row; keep the blocks and use
|
||||
|
||||
@@ -138,4 +138,64 @@ class B12xNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["B12xNvFp4LinearKernel"]
|
||||
class B12xNvFp4W4A16LinearKernel(B12xNvFp4LinearKernel):
|
||||
"""BF16 x NVFP4 GEMM without activation quantization."""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
blockscaled = _import_b12x_blockscaled()
|
||||
assert blockscaled is not None
|
||||
layer.b12x_nvfp4_packed_weight = blockscaled.pack_weight(
|
||||
layer.weight,
|
||||
layer.weight_scale,
|
||||
recipe="nvfp4",
|
||||
global_scale=layer.weight_global_scale,
|
||||
)
|
||||
|
||||
def get_b12x_warmup_unit(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
token_counts: tuple[int, ...],
|
||||
output_dtype: torch.dtype,
|
||||
) -> B12xWarmupUnit:
|
||||
def compile() -> None:
|
||||
for tokens in token_counts:
|
||||
source = torch.zeros(
|
||||
(tokens, layer.weight.shape[1] * 2),
|
||||
dtype=output_dtype,
|
||||
device=layer.weight.device,
|
||||
)
|
||||
self.apply_weights(layer, source)
|
||||
|
||||
return B12xWarmupUnit(
|
||||
name="NVFP4 W4A16",
|
||||
key=(
|
||||
type(self),
|
||||
layer.weight.device,
|
||||
tuple(layer.weight.shape),
|
||||
output_dtype,
|
||||
),
|
||||
compile=compile,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if x.dtype != torch.bfloat16:
|
||||
raise ValueError(f"b12x NVFP4 W4A16 requires BF16 input, got {x.dtype}")
|
||||
blockscaled = _import_b12x_blockscaled()
|
||||
assert blockscaled is not None
|
||||
output = blockscaled.mm(
|
||||
x.reshape(-1, x.shape[-1]).contiguous(),
|
||||
layer.b12x_nvfp4_packed_weight,
|
||||
required_mode="a16",
|
||||
)
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output.view(*x.shape[:-1], layer.weight.shape[0])
|
||||
|
||||
|
||||
__all__ = ["B12xNvFp4LinearKernel", "B12xNvFp4W4A16LinearKernel"]
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
@@ -71,7 +72,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -65,6 +66,7 @@ __all__ = [
|
||||
"PerTensorTorchFP8ScaledMMLinearKernel",
|
||||
"RowWiseTorchFP8ScaledMMLinearKernel",
|
||||
"ROCmFP8ScaledMMLinearKernel",
|
||||
"TritonFP8ScaledMMLinearKernel",
|
||||
"TritonInt8ScaledMMLinearKernel",
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
"Fp8BlockScaledMMLinearKernel",
|
||||
|
||||
@@ -104,7 +104,8 @@ class CutlassInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
# azp_adj is the AZP adjustment term, used to account for weights.
|
||||
# It does not depend on scales or azp, so it is the same for
|
||||
# static and dynamic quantization.
|
||||
# For more details, see csrc/quantization/w8a8/cutlass/Epilogues.md
|
||||
# For more details, see
|
||||
# csrc/quantization/w8a8/cutlass/Epilogues.md
|
||||
# https://github.com/vllm-project/vllm/blob/main/csrc/quantization/w8a8/cutlass/Epilogues.md
|
||||
if not config.input_symmetric:
|
||||
weight = getattr(layer, w_q_name)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .ScaledMMLinearKernel import (
|
||||
@@ -81,7 +82,7 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -148,7 +149,7 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -20,6 +20,8 @@ from .BlockScaledMMLinearKernel import (
|
||||
)
|
||||
from .cutlass import CutlassInt8ScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
@@ -131,9 +133,7 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
x.contiguous(), i_s, i_zp, symmetric=symmetric
|
||||
)
|
||||
|
||||
out = triton_scaled_mm(
|
||||
x_q, w_q, scale_a=x_s, scale_b=w_s, out_dtype=x.dtype, bias=bias
|
||||
)
|
||||
out = torch.ops.vllm.int8_scaled_mm_func(x_q, w_q, x_s, w_s, x.dtype, bias)
|
||||
|
||||
if azp_adj is not None:
|
||||
# Asymmetric quantization: subtract the zero-point correction.
|
||||
@@ -156,6 +156,46 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
return out
|
||||
|
||||
|
||||
class TritonFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda_alike():
|
||||
return False, "requires ROCm or CUDA."
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
act_group = c.activation_quant_key.scale.group_shape
|
||||
weight_group = c.weight_quant_key.scale.group_shape
|
||||
if not (act_group.is_per_tensor() or act_group.is_per_token()):
|
||||
return False, "requires per-tensor or per-token activation scales."
|
||||
if not (
|
||||
weight_group.is_per_tensor()
|
||||
or weight_group.is_per_channel()
|
||||
or weight_group.is_per_token()
|
||||
):
|
||||
return False, "requires per-tensor, per-channel or per-token weight scales."
|
||||
return True, None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
out = triton_scaled_mm(
|
||||
A, B, scale_a=As, scale_b=Bs, out_dtype=out_dtype, bias=bias
|
||||
)
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
@@ -180,6 +220,44 @@ class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
)
|
||||
|
||||
|
||||
def _int8_scaled_mm_func(
|
||||
qx: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return triton_scaled_mm(
|
||||
qx,
|
||||
weight,
|
||||
scale_a=x_scale,
|
||||
scale_b=weight_scale,
|
||||
out_dtype=output_dtype,
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
|
||||
def _int8_scaled_mm_fake(
|
||||
qx: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
output_dtype: torch.dtype,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(qx.size(0), weight.size(1)), dtype=output_dtype, device=qx.device
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
"int8_scaled_mm_func",
|
||||
_int8_scaled_mm_func,
|
||||
fake_impl=_int8_scaled_mm_fake,
|
||||
)
|
||||
|
||||
|
||||
# TODO we should be able to change the type of block_size to GroupShape
|
||||
# after we resolve GroupShape compilation issue
|
||||
# https://github.com/vllm-project/vllm/issues/25270
|
||||
|
||||
@@ -261,6 +261,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.deepseek_v4_utils import get_compress_ratios
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down, round_up
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -661,6 +662,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM120",
|
||||
"FLASH_ATTN_MLA_SPARSE_FA4",
|
||||
"DEEPSEEK_V32_INDEXER",
|
||||
):
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
@@ -708,6 +710,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
use_pcp=self.use_pcp,
|
||||
)
|
||||
|
||||
# TODO(ROCm): both BMM kernels re-quantize kv_b_proj online, ignoring
|
||||
# the precision the checkpoint asked for, so they now default to off.
|
||||
# Turning them back on needs long-context accuracy validation, and the
|
||||
# per-layer decision belongs in --quantization-config.targets rather
|
||||
# than in this layer.
|
||||
self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
|
||||
|
||||
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
|
||||
@@ -751,7 +758,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
|
||||
def _uses_flat_kv_cache(self) -> bool:
|
||||
backend = self.attn_backend.get_name()
|
||||
return backend == "FLASHINFER_MLA_SPARSE" or (
|
||||
return backend in ("FLASHINFER_MLA_SPARSE", "FLASH_ATTN_MLA_SPARSE_FA4") or (
|
||||
backend == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype not in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
)
|
||||
@@ -1376,14 +1383,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
is_index_group_leader=self.indexer is not None,
|
||||
non_causal_multi_token_decode=self.non_causal_multi_token_decode,
|
||||
)
|
||||
# SM100 FlashMLA paged kernels also express TMA coordinates in token rows.
|
||||
uses_tma_rows = (
|
||||
self.attn_backend.get_name() == "FLASHMLA_SPARSE"
|
||||
and self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla")
|
||||
and current_platform.is_device_capability_family(100)
|
||||
)
|
||||
if self._uses_flat_kv_cache() or uses_tma_rows:
|
||||
spec = replace(spec, block_stride_alignment=spec.state_content_size_bytes)
|
||||
return spec
|
||||
|
||||
def _v_up_proj(self, x: torch.Tensor, out: torch.Tensor):
|
||||
@@ -1742,7 +1741,7 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
hf_text_config = model_config.hf_text_config
|
||||
|
||||
# Check if this is a DeepseekV4 config (uses unified head_dim + rope_head_dim)
|
||||
if hasattr(hf_text_config, "compress_ratios"):
|
||||
if get_compress_ratios(hf_text_config):
|
||||
# DeepseekV4 style config: unified head_dim with rope_head_dim
|
||||
head_dim = hf_text_config.head_dim
|
||||
rope_head_dim = hf_text_config.qk_rope_head_dim
|
||||
@@ -1865,6 +1864,9 @@ def _use_masked_mha(
|
||||
seq_len: int,
|
||||
has_context: bool,
|
||||
) -> bool:
|
||||
if backend_name == "FLASH_ATTN_MLA_SPARSE_FA4":
|
||||
# Its prefill batches run on FlashInfer's kernel, so it shares those rows.
|
||||
backend_name = "FLASHINFER_MLA_SPARSE"
|
||||
thresholds = (
|
||||
_MASKED_MHA_THRESHOLDS.get((qk_head_dim, v_head_dim), {})
|
||||
.get(backend_name, {})
|
||||
@@ -2548,16 +2550,26 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
"Non-causal multi-token MLA requires an explicitly supported "
|
||||
"attention group."
|
||||
)
|
||||
query_lens = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1]
|
||||
num_active_reqs = int(torch.count_nonzero(query_lens > 0))
|
||||
uniform_active_queries = num_active_reqs > 0 and bool(
|
||||
torch.all(query_lens[:num_active_reqs] == query_lens[0])
|
||||
# query_start_loc_cpu is host-side and holds one entry per
|
||||
# request, so this check costs more in tensor dispatch than in
|
||||
# the work it does. Read it out once and validate on Python ints:
|
||||
# the block runs on every build, between decode steps.
|
||||
qsl = query_start_loc_cpu.tolist()
|
||||
query_lens = [qsl[i + 1] - qsl[i] for i in range(len(qsl) - 1)]
|
||||
num_active_reqs = 0
|
||||
for qlen in query_lens:
|
||||
if qlen > 0:
|
||||
num_active_reqs += 1
|
||||
uniform_active_queries = num_active_reqs > 0 and all(
|
||||
qlen == query_lens[0] for qlen in query_lens[:num_active_reqs]
|
||||
)
|
||||
trailing_graph_padding = all(
|
||||
qlen == 0 for qlen in query_lens[num_active_reqs:]
|
||||
)
|
||||
trailing_graph_padding = bool(torch.all(query_lens[num_active_reqs:] == 0))
|
||||
if not (uniform_active_queries and trailing_graph_padding):
|
||||
raise ValueError(
|
||||
"Non-causal MLA requires a uniform query block; got query "
|
||||
f"lengths {query_lens.tolist()}."
|
||||
f"lengths {query_lens}."
|
||||
)
|
||||
# Use exact GPU sequence lengths instead of the prefill path's CPU
|
||||
# context-length upper bounds.
|
||||
|
||||
@@ -36,12 +36,18 @@ from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
np_to_pinned_tensor,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backend import AttentionLayer, AttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
from vllm.v1.attention.backends.mla.index_group import (
|
||||
HiSparseMLAIndexGroup,
|
||||
SparseMLAIndexGroup,
|
||||
SparseMLAIndexGroupBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
flat_kv_row_view,
|
||||
triton_convert_req_index_to_global_index,
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.attention.ops.merge_attn_states import merge_attn_states
|
||||
@@ -630,6 +636,9 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
"""Sparse MLA base with dense and masked-MHA prefill paths."""
|
||||
|
||||
is_sparse = True
|
||||
# Reuse the index group's top-k conversion only for uniform decode batches
|
||||
# of multi-layer groups; other batches convert per layer.
|
||||
group_conversion_decode_only: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -728,6 +737,162 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
return_valid_counts=return_valid_counts,
|
||||
)
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: T,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
assert self.topk_indices_buffer is not None
|
||||
num_tokens = q[0].shape[0] if isinstance(q, tuple) else q.shape[0]
|
||||
topk = self.topk_indices_buffer[:num_tokens]
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
block_size = attn_metadata.block_size
|
||||
index_group = self.index_group
|
||||
if not isinstance(index_group, HiSparseMLAIndexGroup):
|
||||
_, block_stride_rows = flat_kv_row_view(kv_c_and_k_pe_cache, block_size)
|
||||
if self.dcp_world_size > 1:
|
||||
physical_topk, valid_counts = triton_filter_and_convert_dcp_index(
|
||||
attn_metadata.req_id_per_token[:num_tokens],
|
||||
attn_metadata.block_table,
|
||||
topk,
|
||||
dcp_size=self.dcp_world_size,
|
||||
dcp_rank=self.dcp_rank,
|
||||
cp_kv_cache_interleave_size=attn_metadata.cp_kv_cache_interleave_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
elif self.group_conversion_decode_only and not (
|
||||
index_group is not None
|
||||
and index_group.num_layers > 1
|
||||
and num_decode_tokens >= num_tokens
|
||||
):
|
||||
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_tokens],
|
||||
attn_metadata.block_table,
|
||||
topk,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=topk.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
else:
|
||||
physical_topk, valid_counts = self._convert_logical_to_physical_topk(
|
||||
topk,
|
||||
attn_metadata,
|
||||
block_stride_rows=block_stride_rows,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
return self._forward_mqa_kernel(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
physical_topk,
|
||||
valid_counts,
|
||||
layer=layer,
|
||||
block_size=block_size,
|
||||
is_decode=num_decode_tokens >= num_tokens,
|
||||
)
|
||||
|
||||
assert self.dcp_world_size == 1
|
||||
layer_index = self.index_group_index
|
||||
hot_cache = index_group.physical_kv_cache(layer_index).view(
|
||||
kv_c_and_k_pe_cache.dtype
|
||||
)
|
||||
assert hot_cache.is_contiguous(), "HiSparse hot buffer must be contiguous"
|
||||
if (
|
||||
num_decode_tokens >= num_tokens
|
||||
or index_group.cache(layer_index).all_context_pages_resident
|
||||
):
|
||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||
layer_index,
|
||||
topk,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
return self._forward_mqa_kernel(
|
||||
q,
|
||||
hot_cache,
|
||||
physical_topk,
|
||||
valid_counts,
|
||||
layer=layer,
|
||||
block_size=block_size,
|
||||
is_decode=num_decode_tokens >= num_tokens,
|
||||
)
|
||||
|
||||
decode_out = decode_lse = None
|
||||
if num_decode_tokens:
|
||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||
layer_index,
|
||||
topk[:num_decode_tokens],
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
q_decode = (
|
||||
(q[0][:num_decode_tokens], q[1][:num_decode_tokens])
|
||||
if isinstance(q, tuple)
|
||||
else q[:num_decode_tokens]
|
||||
)
|
||||
decode_out, decode_lse = self._forward_mqa_kernel(
|
||||
q_decode,
|
||||
hot_cache,
|
||||
physical_topk,
|
||||
valid_counts,
|
||||
layer=layer,
|
||||
block_size=block_size,
|
||||
is_decode=True,
|
||||
)
|
||||
|
||||
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
||||
layer_index, kv_c_and_k_pe_cache, attn_metadata
|
||||
)
|
||||
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
topk[num_decode_tokens:],
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
q_prefill = (
|
||||
(q[0][num_decode_tokens:], q[1][num_decode_tokens:])
|
||||
if isinstance(q, tuple)
|
||||
else q[num_decode_tokens:]
|
||||
)
|
||||
prefill_out, prefill_lse = self._forward_mqa_kernel(
|
||||
q_prefill,
|
||||
prefill_cache,
|
||||
physical_topk,
|
||||
valid_counts,
|
||||
layer=layer,
|
||||
block_size=block_size,
|
||||
is_decode=False,
|
||||
)
|
||||
if decode_out is None:
|
||||
return prefill_out, prefill_lse
|
||||
out = torch.cat((decode_out, prefill_out))
|
||||
if decode_lse is None:
|
||||
return out, None
|
||||
assert prefill_lse is not None
|
||||
return out, torch.cat((decode_lse, prefill_lse))
|
||||
|
||||
def _forward_mqa_kernel(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
valid_counts: torch.Tensor,
|
||||
*,
|
||||
layer: AttentionLayer,
|
||||
block_size: int,
|
||||
is_decode: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def masked_mha_workspace_fits(prefill: MLACommonPrefillMetadata) -> bool:
|
||||
"""Whether this prefill batch's top-k masks fit the workspace."""
|
||||
|
||||
@@ -106,7 +106,11 @@ if HAS_TRITON:
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
@@ -132,7 +136,11 @@ if HAS_TRITON:
|
||||
)
|
||||
|
||||
__all__ += [
|
||||
"AiterExperts",
|
||||
"AiterUnquantizedExperts",
|
||||
"AiterFp8BlockExperts",
|
||||
"AiterFp8TensorExperts",
|
||||
"AiterFp8ChannelExperts",
|
||||
"AiterMxfp4Experts",
|
||||
"fused_topk",
|
||||
"fused_experts",
|
||||
"get_config_file_name",
|
||||
|
||||
@@ -329,14 +329,24 @@ def maybe_make_prepare_finalize(
|
||||
elif moe.use_mori_kernels:
|
||||
assert quant_config is not None
|
||||
|
||||
# MXFP4 activations are dispatched as packed FP4 with their e8m0
|
||||
# block scales, which the experts consume without re-quantizing.
|
||||
use_fp4_dispatch = quant_config.use_mxfp4_w4a4
|
||||
# Note: We may want to use FP8 dispatch just to reduce
|
||||
# data movement.
|
||||
use_fp8_dispatch = (
|
||||
use_fp8_dispatch = not use_fp4_dispatch and (
|
||||
quant_config.is_per_act_token
|
||||
or quant_config.is_block_quantized
|
||||
or quant_config.is_per_tensor
|
||||
)
|
||||
if use_fp8_dispatch:
|
||||
if use_fp4_dispatch:
|
||||
# One e8m0 scale per 32-element block. token_hidden_size stays
|
||||
# the unpacked hidden size: mori sizes its token buffers by the
|
||||
# BF16 input type, which also covers the BF16 combine.
|
||||
quant_dtype = torch.float4_e2m1fn_x2
|
||||
scale_dim = moe.hidden_dim // 32
|
||||
scale_type_size = torch.float8_e8m0fnu.itemsize
|
||||
elif use_fp8_dispatch:
|
||||
# For PTPC (per token per channel) or per-tensor quant,
|
||||
# scale dim is 1. For 1x128 quant, scale dim is
|
||||
# hidden_dim // 128
|
||||
@@ -346,18 +356,20 @@ def maybe_make_prepare_finalize(
|
||||
if (quant_config.is_per_act_token or quant_config.is_per_tensor)
|
||||
else moe.hidden_dim // 128
|
||||
)
|
||||
scale_type_size = torch.float32.itemsize
|
||||
else:
|
||||
# Unquantized dispatch (e.g. AITER with defer_input_quant):
|
||||
# dispatch raw BF16/FP16 data, no scales needed.
|
||||
quant_dtype = moe.in_dtype
|
||||
scale_dim = 0
|
||||
scale_type_size = 0
|
||||
all_to_all_args = dict(
|
||||
rank=all2all_manager.rank,
|
||||
num_ep_ranks=all2all_manager.world_size,
|
||||
quant_dtype=quant_dtype,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
scale_dim=scale_dim,
|
||||
scale_type_size=0 if scale_dim == 0 else torch.float32.itemsize,
|
||||
scale_type_size=scale_type_size,
|
||||
max_num_tokens_per_dp_rank=moe.max_num_tokens,
|
||||
input_dtype=moe.in_dtype,
|
||||
num_local_experts=moe.num_experts // all2all_manager.world_size,
|
||||
@@ -370,6 +382,7 @@ def maybe_make_prepare_finalize(
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
use_fp4_dispatch=use_fp4_dispatch,
|
||||
)
|
||||
|
||||
elif moe.use_fi_nvl_two_sided_kernels:
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.b12x import (
|
||||
B12X_BLOCK_CODECS,
|
||||
B12xWarmupUnit,
|
||||
get_b12x_fused_moe,
|
||||
reuse_packed_weight_storage,
|
||||
@@ -38,6 +39,7 @@ _B12X_MOE_MODES: dict[
|
||||
tuple[torch.dtype | str | None, torch.dtype | str | None],
|
||||
tuple[str, str, str],
|
||||
] = {
|
||||
**{(codec, None): ("w4a16", codec, "w31") for codec in B12X_BLOCK_CODECS},
|
||||
("mxfp4", "mxfp8"): ("w4a8_mx", "fp4_e8m0_k32", "w31"),
|
||||
("mxfp4", None): ("w4a16", "fp4_e8m0_k32", "w31"),
|
||||
("nvfp4", "nvfp4"): ("nvfp4", "modelopt_nvfp4", "w31"),
|
||||
@@ -206,10 +208,13 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
if quant_config.weight_quant_dtype not in ("mxfp4", "nvfp4"):
|
||||
if quant_config.weight_quant_dtype not in (
|
||||
"mxfp4",
|
||||
"nvfp4",
|
||||
*B12X_BLOCK_CODECS,
|
||||
):
|
||||
raise ValueError(
|
||||
"b12x MoE requires MXFP4 or NVFP4 weights, got "
|
||||
f"{quant_config.weight_quant_dtype}"
|
||||
f"unsupported b12x MoE weight format: {quant_config.weight_quant_dtype}"
|
||||
)
|
||||
scheme = (
|
||||
quant_config.weight_quant_dtype,
|
||||
@@ -293,6 +298,31 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
raise RuntimeError(
|
||||
"b12x MoE weights must be prepared before CUDA graph capture"
|
||||
)
|
||||
if self._source_format in B12X_BLOCK_CODECS:
|
||||
fused_moe = _require_b12x_fused_moe()
|
||||
block_size, _ = B12X_BLOCK_CODECS[self._source_format]
|
||||
plan = fused_moe.plan_weights(
|
||||
source=fused_moe.PackedSource(
|
||||
format=self._source_format, w13_layout=self._w13_layout
|
||||
),
|
||||
activation=fused_moe.ActivationSpec(
|
||||
mode="a16",
|
||||
nonlinearity=_b12x_activation_name(activation),
|
||||
io_dtype=params_dtype,
|
||||
),
|
||||
geometry=fused_moe.MoEGeometry(
|
||||
num_experts=w1.shape[0],
|
||||
hidden_size=w2.shape[1],
|
||||
intermediate_size=w2.shape[2] * block_size,
|
||||
),
|
||||
)
|
||||
prepared = fused_moe.prepare_weights(
|
||||
plan=plan,
|
||||
weights=fused_moe.BlockQuantWeights(
|
||||
w13=w1, w2=w2, codec=self._source_format
|
||||
),
|
||||
)
|
||||
return prepared._impl
|
||||
if self.w1_scale is None or self.w2_scale is None:
|
||||
raise ValueError("b12x MoE requires w1 and w2 block scales")
|
||||
|
||||
@@ -345,6 +375,8 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
|
||||
def _refresh_quant_config(self, layer: torch.nn.Module) -> None:
|
||||
if self._source_format in B12X_BLOCK_CODECS:
|
||||
return
|
||||
self.quant_config._w1.scale = layer.w13_weight_scale
|
||||
self.quant_config._w2.scale = layer.w2_weight_scale
|
||||
if self._source_format != "modelopt_nvfp4":
|
||||
@@ -446,8 +478,10 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"MXFP4 W4A8 requires hidden size divisible by 256 and "
|
||||
"per-rank intermediate size divisible by 32",
|
||||
(
|
||||
"MXFP4 W4A8 requires hidden size divisible by 256 and "
|
||||
"per-rank intermediate size divisible by 32"
|
||||
),
|
||||
)
|
||||
return mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
@@ -530,14 +564,15 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
if w1.numel() and w2.numel():
|
||||
if w1.numel() and w2.numel() and self._source_format not in B12X_BLOCK_CODECS:
|
||||
return super().moe_problem_size(a1, w1, w2, topk_ids)
|
||||
prepared = self._prepared()
|
||||
tokens = int(a1.shape[0] if a1.ndim == 2 else a1.shape[1])
|
||||
return (
|
||||
int(prepared.num_experts),
|
||||
tokens,
|
||||
int(prepared.intermediate_size) * 2,
|
||||
int(prepared.intermediate_size)
|
||||
* (2 if self.moe_config.activation.is_gated else 1),
|
||||
int(a1.shape[-1]),
|
||||
int(topk_ids.shape[1]),
|
||||
)
|
||||
|
||||
@@ -1293,6 +1293,8 @@ class FusedMoEConfig:
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
has_hash_routing: bool = False
|
||||
# Routes are adjusted between routing and expert execution.
|
||||
require_decomposed_backend: bool = False
|
||||
shared_expert_prefix: str | None = None
|
||||
|
||||
# When True, the MoE skips its final cross-rank all-reduce (and the separate
|
||||
|
||||
@@ -18,19 +18,15 @@ from vllm.platforms import current_platform
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
|
||||
"""Reinterpret a uint8 UE8M0 exponent tensor as float32 scale values."""
|
||||
return (sf.to(torch.int32) << 23).view(torch.float32)
|
||||
|
||||
|
||||
class DeepGemmMegaMoEBackend:
|
||||
"""Abstract MegaMoE backend that hides deep_gemm-specific details.
|
||||
|
||||
A backend advertises its expected hidden-state quantization layout via
|
||||
``hidden_quant`` and its expected MMA type via ``mma_type``. It is
|
||||
responsible for the routed-expert weight transform run at load time, the
|
||||
optional shared-expert weight transform, and the actual MegaMoE kernel
|
||||
invocation at inference time. Runtime capability checks live in
|
||||
``hidden_quant`` and its expected MMA type via ``mma_type``. It owns the
|
||||
kernel weight layout: :meth:`create_weights` registers parameters in that
|
||||
layout and :meth:`load_weight` writes each checkpoint tensor into it, so
|
||||
loading (and reloading) needs no post-load transform. It also runs the
|
||||
MegaMoE kernel. Runtime capability checks live in
|
||||
:func:`get_deep_gemm_mega_moe_backend`: obtaining a backend already implies
|
||||
that the current device and shapes are supported.
|
||||
"""
|
||||
@@ -39,49 +35,39 @@ class DeepGemmMegaMoEBackend:
|
||||
mma_type: str = "fp8xfp4"
|
||||
hidden_quant: QuantKey = kMxfp8Dynamic
|
||||
|
||||
def transform_weights(
|
||||
def create_weights(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
*,
|
||||
w13_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
num_local_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
activation: str | None = None,
|
||||
) -> tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]:
|
||||
num_shared_experts: int,
|
||||
) -> None:
|
||||
"""Register the expert (and fused shared-expert) kernel-layout parameters:
|
||||
``w13_weight``, ``w13_weight_scale``, ``w2_weight``, ``w2_weight_scale``
|
||||
and, if ``num_shared_experts``, ``shared_l{1,2}_weight[_scale]``. Routed
|
||||
parameters are indexed by local expert on dim 0, which EPLB moves."""
|
||||
raise NotImplementedError
|
||||
|
||||
def supports_shared_experts(self, deep_gemm) -> bool:
|
||||
return False
|
||||
|
||||
def transform_shared_expert_weights(
|
||||
def load_weight(
|
||||
self,
|
||||
*,
|
||||
shared_experts: nn.Module,
|
||||
num_shared_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
prefix: str,
|
||||
) -> (
|
||||
tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]
|
||||
| None
|
||||
):
|
||||
"""Return ``(l1, l2)`` transformed shared-expert weights, or ``None``.
|
||||
dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
shard_id: str,
|
||||
is_scale: bool,
|
||||
) -> None:
|
||||
"""Write checkpoint shard ``src`` (``w1``/``w2``/``w3``) into ``dst``, one
|
||||
expert's slice or a shared-expert parameter, in the kernel layout."""
|
||||
raise NotImplementedError
|
||||
|
||||
``shared_experts`` must expose ``gate_up_proj`` and ``down_proj``
|
||||
linear layers. A ``None`` return means the backend cannot fuse shared
|
||||
experts for this module; the caller should fall back to a separate
|
||||
shared MLP.
|
||||
def supports_shared_experts(self, shared_experts: nn.Module) -> bool:
|
||||
"""Whether ``shared_experts`` can be fused into the MegaMoE kernel.
|
||||
|
||||
Decided at construction from the installed kernel API and the shared
|
||||
MLP's parameters, so the fused layout is fixed before any weight loads.
|
||||
"""
|
||||
return None
|
||||
return False
|
||||
|
||||
def run_mega_moe(
|
||||
self,
|
||||
@@ -133,229 +119,153 @@ class DeepGemmSm100MegaMoEBackend(DeepGemmMegaMoEBackend):
|
||||
mma_type = "fp8xfp4"
|
||||
hidden_quant = kMxfp8Dynamic
|
||||
|
||||
def transform_weights(
|
||||
def create_weights(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
*,
|
||||
w13_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
num_local_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
activation: str | None = None,
|
||||
) -> tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]:
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
w13_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
ue8m0_uint8_to_float(w13_weight_scale).contiguous(),
|
||||
2 * intermediate_size,
|
||||
hidden_size,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
w2_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
ue8m0_uint8_to_float(w2_weight_scale).contiguous(),
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
(1, 32),
|
||||
num_local_experts,
|
||||
)
|
||||
kwargs = {} if activation is None else {"activation": activation}
|
||||
return deep_gemm.transform_weights_for_mega_moe(
|
||||
(w13_weight.view(torch.int8).contiguous(), w13_scale),
|
||||
(w2_weight.view(torch.int8).contiguous(), w2_scale),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def supports_shared_experts(self, deep_gemm) -> bool:
|
||||
"""Check the Python API before touching a symmetric-memory group.
|
||||
|
||||
This also gives users of an older precompiled vLLM wheel a safe serial
|
||||
fallback instead of failing halfway through multi-rank buffer setup.
|
||||
"""
|
||||
try:
|
||||
buffer_params = signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters
|
||||
kernel_params = signature(deep_gemm.fp8_fp4_mega_moe).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
hasattr(deep_gemm, "get_block_m_for_mega_moe")
|
||||
and hasattr(deep_gemm, "transform_weights_for_mega_moe")
|
||||
and "num_shared_experts" in buffer_params
|
||||
and "shared_l1_weights" in kernel_params
|
||||
and "shared_l2_weights" in kernel_params
|
||||
)
|
||||
|
||||
def transform_shared_expert_weights(
|
||||
self,
|
||||
*,
|
||||
shared_experts: nn.Module,
|
||||
num_shared_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
prefix: str,
|
||||
) -> (
|
||||
tuple[
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
]
|
||||
| None
|
||||
):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
) -> None:
|
||||
def register(name: str, data: torch.Tensor) -> None:
|
||||
layer.register_parameter(name, nn.Parameter(data, requires_grad=False))
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
gate_up = shared_experts.gate_up_proj
|
||||
down = shared_experts.down_proj
|
||||
gate_up_weight = gate_up.weight.data
|
||||
gate_up_scale = (
|
||||
gate_up.weight_scale
|
||||
if hasattr(gate_up, "weight_scale")
|
||||
else gate_up.weight_scale_inv
|
||||
).data
|
||||
down_weight = down.weight.data
|
||||
down_scale = (
|
||||
down.weight_scale
|
||||
if hasattr(down, "weight_scale")
|
||||
else down.weight_scale_inv
|
||||
).data
|
||||
|
||||
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
|
||||
# while the checkpoint uses coarser block-FP8 scales (usually
|
||||
# 128x128). Build a dedicated, numerically equivalent scale view.
|
||||
ue8m0_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8, torch.int32)
|
||||
if (
|
||||
gate_up_scale.dtype in ue8m0_scale_dtypes
|
||||
and down_scale.dtype in ue8m0_scale_dtypes
|
||||
):
|
||||
gate_up_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
gate_up,
|
||||
gate_up_scale,
|
||||
gate_up_weight.shape[0],
|
||||
gate_up_weight.shape[1],
|
||||
prefix,
|
||||
)
|
||||
down_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
down,
|
||||
down_scale,
|
||||
down_weight.shape[0],
|
||||
down_weight.shape[1],
|
||||
prefix,
|
||||
)
|
||||
|
||||
if gate_up_scale is None or down_scale is None:
|
||||
return None
|
||||
|
||||
shared_intermediate_size = intermediate_size * num_shared_experts
|
||||
expected_gate_up_shape = (
|
||||
2 * shared_intermediate_size,
|
||||
hidden_size,
|
||||
# FP4 weights two per byte; UE8M0 scales MN-major, four per int32 along K.
|
||||
e, h, i = num_local_experts, hidden_size, intermediate_size
|
||||
register("w13_weight", torch.zeros(e, 2 * i, h // 2, dtype=torch.int8))
|
||||
register(
|
||||
"w13_weight_scale",
|
||||
torch.zeros(e, h // 128, 2 * i, dtype=torch.int32).transpose(1, 2),
|
||||
)
|
||||
expected_down_shape = (hidden_size, shared_intermediate_size)
|
||||
if (
|
||||
gate_up_weight.dtype != torch.float8_e4m3fn
|
||||
or down_weight.dtype != torch.float8_e4m3fn
|
||||
or gate_up_scale.dtype != torch.int32
|
||||
or down_scale.dtype != torch.int32
|
||||
or tuple(gate_up_weight.shape) != expected_gate_up_shape
|
||||
or tuple(down_weight.shape) != expected_down_shape
|
||||
):
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: expected "
|
||||
"replicated block-FP8 weights with gate_up=%s, down=%s, and "
|
||||
"DeepGEMM int32 scales; got gate_up=%s/%s/%s and down=%s/%s/%s.",
|
||||
prefix,
|
||||
expected_gate_up_shape,
|
||||
expected_down_shape,
|
||||
tuple(gate_up_weight.shape),
|
||||
gate_up_weight.dtype,
|
||||
gate_up_scale.dtype,
|
||||
tuple(down_weight.shape),
|
||||
down_weight.dtype,
|
||||
down_scale.dtype,
|
||||
)
|
||||
return None
|
||||
|
||||
transformed_l1, transformed_l2 = deep_gemm.transform_weights_for_mega_moe(
|
||||
(gate_up_weight, gate_up_scale),
|
||||
(down_weight, down_scale),
|
||||
register("w2_weight", torch.zeros(e, h, i // 2, dtype=torch.int8))
|
||||
register(
|
||||
"w2_weight_scale",
|
||||
torch.zeros(e, i // 128, h, dtype=torch.int32).transpose(1, 2),
|
||||
)
|
||||
# L1 interleaving allocates a full copy. Re-home the loader Parameter on
|
||||
# that storage so the original 2*intermediate*hidden FP8 tensor can be
|
||||
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
|
||||
# The generic linear post-load hook may still repack the serial scales,
|
||||
# but this shared MLP is never called after native fusion is enabled.
|
||||
gate_up.weight.data = transformed_l1[0]
|
||||
transformed_l1 = (gate_up.weight.data, transformed_l1[1])
|
||||
return transformed_l1, transformed_l2
|
||||
if num_shared_experts == 0:
|
||||
return
|
||||
# The shared expert is FP8 with the layout of one routed expert slice.
|
||||
s, fp8 = i * num_shared_experts, torch.float8_e4m3fn
|
||||
register("shared_l1_weight", torch.zeros(2 * s, h, dtype=fp8))
|
||||
register(
|
||||
"shared_l1_weight_scale",
|
||||
torch.zeros(h // 128, 2 * s, dtype=torch.int32).transpose(0, 1),
|
||||
)
|
||||
register("shared_l2_weight", torch.zeros(h, s, dtype=fp8))
|
||||
register(
|
||||
"shared_l2_weight_scale",
|
||||
torch.zeros(s // 128, h, dtype=torch.int32).transpose(0, 1),
|
||||
)
|
||||
|
||||
def load_weight(
|
||||
self,
|
||||
dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
shard_id: str,
|
||||
is_scale: bool,
|
||||
) -> None:
|
||||
if shard_id not in ("w1", "w2", "w3"):
|
||||
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
||||
rows = dst.shape[0] // (1 if shard_id == "w2" else 2)
|
||||
if is_scale:
|
||||
src = self._pack_scale(src, rows, dst.shape[1] * 4)
|
||||
else:
|
||||
src = src.view(dst.dtype)
|
||||
if tuple(src.shape) != (rows, dst.shape[1]):
|
||||
raise ValueError(
|
||||
f"MegaMoE {shard_id} weight shape {tuple(src.shape)} does not "
|
||||
f"match the kernel shard ({rows}, {dst.shape[1]})"
|
||||
)
|
||||
dst, src = self._kernel_layout_shard(dst, src, shard_id, is_scale)
|
||||
dst.copy_(src)
|
||||
|
||||
@staticmethod
|
||||
def _prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
linear: nn.Module,
|
||||
scale: torch.Tensor,
|
||||
mn: int,
|
||||
k: int,
|
||||
prefix: str,
|
||||
) -> torch.Tensor | None:
|
||||
block_size = getattr(linear, "weight_block_size", None)
|
||||
if block_size is None or len(block_size) != 2:
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: "
|
||||
"shared FP8 weight block size is unavailable.",
|
||||
prefix,
|
||||
def _pack_scale(sf: torch.Tensor, rows: int, cols: int) -> torch.Tensor:
|
||||
"""Expand block UE8M0 scales to one per 1x32 and pack four along K per int32."""
|
||||
sf = sf.view(torch.uint8)
|
||||
if rows % sf.shape[0] or cols % sf.shape[1]:
|
||||
raise ValueError(
|
||||
f"Scale shape {tuple(sf.shape)} does not tile ({rows}, {cols})"
|
||||
)
|
||||
return None
|
||||
sf = sf.repeat_interleave(rows // sf.shape[0], 0)
|
||||
sf = sf.repeat_interleave(cols // sf.shape[1], 1)
|
||||
return sf.contiguous().clone().view(torch.int32)
|
||||
|
||||
block_m, block_k = block_size
|
||||
num_k_blocks = (k + block_k - 1) // block_k
|
||||
# The linear hook's DeepGEMM layout packs four k-block bytes per int32.
|
||||
packed = scale.dtype == torch.int32
|
||||
expected_shape = (
|
||||
(mn, (num_k_blocks + 3) // 4)
|
||||
if packed
|
||||
else ((mn + block_m - 1) // block_m, num_k_blocks)
|
||||
)
|
||||
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: "
|
||||
"cannot convert shared scale shape %s with block size %s "
|
||||
"to MegaMoE's 1x32 layout for weight (%d, %d).",
|
||||
prefix,
|
||||
tuple(scale.shape),
|
||||
tuple(block_size),
|
||||
mn,
|
||||
k,
|
||||
@staticmethod
|
||||
def _kernel_layout_shard(
|
||||
dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
shard_id: str,
|
||||
is_scale: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pair a checkpoint shard with its destination in the MegaMoE layout."""
|
||||
k = src.shape[-1]
|
||||
if shard_id == "w2":
|
||||
if not is_scale:
|
||||
return dst, src
|
||||
# UTCCP: scale row 128*b + 32*j + i -> 128*b + 4*i + j.
|
||||
return dst.view(-1, 32, 4, k).transpose(1, 2), src.reshape(-1, 4, 32, k)
|
||||
half = 0 if shard_id == "w1" else 1
|
||||
if not is_scale:
|
||||
# Interleave gate/up by 8 rows: r -> (r // 8) * 16 + 8 * half + r % 8.
|
||||
return dst.view(-1, 2, 8, k)[:, half], src.reshape(-1, 8, k)
|
||||
# Interleave + UTCCP: 64b + 16j + 8q + m -> 128b + 64q + 32half + 4m + j.
|
||||
dst = dst.view(-1, 2, 2, 8, 4, k)[:, :, half].permute(0, 3, 1, 2, 4)
|
||||
return dst, src.reshape(-1, 4, 2, 8, k)
|
||||
|
||||
def supports_shared_experts(self, shared_experts: nn.Module) -> bool:
|
||||
for linear in (shared_experts.gate_up_proj, shared_experts.down_proj):
|
||||
weight = linear.weight
|
||||
scale = getattr(linear, "weight_scale", None)
|
||||
if scale is None:
|
||||
scale = getattr(linear, "weight_scale_inv", None)
|
||||
if not self._tiles_1x32(weight, scale):
|
||||
logger.warning_once(
|
||||
"Disabling native MegaMoE shared-expert fusion: needs FP8 "
|
||||
"weights with UE8M0 block scales that tile to 1x32; got "
|
||||
"weight %s/%s and scale %s/%s.",
|
||||
tuple(weight.shape),
|
||||
weight.dtype,
|
||||
None if scale is None else tuple(scale.shape),
|
||||
None if scale is None else scale.dtype,
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
try:
|
||||
api = {
|
||||
*signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters,
|
||||
*signature(deep_gemm.fp8_fp4_mega_moe).parameters,
|
||||
}
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
api = set()
|
||||
if not (
|
||||
hasattr(deep_gemm, "get_block_m_for_mega_moe")
|
||||
and {"num_shared_experts", "shared_l1_weights", "shared_l2_weights"} <= api
|
||||
):
|
||||
logger.warning_once(
|
||||
"Disabling native MegaMoE shared-expert fusion because the "
|
||||
"installed DeepGEMM Python API is older than the vLLM "
|
||||
"source. Rebuild the vendored _deep_gemm_C extension to enable it.",
|
||||
)
|
||||
return None
|
||||
return False
|
||||
return True
|
||||
|
||||
if packed:
|
||||
row_scale = scale.flatten().view(torch.uint8).view(mn, -1)[:, :num_k_blocks]
|
||||
else:
|
||||
row_scale = scale.view(torch.uint8).repeat_interleave(block_m, dim=0)[:mn]
|
||||
scale_1x32 = (
|
||||
ue8m0_uint8_to_float(row_scale)
|
||||
.repeat_interleave(block_k // 32, dim=1)[:, : k // 32]
|
||||
.contiguous()
|
||||
@staticmethod
|
||||
def _tiles_1x32(weight: torch.Tensor, scale: torch.Tensor | None) -> bool:
|
||||
if scale is None or weight.dim() != 2 or scale.dim() != 2:
|
||||
return False
|
||||
(rows, cols), (scale_rows, scale_cols) = weight.shape, scale.shape
|
||||
return (
|
||||
weight.dtype == torch.float8_e4m3fn
|
||||
and scale.dtype in (torch.uint8, torch.float8_e8m0fnu)
|
||||
and rows % scale_rows == 0
|
||||
and cols % scale_cols == 0
|
||||
and (cols // scale_cols) % 32 == 0
|
||||
)
|
||||
# The grouped API is used with a singleton dimension to request the
|
||||
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
|
||||
# 2D layout required for a shared expert.
|
||||
return deep_gemm.transform_sf_into_required_layout(
|
||||
scale_1x32.unsqueeze(0),
|
||||
mn,
|
||||
k,
|
||||
(1, 32),
|
||||
1,
|
||||
).squeeze(0)
|
||||
|
||||
def run_mega_moe(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Constant expert substitution for MoNE-pruned checkpoints.
|
||||
|
||||
MoNE (https://arxiv.org/abs/2507.00390) replaces redundant experts with
|
||||
"novices" that return a constant vector. Checkpoints list the replaced experts
|
||||
per decoder layer either in ``config.approximate_experts``, with constants named
|
||||
``<experts prefix>.<expert id>.approx_value``, or in
|
||||
``compression_config.transform_config.expert_substitution``, with explicitly
|
||||
named constants. Retained experts keep their logical checkpoint names and are
|
||||
loaded into compact physical rows.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
|
||||
|
||||
APPROX_VALUE_SUFFIX = ".approx_value"
|
||||
|
||||
|
||||
def decoder_layer_index(path: str) -> int | None:
|
||||
"""Return the only integer component of a module path, if unique."""
|
||||
indices = [int(part) for part in path.split(".") if part.isdigit()]
|
||||
return indices[0] if len(indices) == 1 else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExpertSubstitutionSpec:
|
||||
# Sorted substituted expert IDs per decoder layer.
|
||||
experts: dict[int, tuple[int, ...]]
|
||||
# Explicitly named constant tensors and the (layer, expert) pairs they
|
||||
# supply. Without an entry, constants use the ``approx_value`` naming.
|
||||
value_names: dict[str, tuple[tuple[int, int], ...]] = field(default_factory=dict)
|
||||
|
||||
|
||||
# The only router semantics that the constant-v1 format supports.
|
||||
_ROUTER_SEMANTICS = {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
}
|
||||
|
||||
|
||||
def _expert_ids(layer_idx: int, expert_ids: Any) -> tuple[int, ...]:
|
||||
ids = tuple(sorted(int(expert_id) for expert_id in expert_ids or ()))
|
||||
if len(set(ids)) != len(ids) or any(expert_id < 0 for expert_id in ids):
|
||||
raise ValueError(
|
||||
f"substituted experts of layer {layer_idx} must be unique, "
|
||||
f"non-negative expert IDs, got {list(ids)}"
|
||||
)
|
||||
return ids
|
||||
|
||||
|
||||
def _parse_approximate_experts(raw: Any) -> ExpertSubstitutionSpec:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise ValueError("approximate_experts must map layer indices to expert IDs")
|
||||
experts: dict[int, tuple[int, ...]] = {}
|
||||
for layer, expert_ids in raw.items():
|
||||
try:
|
||||
layer_idx = int(layer)
|
||||
experts[layer_idx] = _expert_ids(layer_idx, expert_ids)
|
||||
except (TypeError, ValueError) as e:
|
||||
raise ValueError(
|
||||
f"approximate_experts has an invalid entry for layer {layer!r}"
|
||||
) from e
|
||||
return ExpertSubstitutionSpec({k: v for k, v in experts.items() if v})
|
||||
|
||||
|
||||
def _parse_expert_substitution(raw: Any) -> ExpertSubstitutionSpec:
|
||||
experts: dict[int, tuple[int, ...]] = {}
|
||||
value_names: dict[str, list[tuple[int, int]]] = {}
|
||||
try:
|
||||
if raw["version"] != 1 or raw["router_semantics"] != _ROUTER_SEMANTICS:
|
||||
raise ValueError("only version 1 with unchanged routing is supported")
|
||||
for path, target in raw["targets"].items():
|
||||
layer_idx = decoder_layer_index(path)
|
||||
if layer_idx is None or layer_idx in experts:
|
||||
raise ValueError(f"target {path!r} must name one decoder layer")
|
||||
if target["weight_layout"] != "compact_retained_experts":
|
||||
raise ValueError(f"unsupported weight layout for {path!r}")
|
||||
for expert_id, replacement in target["replacements"].items():
|
||||
if replacement["format"] != "constant-v1":
|
||||
raise ValueError(f"unsupported replacement format for {path!r}")
|
||||
value_names.setdefault(replacement["tensors"]["value"], []).append(
|
||||
(layer_idx, int(expert_id))
|
||||
)
|
||||
experts[layer_idx] = _expert_ids(layer_idx, target["replacements"])
|
||||
except (AttributeError, KeyError, TypeError, ValueError) as e:
|
||||
raise ValueError(f"invalid expert_substitution metadata: {e}") from e
|
||||
return ExpertSubstitutionSpec(
|
||||
experts, {name: tuple(pairs) for name, pairs in value_names.items()}
|
||||
)
|
||||
|
||||
|
||||
def get_expert_substitution_spec(model_config: Any) -> ExpertSubstitutionSpec | None:
|
||||
"""Parse the substituted experts declared in a model's HF config."""
|
||||
hf_configs = [
|
||||
getattr(model_config, name, None) for name in ("hf_text_config", "hf_config")
|
||||
]
|
||||
approximate = None
|
||||
for config in hf_configs:
|
||||
approximate = getattr(config, "approximate_experts", None)
|
||||
if approximate is not None:
|
||||
break
|
||||
compression_config = getattr(hf_configs[1], "compression_config", None)
|
||||
transform_config = (
|
||||
compression_config.get("transform_config")
|
||||
if isinstance(compression_config, Mapping)
|
||||
else None
|
||||
)
|
||||
declared = (
|
||||
transform_config.get("expert_substitution")
|
||||
if isinstance(transform_config, Mapping)
|
||||
else None
|
||||
)
|
||||
if approximate is not None and declared is not None:
|
||||
raise ValueError(
|
||||
"config declares both approximate_experts and "
|
||||
"compression_config.transform_config.expert_substitution"
|
||||
)
|
||||
if approximate is not None:
|
||||
spec = _parse_approximate_experts(approximate)
|
||||
elif declared is not None:
|
||||
spec = _parse_expert_substitution(declared)
|
||||
else:
|
||||
return None
|
||||
return spec if spec.experts else None
|
||||
|
||||
|
||||
class ConstantExpertSubstitution(nn.Module):
|
||||
"""Constant outputs for the substituted experts of one MoE layer.
|
||||
|
||||
Router outputs keep logical expert IDs. ``transform_routes`` maps retained
|
||||
experts to compact physical rows and turns substituted routes into
|
||||
zero-weight routes to physical expert 0, so any decomposed MoE backend can
|
||||
run them unchanged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer_idx: int,
|
||||
num_logical_experts: int,
|
||||
substituted_expert_ids: tuple[int, ...],
|
||||
hidden_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
substituted = set(substituted_expert_ids)
|
||||
if not substituted <= set(range(num_logical_experts)):
|
||||
raise ValueError(
|
||||
f"approximate_experts for layer {layer_idx} must be within "
|
||||
f"[0, {num_logical_experts}), got {sorted(substituted)}"
|
||||
)
|
||||
if len(substituted) == num_logical_experts:
|
||||
raise ValueError(
|
||||
f"approximate_experts for layer {layer_idx} must retain at least "
|
||||
"one expert"
|
||||
)
|
||||
self.layer_idx = layer_idx
|
||||
self.substituted_expert_ids = tuple(sorted(substituted))
|
||||
retained = [i for i in range(num_logical_experts) if i not in substituted]
|
||||
self.num_compute_experts = len(retained)
|
||||
|
||||
# Retained experts map to their physical row r >= 0, substituted
|
||||
# experts to -1 - (their row in `values`).
|
||||
routes = [0] * num_logical_experts
|
||||
for row, expert_id in enumerate(retained):
|
||||
routes[expert_id] = row
|
||||
for row, expert_id in enumerate(self.substituted_expert_ids):
|
||||
routes[expert_id] = -1 - row
|
||||
self._routes = tuple(routes)
|
||||
self.register_buffer(
|
||||
"expert_substitution_routes",
|
||||
torch.tensor(routes, dtype=torch.int32),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
# NaN marks values that the checkpoint has not provided yet.
|
||||
self.values = nn.Parameter(
|
||||
torch.full((len(substituted), hidden_size), torch.nan, dtype=params_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.values.weight_loader = self.weight_loader
|
||||
|
||||
@property
|
||||
def num_logical_experts(self) -> int:
|
||||
return len(self._routes)
|
||||
|
||||
def physical_expert_id(self, expert_id: int) -> int:
|
||||
"""Physical row of a retained expert, or -1 for a substituted one."""
|
||||
return max(self._routes[expert_id], -1)
|
||||
|
||||
def weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, expert_id: int
|
||||
) -> None:
|
||||
if not 0 <= expert_id < self.num_logical_experts or (
|
||||
self._routes[expert_id] >= 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"expert {expert_id} of layer {self.layer_idx} is not listed in "
|
||||
"approximate_experts"
|
||||
)
|
||||
row = param.data[-1 - self._routes[expert_id]]
|
||||
if loaded_weight.shape != row.shape:
|
||||
raise ValueError(
|
||||
f"approx_value for expert {expert_id} of layer {self.layer_idx} has "
|
||||
f"shape {tuple(loaded_weight.shape)}, expected {tuple(row.shape)}"
|
||||
)
|
||||
row.copy_(loaded_weight)
|
||||
|
||||
def validate_loaded_values(self, prefix: str) -> None:
|
||||
missing_rows = torch.isnan(self.values).any(dim=1).nonzero().flatten().tolist()
|
||||
if missing_rows:
|
||||
missing = [self.substituted_expert_ids[row] for row in missing_rows]
|
||||
raise ValueError(
|
||||
f"{prefix} is missing approx_value for {len(missing)} "
|
||||
f"expert(s), first missing expert IDs: {missing[:8]}"
|
||||
)
|
||||
|
||||
def transform_routes(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Remap routes in place and return the constant experts' output."""
|
||||
valid = (topk_ids >= 0) & (topk_ids < self.num_logical_experts)
|
||||
routes = self.expert_substitution_routes[
|
||||
torch.where(valid, topk_ids, torch.zeros_like(topk_ids)).long()
|
||||
].long()
|
||||
substituted = valid & (routes < 0)
|
||||
|
||||
# Sum each token's router weights per substituted expert in FP32; other
|
||||
# routes land in a trailing column that is dropped.
|
||||
num_values, value_size = self.values.shape
|
||||
scales = topk_weights.new_zeros(
|
||||
(topk_ids.shape[0], num_values + 1), dtype=torch.float32
|
||||
)
|
||||
scales.scatter_add_(
|
||||
1,
|
||||
torch.where(substituted, -1 - routes, num_values),
|
||||
topk_weights.float(),
|
||||
)
|
||||
output = torch.zeros_like(hidden_states)
|
||||
output[..., :value_size] = scales[:, :num_values] @ self.values.float()
|
||||
|
||||
topk_weights.masked_fill_(~valid | substituted, 0.0)
|
||||
topk_ids.copy_(routes.clamp_min(0).to(topk_ids.dtype))
|
||||
return output
|
||||
|
||||
|
||||
def make_expert_substitution(
|
||||
model_config: Any,
|
||||
prefix: str,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
params_dtype: torch.dtype | None,
|
||||
) -> ConstantExpertSubstitution | None:
|
||||
spec = get_expert_substitution_spec(model_config)
|
||||
if spec is None:
|
||||
return None
|
||||
layer_idx = decoder_layer_index(prefix)
|
||||
if layer_idx is None:
|
||||
raise ValueError(
|
||||
f"cannot match MoE layer {prefix!r} to substituted experts: expected "
|
||||
"exactly one layer index in its prefix"
|
||||
)
|
||||
expert_ids = spec.experts.get(layer_idx)
|
||||
if expert_ids is None:
|
||||
return None
|
||||
return ConstantExpertSubstitution(
|
||||
layer_idx,
|
||||
num_experts,
|
||||
expert_ids,
|
||||
hidden_size,
|
||||
params_dtype or torch.get_default_dtype(),
|
||||
)
|
||||
|
||||
|
||||
class SubstitutedRoutedExperts(RoutedExperts):
|
||||
"""Routed experts that hold weights only for retained experts."""
|
||||
|
||||
def __init__(
|
||||
self, *args: Any, expert_substitution: ConstantExpertSubstitution, **kwargs: Any
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.expert_substitution = expert_substitution
|
||||
moe_config = self.moe_config
|
||||
parallel_config = moe_config.moe_parallel_config
|
||||
unsupported = [
|
||||
name
|
||||
for name, enabled in (
|
||||
(
|
||||
"quantized MoE weights",
|
||||
not isinstance(self.quant_method, UnquantizedFusedMoEMethod),
|
||||
),
|
||||
("monolithic MoE backends", self.quant_method.is_monolithic),
|
||||
("MoE LoRA", moe_config.is_lora_enabled),
|
||||
(
|
||||
"EPLB",
|
||||
parallel_config.enable_eplb
|
||||
or moe_config.num_experts
|
||||
!= expert_substitution.num_compute_experts,
|
||||
),
|
||||
(
|
||||
"fused shared experts",
|
||||
self.expert_map_manager.num_fused_shared_experts > 0,
|
||||
),
|
||||
("expert parallelism", parallel_config.use_ep),
|
||||
("data parallelism", parallel_config.dp_size > 1),
|
||||
("prefill context parallelism", parallel_config.pcp_size > 1),
|
||||
("sequence parallelism", parallel_config.is_sequence_parallel),
|
||||
("router weights on expert inputs", self.apply_router_weight_on_input),
|
||||
)
|
||||
if enabled
|
||||
]
|
||||
if unsupported:
|
||||
raise NotImplementedError(
|
||||
"expert substitution does not support: " + ", ".join(unsupported)
|
||||
)
|
||||
|
||||
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
|
||||
# Checkpoints name retained experts by their logical ID.
|
||||
physical_id = self.expert_substitution.physical_expert_id(expert_id)
|
||||
if physical_id < 0:
|
||||
return -1
|
||||
return super()._map_global_expert_id_to_local_expert_id(physical_id)
|
||||
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name: str | None = None,
|
||||
ckpt_down_proj_name: str | None = None,
|
||||
ckpt_up_proj_name: str | None = None,
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
# Fused checkpoint tensors would cover every logical expert at once.
|
||||
return self.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name or self.ckpt_down_proj_name,
|
||||
ckpt_up_proj_name or self.ckpt_up_proj_name,
|
||||
num_experts=self.expert_substitution.num_logical_experts,
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
)
|
||||
|
||||
def forward_modular(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: "SharedExperts | None" = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
constant_output = self.expert_substitution.transform_routes(
|
||||
x, topk_weights, topk_ids
|
||||
)
|
||||
output = super().forward_modular(
|
||||
x, topk_weights, topk_ids, shared_experts, shared_experts_input
|
||||
)
|
||||
if isinstance(output, UnfinalizedMoEOutput):
|
||||
raise NotImplementedError(
|
||||
"expert substitution does not support deferred MoE finalize"
|
||||
)
|
||||
# The constant output is replicated across TP ranks, whose partial
|
||||
# outputs are all-reduced later, so add it on one rank only.
|
||||
if self.moe_config.tp_rank == 0:
|
||||
output.add_(constant_output[..., : output.shape[-1]])
|
||||
return output
|
||||
@@ -51,6 +51,10 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
def _supports_parallel_config(moe_parallel_config) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_batch_invariance() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
|
||||
@@ -973,34 +973,6 @@ class CPUExpertsInt4(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# INT8 W8A8 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def prepare_int8_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Prepack INT8 MoE weights for the current CPU architecture."""
|
||||
# SMMLA packing for AArch64
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
return (
|
||||
cpu_prepack_moe_weight_int8(w13, "neon"),
|
||||
cpu_prepack_moe_weight_int8(w2, "neon"),
|
||||
)
|
||||
# VSX packing for POWER
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC:
|
||||
return (
|
||||
cpu_prepack_moe_weight_int8(w13, "vsx"),
|
||||
cpu_prepack_moe_weight_int8(w2, "vsx"),
|
||||
)
|
||||
# VNNI packing for x86
|
||||
packed_w13 = torch.ops._C.convert_weight_packed(w13)
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
return packed_w13, packed_w2
|
||||
|
||||
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
|
||||
modular MoE experts."""
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
remap_topk_to_local,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -412,10 +413,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# CutlassExpertsFp8 does not support expert map, which is
|
||||
# needed for STANDARD activation format kernels in DP/EP mode.
|
||||
# Note that the BATCHED activation format does not use
|
||||
# the expert map for identifying experts.
|
||||
# EP is supported only with allgather_reducescatter (expert_map remapping).
|
||||
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
|
||||
# post-dispatch layouts for Standard-format grouped GEMM.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
@@ -491,6 +491,15 @@ FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
|
||||
def _prepare_topk_ids_for_cutlass_fp4_moe(
|
||||
topk_ids: torch.Tensor,
|
||||
expert_map: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if expert_map is None:
|
||||
return topk_ids
|
||||
return remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
|
||||
|
||||
|
||||
def run_cutlass_moe_fp4(
|
||||
output: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
@@ -513,6 +522,7 @@ def run_cutlass_moe_fp4(
|
||||
e: int,
|
||||
device: torch.device,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
*,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> None:
|
||||
@@ -543,6 +553,8 @@ def run_cutlass_moe_fp4(
|
||||
|
||||
assumes that topk < k < n to satisfy - up/down projection expectations.
|
||||
"""
|
||||
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
|
||||
|
||||
is_gated = activation.is_gated
|
||||
# For gated activations (e.g. SiLU), w1 output is 2*n (gate + up).
|
||||
# For non-gated activations (e.g. SiLU_NO_MUL), w1 output is n (up only).
|
||||
@@ -729,9 +741,14 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# CutlassExpertsFp4 does not support expert map, which is
|
||||
# needed for STANDARD activation format kernels in EP mode.
|
||||
return moe_parallel_config.ep_size == 1
|
||||
# EP is supported only with allgather_reducescatter (expert_map remapping).
|
||||
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
|
||||
# post-dispatch layouts for Standard-format grouped GEMM.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_batch_invariance() -> bool:
|
||||
@@ -806,6 +823,7 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
|
||||
e=e,
|
||||
device=hidden_states.device,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
expert_map=expert_map,
|
||||
activation_config=self.activation_config,
|
||||
)
|
||||
|
||||
@@ -828,10 +846,13 @@ def run_cutlass_moe_mxfp4(
|
||||
e: int,
|
||||
device: torch.device,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
*,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> None:
|
||||
"""MXFP4 x MXFP4 MoE implementation using CUTLASS grouped GEMM."""
|
||||
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
|
||||
|
||||
is_gated = activation.is_gated
|
||||
w1_n = n * 2 if is_gated else n
|
||||
|
||||
@@ -1036,7 +1057,14 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return moe_parallel_config.ep_size == 1
|
||||
# EP is supported only with allgather_reducescatter (expert_map remapping).
|
||||
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
|
||||
# post-dispatch layouts for Standard-format grouped GEMM.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -1103,6 +1131,7 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
|
||||
e=e,
|
||||
device=hidden_states.device,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
expert_map=expert_map,
|
||||
activation_config=self.activation_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -450,6 +450,8 @@ def invoke_moe_batched_triton_kernel(
|
||||
BLOCK_M = config["BLOCK_SIZE_M"]
|
||||
BLOCK_N = config["BLOCK_SIZE_N"]
|
||||
BLOCK_K = config["BLOCK_SIZE_K"]
|
||||
if block_shape is not None:
|
||||
BLOCK_K = min(BLOCK_K, min(block_shape[0], block_shape[1]))
|
||||
|
||||
grid = (
|
||||
expert_num_tokens.size(0),
|
||||
|
||||
@@ -22,6 +22,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
|
||||
fused_globalize_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
@@ -63,6 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
@@ -137,7 +141,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
"w13": humming_quant_config.w1_humming_config,
|
||||
"w2": humming_quant_config.w2_humming_config,
|
||||
}
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
|
||||
self.locks = create_humming_locks(moe_config.device)
|
||||
self.num_experts = moe_config.num_local_experts
|
||||
self.global_num_experts = moe_config.num_experts
|
||||
self.quant_config = humming_quant_config
|
||||
@@ -214,12 +218,20 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
if scatter_idx is not None:
|
||||
layout = "scatter"
|
||||
|
||||
hadamard_block_size = getattr(quant_config, f"{prefix}_hadamard_block_size")
|
||||
has_input_processing = (
|
||||
mode.should_quantize
|
||||
or (hadamard_block_size is not None and hadamard_block_size > 1)
|
||||
or activation is not None
|
||||
or scatter_idx is not None
|
||||
or num_valid_tokens is not None
|
||||
)
|
||||
inputs, group_scales, token_scales = may_process_input(
|
||||
config,
|
||||
inputs=inputs,
|
||||
outputs=quanted_input,
|
||||
outputs=quanted_input if has_input_processing else None,
|
||||
token_scales=scale2 if mode.has_secondary_scale else scale,
|
||||
hadamard_block_size=getattr(quant_config, f"{prefix}_hadamard_block_size"),
|
||||
hadamard_block_size=hadamard_block_size,
|
||||
layout=layout,
|
||||
expert_tokens=expert_tokens if layout == "grouped_mask" else None,
|
||||
scatter_idx=scatter_idx,
|
||||
@@ -702,16 +714,45 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
num_sms = num_compute_units(topk_ids.get_device())
|
||||
use_scatter = topk_ids.numel() > 2 * num_sms
|
||||
|
||||
alignment = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
block_size=moe_block_size,
|
||||
num_experts=self.global_num_experts,
|
||||
expert_map=expert_map,
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=use_scatter,
|
||||
# When DeepEP-v2 deferred globalization, fuse globalize + align + sort
|
||||
# into one launch (topk_ids globalized in place); else align only.
|
||||
rank_expert_offset = (
|
||||
expert_tokens_meta.rank_expert_offset
|
||||
if expert_tokens_meta is not None
|
||||
else None
|
||||
)
|
||||
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
|
||||
scatter_idx = alignment[3] if len(alignment) == 4 else None
|
||||
psum = (
|
||||
expert_tokens_meta.psum_recv_per_rank
|
||||
if expert_tokens_meta is not None
|
||||
else None
|
||||
)
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and rank_expert_offset is not None
|
||||
and psum is not None
|
||||
):
|
||||
alignment = fused_globalize_align_block_size(
|
||||
recv_topk_idx=topk_ids,
|
||||
psum_recv_per_rank=psum,
|
||||
rank_expert_offset=rank_expert_offset,
|
||||
global_num_experts=self.global_num_experts,
|
||||
local_num_experts=self.num_experts,
|
||||
block_size=moe_block_size,
|
||||
return_scatter_idx=use_scatter,
|
||||
)
|
||||
_, sorted_ids, expert_ids, num_tokens_padded = alignment[:4]
|
||||
scatter_idx = alignment[4] if use_scatter else None
|
||||
else:
|
||||
alignment = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
block_size=moe_block_size,
|
||||
num_experts=self.global_num_experts,
|
||||
expert_map=expert_map,
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=use_scatter,
|
||||
)
|
||||
sorted_ids, expert_ids, num_tokens_padded = alignment[:3]
|
||||
scatter_idx = alignment[3] if len(alignment) == 4 else None
|
||||
|
||||
moe_common_kwargs = {
|
||||
"sorted_ids": sorted_ids,
|
||||
|
||||
@@ -21,7 +21,10 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
remap_topk_to_local,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
@@ -844,6 +847,20 @@ def triton_kernel_fused_experts(
|
||||
return output_tensor
|
||||
|
||||
|
||||
def _tk38_routing_data(
|
||||
col_sum: torch.Tensor,
|
||||
order: torch.Tensor,
|
||||
flat_w: torch.Tensor,
|
||||
topk: int,
|
||||
) -> tuple["RoutingData", torch.Tensor, torch.Tensor]:
|
||||
"""Build 3.8 routing from per-expert counts and the stable expert order."""
|
||||
nrows = order.shape[0]
|
||||
ragged = make_ragged_tensor_metadata(col_sum, nrows)
|
||||
gather_tok = (order // topk).to(torch.int32)
|
||||
routing = RoutingData(ragged, flat_w[order], gather_tok, topk, nrows)
|
||||
return routing, gather_tok, gather_tok
|
||||
|
||||
|
||||
def make_routing_data(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
@@ -851,21 +868,18 @@ def make_routing_data(
|
||||
) -> tuple["RoutingData", torch.Tensor, torch.Tensor]:
|
||||
if triton_kernels_version == "3.8":
|
||||
_, topk = topk_ids.shape
|
||||
flat_e = topk_ids.reshape(-1)
|
||||
valid = flat_e >= 0
|
||||
flat_e = flat_e.to(torch.int64).clamp(min=0)
|
||||
flat_w = topk_weights.reshape(-1).to(torch.float32) * valid.to(torch.float32)
|
||||
nrows = flat_e.shape[0]
|
||||
order = torch.argsort(flat_e, stable=True)
|
||||
col_sum = torch.zeros(
|
||||
num_local_experts, dtype=torch.int32, device=flat_e.device
|
||||
flat_ids = topk_ids.reshape(-1)
|
||||
# -1 (non-local) slots go to expert 0 with zero weight.
|
||||
keys = flat_ids.clamp(min=0)
|
||||
sorted_keys, order = torch.sort(keys, stable=True)
|
||||
experts = torch.arange(
|
||||
num_local_experts + 1, dtype=keys.dtype, device=keys.device
|
||||
)
|
||||
col_sum.scatter_add_(0, flat_e, torch.ones_like(flat_e, dtype=torch.int32))
|
||||
ragged = make_ragged_tensor_metadata(col_sum, nrows)
|
||||
gather_tok = (order // topk).to(torch.int32)
|
||||
gammas = flat_w[order]
|
||||
routing = RoutingData(ragged, gammas, gather_tok, topk, nrows)
|
||||
return routing, gather_tok, gather_tok
|
||||
edges = torch.searchsorted(sorted_keys, experts, out_int32=True)
|
||||
col_sum = edges[1:] - edges[:-1]
|
||||
valid = (flat_ids >= 0).to(torch.float32)
|
||||
flat_w = topk_weights.reshape(-1).to(torch.float32) * valid
|
||||
return _tk38_routing_data(col_sum, order, flat_w, topk)
|
||||
|
||||
topk_ids = topk_ids.to(torch.int16)
|
||||
topk_weights = topk_weights.to(torch.bfloat16)
|
||||
@@ -933,8 +947,14 @@ def routing_data_from_sparse_topk(
|
||||
remapping) that work is redundant.
|
||||
"""
|
||||
if triton_kernels_version == "3.8":
|
||||
# 3.8 routing is rebuilt from raw ids; no SparseMatrix reuse.
|
||||
return make_routing_data(sparse_topk.indx, sparse_topk.vals, n_expts)
|
||||
# topk ids have no -1 slots, so the topk metadata matches make_routing_data.
|
||||
mask_metadata = sparse_topk.mask_metadata
|
||||
return _tk38_routing_data(
|
||||
mask_metadata.col_sum,
|
||||
mask_metadata.col_sorted_indx,
|
||||
sparse_topk.vals.reshape(-1).to(torch.float32),
|
||||
sparse_topk.indx.shape[1],
|
||||
)
|
||||
|
||||
dispatch_indx = sparse_topk.mask_metadata.row_sorted_indx
|
||||
combine_indx = sparse_topk.mask_metadata.col_sorted_indx
|
||||
@@ -994,49 +1014,6 @@ def masked_moe_sum(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _remap_topk_to_local_kernel(
|
||||
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
|
||||
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
|
||||
out_ptr, # [n] int64 local expert IDs (-1 for invalid/non-local)
|
||||
n_elements,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < n_elements
|
||||
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
|
||||
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
|
||||
# rows don't read OOB, then select -1 for them. Matches
|
||||
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
|
||||
# plain expert_map[-1] would wrap to a valid local id and misroute).
|
||||
valid = tid >= 0
|
||||
idx = tl.where(valid, tid, 0)
|
||||
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
|
||||
out = tl.where(valid, local.to(tl.int64), -1)
|
||||
tl.store(out_ptr + offs, out, mask=mask)
|
||||
|
||||
|
||||
def remap_topk_to_local(
|
||||
topk_ids: torch.Tensor, expert_map: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Fused global->local expert-id mapping over a topk_ids tensor, preserving -1.
|
||||
|
||||
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
|
||||
with one kernel. Returns a NEW int64 tensor -- the caller keeps the original
|
||||
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
|
||||
|
||||
(Distinct from ``deep_gemm_utils.apply_expert_map``, which is a scalar
|
||||
``@triton.jit`` device helper called from within other kernels.)
|
||||
"""
|
||||
out = torch.empty_like(topk_ids, dtype=torch.int64)
|
||||
n = topk_ids.numel()
|
||||
BLOCK = 1024
|
||||
grid = (triton.cdiv(n, BLOCK),)
|
||||
_remap_topk_to_local_kernel[grid](topk_ids, expert_map, out, n, BLOCK=BLOCK)
|
||||
return out
|
||||
|
||||
|
||||
class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.experts.lora_context import MoELoRAContext
|
||||
|
||||
|
||||
@@ -22,6 +24,7 @@ class LoRAExpertsMixin:
|
||||
state is on lora_context or passed as arguments.
|
||||
"""
|
||||
|
||||
moe_config: FusedMoEConfig
|
||||
_lora_context: MoELoRAContext | None = None
|
||||
|
||||
def set_lora_context(self, ctx: MoELoRAContext) -> None:
|
||||
@@ -70,8 +73,13 @@ class LoRAExpertsMixin:
|
||||
# concat swap afterwards.
|
||||
w13_lora_a_stacked = w13_lora_a_stacked[::-1]
|
||||
w13_lora_b_stacked = w13_lora_b_stacked[::-1]
|
||||
return lora_context.punica_wrapper.add_lora_w13(
|
||||
y,
|
||||
interleave_w13 = (
|
||||
lora_context.w13_num_slices == 2
|
||||
and self.moe_config.activation == MoEActivation.SWIGLUOAI
|
||||
)
|
||||
lora_output = torch.zeros_like(y) if interleave_w13 else y
|
||||
result = lora_context.punica_wrapper.add_lora_w13(
|
||||
lora_output,
|
||||
x,
|
||||
w13_lora_a_stacked,
|
||||
w13_lora_b_stacked,
|
||||
@@ -89,9 +97,18 @@ class LoRAExpertsMixin:
|
||||
lora_context.w13_num_slices,
|
||||
lora_context.fully_sharded,
|
||||
lora_context.use_tuned_config,
|
||||
add_inputs=add_inputs,
|
||||
add_inputs=add_inputs or interleave_w13,
|
||||
token_lora_mapping=lora_context.local_token_lora_mapping,
|
||||
)
|
||||
if interleave_w13:
|
||||
# Map gate/up slices onto even/odd output channels without a copy.
|
||||
gate_up_delta = lora_output.unflatten(-1, (2, -1))
|
||||
gate_up_output = y.unflatten(-1, (-1, 2)).transpose(-1, -2)
|
||||
if add_inputs:
|
||||
gate_up_output.add_(gate_up_delta)
|
||||
else:
|
||||
gate_up_output.copy_(gate_up_delta)
|
||||
return result
|
||||
|
||||
def apply_w2_lora(
|
||||
self,
|
||||
|
||||
@@ -218,7 +218,7 @@ class OCP_MXQuantizationEmulationTritonExperts(TritonExperts):
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
a2_scale=a2_scale,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
|
||||
@@ -352,7 +352,11 @@ def rocm_aiter_fused_experts(
|
||||
intermediate_pad = 0
|
||||
assert moe_config.hidden_dim_unpadded is not None
|
||||
assert moe_config.intermediate_size_per_partition_unpadded is not None
|
||||
hidden_pad = hidden_states.shape[1] - moe_config.hidden_dim_unpadded
|
||||
hidden_size = hidden_states.shape[1]
|
||||
if hidden_states.dtype == torch.float4_e2m1fn_x2:
|
||||
# Pre-quantized MXFP4 from MoRI dispatch, packed two per byte.
|
||||
hidden_size *= 2
|
||||
hidden_pad = hidden_size - moe_config.hidden_dim_unpadded
|
||||
intermediate_pad = (
|
||||
(
|
||||
moe_config.intermediate_size_per_partition
|
||||
@@ -448,7 +452,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# When paired with MoRI, the prepare/finalize handles FP8
|
||||
# When paired with MoRI, the prepare/finalize handles FP8 or MXFP4
|
||||
# quantization during dispatch to reduce network traffic,
|
||||
# so we should not defer input quantization.
|
||||
# Otherwise, AITER fused MoE kernels handle input quantization
|
||||
@@ -482,29 +486,6 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(None, None),
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
(kFp8StaticChannelSym, kFp8DynamicTokenSym),
|
||||
(kMxfp4Static, None),
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
]
|
||||
if (weight_key, activation_key) not in SUPPORTED_W_A:
|
||||
return False
|
||||
if weight_key == kMxfp4Static:
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
|
||||
if not on_gfx950() or on_gfx1250():
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [
|
||||
@@ -539,7 +520,12 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
# Workspaces are managed internally by AITER.
|
||||
workspace1 = (0,)
|
||||
workspace2 = (0,)
|
||||
output = (M, K)
|
||||
# MoRI dispatches MXFP4 activations packed two per byte, so K is
|
||||
# half the hidden size there.
|
||||
packed_fp4_input = (
|
||||
self.moe_config.use_mori_kernels and self.quant_config.use_mxfp4_w4a4
|
||||
)
|
||||
output = (M, K * 2 if packed_fp4_input else K)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def apply(
|
||||
@@ -597,3 +583,77 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
output.set_(result)
|
||||
else:
|
||||
output.copy_(result)
|
||||
|
||||
|
||||
class AiterUnquantizedExperts(AiterExperts):
|
||||
"""Unquantized (bf16/fp16) weights and activations."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [(None, None)]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterFp8BlockExperts(AiterExperts):
|
||||
"""128x128 block-quantized weight scale, 128-grouped dynamic
|
||||
activation scale."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [(kFp8Static128BlockSym, kFp8Dynamic128Sym)]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterFp8TensorExperts(AiterExperts):
|
||||
"""Per-tensor static weight scale, either per-tensor static or
|
||||
per-tensor dynamic activation scale."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterFp8ChannelExperts(AiterExperts):
|
||||
"""Per-output-channel static weight scale, per-token dynamic
|
||||
activation scale."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [(kFp8StaticChannelSym, kFp8DynamicTokenSym)]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
class AiterMxfp4Experts(AiterExperts):
|
||||
"""MXFP4-quantized weights, with either unquantized (bf16/fp16) or
|
||||
MXFP4-quantized dynamic activations. gfx950-only (not gfx1250)."""
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp4Static, None),
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
]
|
||||
if (weight_key, activation_key) not in SUPPORTED_W_A:
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
|
||||
return on_gfx950() and not on_gfx1250()
|
||||
|
||||
@@ -315,13 +315,14 @@ class FlashInferMoeEp:
|
||||
w2_scale=weights.w2_scale,
|
||||
)
|
||||
if spec.kernel == "cutedsl":
|
||||
# In-kernel fc2 reduce keeps its default (off); FlashInfer 0.7.1rc1
|
||||
# renamed in_kernel_fc2_reduce to enable_in_kernel_fc2_reduce.
|
||||
megakernel = api.Nvfp4CutedslMegaMoeConfig(
|
||||
intermediate_size=moe.intermediate_size,
|
||||
top_k=moe.experts_per_token,
|
||||
gate_up_clamp=moe.swiglu_limit,
|
||||
fast_math=True,
|
||||
apply_topk_in_fc1=apply_topk_in_fc1,
|
||||
enable_in_kernel_fc2_reduce=False,
|
||||
combine_dtype="bf16",
|
||||
input_norm_const=epilogue.input_norm_const,
|
||||
fc1_alpha=None,
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
|
||||
def fused_globalize_align_block_size(
|
||||
recv_topk_idx: torch.Tensor,
|
||||
psum_recv_per_rank: torch.Tensor,
|
||||
rank_expert_offset: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
block_size: int,
|
||||
return_scatter_idx: bool = False,
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
"""Fuse the DeepEP-v2 INDEXED decode metadata chain into one launch.
|
||||
|
||||
Replaces ``_globalize_recv_topk_idx`` + ``moe_align_block_size`` +
|
||||
``count_and_sort_expert_tokens``. ``recv_topk_idx`` holds LOCAL expert ids
|
||||
and is globalized in place; valid-token count is read on-device from
|
||||
``psum_recv_per_rank[-1]`` (CUDA-graph safe).
|
||||
|
||||
Returns the globalized ``recv_topk_idx`` and the ``moe_align_block_size``
|
||||
outputs ``(sorted_ids, expert_ids, num_tokens_post_pad)``. When requested,
|
||||
the final output is a masked identity scatter index for valid routes.
|
||||
"""
|
||||
assert recv_topk_idx.dtype == torch.int64
|
||||
_, topk = recv_topk_idx.shape
|
||||
numel = recv_topk_idx.numel()
|
||||
device = recv_topk_idx.device
|
||||
|
||||
max_num_tokens_padded = numel + local_num_experts * (block_size - 1)
|
||||
if numel < local_num_experts:
|
||||
max_num_tokens_padded = min(numel * block_size, max_num_tokens_padded)
|
||||
max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size)
|
||||
|
||||
sorted_ids = torch.empty((max_num_tokens_padded,), dtype=torch.int32, device=device)
|
||||
expert_ids = torch.empty((max_num_m_blocks,), dtype=torch.int32, device=device)
|
||||
num_tokens_post_pad = torch.empty((1,), dtype=torch.int32, device=device)
|
||||
scatter_idx = (
|
||||
torch.empty((numel, 1), dtype=torch.int32, device=device)
|
||||
if return_scatter_idx
|
||||
else None
|
||||
)
|
||||
|
||||
ops.fused_globalize_align_block_size(
|
||||
recv_topk_idx,
|
||||
psum_recv_per_rank,
|
||||
rank_expert_offset,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
block_size,
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
scatter_idx,
|
||||
)
|
||||
|
||||
outputs = (recv_topk_idx, sorted_ids, expert_ids, num_tokens_post_pad)
|
||||
return (*outputs, scatter_idx) if scatter_idx is not None else outputs
|
||||
@@ -23,6 +23,10 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
|
||||
ExpertMapManager,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
SubstitutedRoutedExperts,
|
||||
make_expert_substitution,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
|
||||
FusedMoERouter,
|
||||
@@ -253,6 +257,12 @@ def FusedMoEFactory(
|
||||
fuse_shared_experts,
|
||||
)
|
||||
)
|
||||
expert_substitution = make_expert_substitution(
|
||||
vllm_config.model_config, layer_name, num_experts, hidden_size, params_dtype
|
||||
)
|
||||
if expert_substitution is not None:
|
||||
# Route over all logical experts but allocate only the retained ones.
|
||||
global_num_experts -= len(expert_substitution.substituted_expert_ids)
|
||||
|
||||
# Initialize EPLB manager (or None?)
|
||||
eplb_state: EplbLayerState | None = None
|
||||
@@ -367,6 +377,7 @@ def FusedMoEFactory(
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=router.routing_method_type, # Not ideal
|
||||
has_hash_routing=hash_indices_table is not None,
|
||||
require_decomposed_backend=expert_substitution is not None,
|
||||
swiglu_limit=swiglu_limit,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
@@ -380,7 +391,20 @@ def FusedMoEFactory(
|
||||
|
||||
# Create RoutedExperts instance BEFORE create_weights()
|
||||
# This will hold all expert weight parameters
|
||||
if routed_experts_cls is None:
|
||||
if expert_substitution is not None:
|
||||
if routed_experts_cls not in (None, RoutedExperts) or (
|
||||
routed_input_transform is not None or routed_output_transform is not None
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"expert substitution requires the default RoutedExperts and no "
|
||||
"routed input or output transforms"
|
||||
)
|
||||
routed_experts_cls = SubstitutedRoutedExperts
|
||||
routed_experts_args = {
|
||||
**(routed_experts_args or {}),
|
||||
"expert_substitution": expert_substitution,
|
||||
}
|
||||
elif routed_experts_cls is None:
|
||||
routed_experts_cls = RoutedExperts
|
||||
|
||||
assert params_dtype is not None
|
||||
|
||||
@@ -103,6 +103,9 @@ class ExpertTokensMetadata:
|
||||
expert_num_tokens: torch.Tensor | None
|
||||
expert_num_tokens_cpu: torch.Tensor | None
|
||||
psum_recv_per_rank: torch.Tensor | None = None
|
||||
# local->global expert-id offset, set when globalization is deferred to the
|
||||
# experts' fused align kernel.
|
||||
rank_expert_offset: int | None = None
|
||||
|
||||
@staticmethod
|
||||
def make_from_list(
|
||||
@@ -255,6 +258,15 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
def supports_dbo(self) -> bool:
|
||||
"""Indicates whether or not this class can run under DBO.
|
||||
|
||||
Defaults to supports_async(). A class whose synchronous prepare and
|
||||
finalize do their own DBO compute/comm handoff can return True here
|
||||
while keeping supports_async() False.
|
||||
"""
|
||||
return self.supports_async()
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
"""Whether ``finalize`` can be skipped for a deferring consumer.
|
||||
|
||||
@@ -569,6 +581,8 @@ class FusedMoEExperts(ABC):
|
||||
)
|
||||
elif moe_config.has_hash_routing and cls.is_monolithic():
|
||||
return False, _make_reason("hash routing")
|
||||
elif moe_config.require_decomposed_backend and cls.is_monolithic():
|
||||
return False, _make_reason("routes adjusted after routing")
|
||||
elif not cls._supports_routing_method(
|
||||
moe_config.routing_method, weight_key, activation_key
|
||||
):
|
||||
@@ -1046,6 +1060,18 @@ class FusedMoEKernelModularImpl:
|
||||
):
|
||||
self.prepare_finalize = prepare_finalize
|
||||
self.fused_experts = fused_experts
|
||||
# Experts that globalize recv_topk_idx themselves let a capable prepare
|
||||
# stage skip its standalone launch.
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingIndexedExperts,
|
||||
)
|
||||
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and isinstance(fused_experts, HummingIndexedExperts)
|
||||
and hasattr(prepare_finalize, "defer_globalize")
|
||||
):
|
||||
prepare_finalize.defer_globalize = True
|
||||
self.shared_experts: SharedExperts | None = None
|
||||
moe_parallel_config = fused_experts.moe_config.moe_parallel_config
|
||||
self.moe_parallel_config = moe_parallel_config
|
||||
@@ -1189,7 +1215,7 @@ class FusedMoEKernelModularImpl:
|
||||
# We shouldn't be running an a2a kernel that doesn't
|
||||
# support async prepare/finalize
|
||||
# TODO(lucas): enable in follow-up
|
||||
assert not dbo_enabled()
|
||||
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
|
||||
|
||||
(
|
||||
a1q,
|
||||
@@ -1377,7 +1403,7 @@ class FusedMoEKernelModularImpl:
|
||||
|
||||
"""
|
||||
if not self.prepare_finalize.supports_async():
|
||||
assert not dbo_enabled()
|
||||
assert not dbo_enabled() or self.prepare_finalize.supports_dbo()
|
||||
|
||||
self.prepare_finalize.finalize(
|
||||
output,
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -40,6 +41,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import should_auto_disable_deep_gemm
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -211,10 +213,12 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == Fp8MoeBackend.AITER:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterFp8BlockExperts, AiterFp8TensorExperts, AiterFp8ChannelExperts]
|
||||
|
||||
elif backend == Fp8MoeBackend.VLLM_CUTLASS:
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_cutlass_moe import (
|
||||
@@ -490,6 +494,13 @@ def select_fp8_moe_backend(
|
||||
return _return_or_raise(
|
||||
backend, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
elif (model_config := get_current_vllm_config().model_config) is not None and (
|
||||
should_auto_disable_deep_gemm(
|
||||
getattr(model_config.hf_text_config, "model_type", None)
|
||||
)
|
||||
):
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.DEEPGEMM)
|
||||
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.BATCHED_DEEPGEMM)
|
||||
|
||||
# Handle explicit AITER FP8 configuration.
|
||||
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
|
||||
|
||||
@@ -284,10 +284,10 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterMxfp4Experts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterMxfp4Experts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16:
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
@@ -305,10 +305,10 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterMxfp4Experts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterMxfp4Experts]
|
||||
|
||||
elif backend == Mxfp4MoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMxFp4
|
||||
@@ -875,13 +875,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
triton_uses_128 = (
|
||||
backend == Mxfp4MoeBackend.TRITON_UNFUSED and get_cdna_version() != 4
|
||||
)
|
||||
aiter_w4a4 = backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4
|
||||
|
||||
alignment = (
|
||||
128 if is_situ_or_silu and (aiter_uses_128 or triton_uses_128) else 256
|
||||
intermediate_alignment = (
|
||||
128
|
||||
if aiter_w4a4 or (is_situ_or_silu and (aiter_uses_128 or triton_uses_128))
|
||||
else 256
|
||||
)
|
||||
# W4A4 stage 1 still requires a 256-aligned hidden dimension.
|
||||
hidden_alignment = 256 if aiter_w4a4 else intermediate_alignment
|
||||
|
||||
intermediate_size = round_up(intermediate_size, alignment)
|
||||
hidden_size = round_up(hidden_size, alignment)
|
||||
intermediate_size = round_up(intermediate_size, intermediate_alignment)
|
||||
hidden_size = round_up(hidden_size, hidden_alignment)
|
||||
elif backend == Mxfp4MoeBackend.CPU:
|
||||
# CPU AMX kernel uses BLOCK_N=32, align to 32
|
||||
intermediate_size = round_up(intermediate_size, 32)
|
||||
@@ -1475,6 +1480,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
_cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None,
|
||||
activation: MoEActivation | None = None,
|
||||
use_separated_a4w4: bool = False,
|
||||
activation_key: QuantKey | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -1536,8 +1542,20 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
# The env override wins; otherwise use the checkpoint's activation
|
||||
# scheme instead of Humming's default BF16 activations.
|
||||
input_schema = None
|
||||
if activation_key is not None and not envs.VLLM_HUMMING_INPUT_QUANT_CONFIG:
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
|
||||
input_schema = quant_key_to_input_schema(activation_key)
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config={"quant_method": "mxfp4"}
|
||||
layer,
|
||||
quant_config={"quant_method": "mxfp4"},
|
||||
input_schema=input_schema,
|
||||
allow_input_schema_fallback=False,
|
||||
)
|
||||
return (
|
||||
layer.w13_weight,
|
||||
@@ -2128,6 +2146,59 @@ def make_mxfp4_moe_quant_config(
|
||||
)
|
||||
|
||||
|
||||
_MXFP4_SCHEME_CANDIDATES: list[tuple[QuantKey, QuantKey | None]] = [
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
(kMxfp4Static, kMxfp8Dynamic),
|
||||
(kMxfp4Static, kFp8Dynamic128Sym),
|
||||
(kMxfp4Static, kFp8StaticTensorSym),
|
||||
(kMxfp4Static, None),
|
||||
]
|
||||
|
||||
|
||||
def _mxfp4_quant_keys_for_experts(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
) -> tuple[QuantKey | None, QuantKey | None]:
|
||||
for weight_key, activation_key in _MXFP4_SCHEME_CANDIDATES:
|
||||
if experts_cls._supports_quant_scheme(weight_key, activation_key):
|
||||
return weight_key, activation_key
|
||||
return kMxfp4Static, _backend_activation_key(mxfp4_backend)
|
||||
|
||||
|
||||
def _activation_format_for_config(
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> mk.FusedMoEActivationFormat:
|
||||
return (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
if moe_config.moe_parallel_config.use_batched_activation_format
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
|
||||
def _validate_mxfp4_experts_config(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
) -> None:
|
||||
"""Fail closed when a caller bypasses the oracle and picks experts directly."""
|
||||
activation_format = _activation_format_for_config(moe_config)
|
||||
weight_key, activation_key = _mxfp4_quant_keys_for_experts(
|
||||
experts_cls, mxfp4_backend
|
||||
)
|
||||
supported, reason = experts_cls.is_supported_config(
|
||||
experts_cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if not supported:
|
||||
raise ValueError(
|
||||
f"{experts_cls.__name__} does not support this MoE deployment"
|
||||
f"{f' since {reason}' if reason else ''}."
|
||||
)
|
||||
|
||||
|
||||
def make_mxfp4_moe_kernel(
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
@@ -2136,6 +2207,8 @@ def make_mxfp4_moe_kernel(
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Create a FusedMoEKernel for the given MXFP4 backend."""
|
||||
_validate_mxfp4_experts_config(experts_cls, moe_config, mxfp4_backend)
|
||||
|
||||
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
|
||||
@@ -126,10 +126,10 @@ def backend_to_kernel_cls(
|
||||
|
||||
elif backend == UnquantizedMoeBackend.AITER:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
|
||||
return [AiterExperts]
|
||||
return [AiterUnquantizedExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
@@ -303,9 +303,10 @@ def select_unquantized_moe_backend(
|
||||
)
|
||||
|
||||
runner_backend = moe_config.moe_backend
|
||||
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
|
||||
# modules_to_not_convert) falls through to auto instead of erroring.
|
||||
if runner_backend not in ["auto", "humming"]:
|
||||
# 'humming' and 'marlin' are quantization-only; an unquantized layer (e.g.
|
||||
# excluded via modules_to_not_convert, or an unquantized MTP draft of a
|
||||
# quantized model) falls through to auto instead of erroring.
|
||||
if runner_backend not in ["auto", "humming", "marlin"]:
|
||||
requested_backend = map_unquantized_backend(runner_backend)
|
||||
if (
|
||||
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@@ -127,6 +127,11 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
self.use_cudagraph = use_cudagraph
|
||||
self.sp_size = sp_size
|
||||
|
||||
# Set by FusedMoEKernelModularImpl when the experts globalize
|
||||
# recv_topk_idx themselves, letting the decode path skip the standalone
|
||||
# _globalize_recv_topk_idx launch.
|
||||
self.defer_globalize = False
|
||||
|
||||
# DBO microbatching: one handle slot per micro-batch.
|
||||
self.handles: list[deep_ep.EPHandle | None] = [None, None]
|
||||
|
||||
@@ -275,6 +280,7 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
# during profile_run when CUDA graphs are enabled.
|
||||
expert_tokens_meta = None
|
||||
|
||||
globalize_deferred = False
|
||||
if recv_topk_idx is None:
|
||||
# do_expand=True (prefill mode): build topk_ids from
|
||||
# per-expert token counts.
|
||||
@@ -307,12 +313,15 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
# be treated as real routed tokens by experts that build routing
|
||||
# over *all* rows (e.g. triton MoE backend's make_routing_data),
|
||||
# polluting the per-expert token lists and corrupting real tokens.
|
||||
recv_topk_idx = _globalize_recv_topk_idx(
|
||||
recv_topk_idx,
|
||||
psum_recv_per_rank,
|
||||
self.rank_expert_offset,
|
||||
self.num_experts,
|
||||
)
|
||||
if self.defer_globalize and self.use_cudagraph:
|
||||
globalize_deferred = True
|
||||
else:
|
||||
recv_topk_idx = _globalize_recv_topk_idx(
|
||||
recv_topk_idx,
|
||||
psum_recv_per_rank,
|
||||
self.rank_expert_offset,
|
||||
self.num_experts,
|
||||
)
|
||||
|
||||
# Reshape recv_topk_weights to match recv_topk_idx shape [N, 1]
|
||||
if recv_topk_weights is not None and recv_topk_weights.ndim == 1:
|
||||
@@ -329,6 +338,9 @@ class DeepEPV2PrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
expert_num_tokens_cpu=None,
|
||||
)
|
||||
expert_tokens_meta.psum_recv_per_rank = psum_recv_per_rank
|
||||
if globalize_deferred:
|
||||
# recv_topk_idx still holds LOCAL ids; experts globalize it.
|
||||
expert_tokens_meta.rank_expert_offset = self.rank_expert_offset
|
||||
|
||||
if _quantize_before_dispatch(quant_config, defer_input_quant):
|
||||
if quant_config.quant_dtype == "mxfp8" and expert_x_scale is not None:
|
||||
|
||||
@@ -21,12 +21,15 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
max_tokens_per_rank: int,
|
||||
num_dispatchers: int,
|
||||
use_fp8_dispatch: bool = False,
|
||||
use_fp4_dispatch: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
assert not (use_fp8_dispatch and use_fp4_dispatch)
|
||||
self.mori_op = mori_op
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
self.max_tokens_per_rank = max_tokens_per_rank
|
||||
self.use_fp8_dispatch = use_fp8_dispatch
|
||||
self.use_fp4_dispatch = use_fp4_dispatch
|
||||
self._dispatch_topk_ids: torch.Tensor | None = None
|
||||
|
||||
@property
|
||||
@@ -73,7 +76,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
)
|
||||
scale = None
|
||||
# When defer_input_quant is True, the expert kernel handles
|
||||
# quantization internally, so skip FP8 dispatch quantization.
|
||||
# quantization internally, so skip dispatch quantization.
|
||||
if self.use_fp8_dispatch and not defer_input_quant:
|
||||
from aiter import QuantType, get_hip_quant
|
||||
|
||||
@@ -92,6 +95,14 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
)
|
||||
# mori expects one scale slot per token; broadcast.
|
||||
scale = scale.expand(a1.shape[0], 1).contiguous()
|
||||
elif self.use_fp4_dispatch and not defer_input_quant:
|
||||
from aiter import QuantType, get_hip_quant
|
||||
|
||||
# Packed MXFP4 [M, K // 2] with e8m0 scales [M, K // 32]. The
|
||||
# scales stay unshuffled: AITER fused_moe sorts them itself
|
||||
# when it receives pre-quantized FP4 input.
|
||||
quant_func = get_hip_quant(QuantType.per_1x32)
|
||||
a1, scale = quant_func(a1, shuffle=False)
|
||||
|
||||
# mori's combine() reduces over this rank's own [num_tokens, topk]
|
||||
# routing. The modular kernel rebinds topk_ids to the dispatched ids
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -11,7 +14,13 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import nvfp4_block_scale_interleave
|
||||
from vllm.v1.worker.ubatching import (
|
||||
dbo_enabled,
|
||||
dbo_switch_to_compute_sync,
|
||||
dbo_yield_and_switch_from_compute_to_comm,
|
||||
)
|
||||
|
||||
|
||||
def _quantize_and_setup_dispatch(
|
||||
@@ -133,6 +142,34 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
return self._allocate_combine_input(shape, dtype, device)
|
||||
return None
|
||||
|
||||
def supports_dbo(self) -> bool:
|
||||
# prepare()/finalize() hand the collective to the comm stream
|
||||
# themselves (see _comm_region), so the synchronous path is DBO-safe.
|
||||
# supports_async() stays False: it also selects MK-internal
|
||||
# shared-expert overlap. Only validated on ROCm so far.
|
||||
return current_platform.is_rocm()
|
||||
|
||||
@contextmanager
|
||||
def _comm_region(self) -> Iterator[None]:
|
||||
"""Scope the dispatch/combine collective runs in.
|
||||
|
||||
Without DBO the collective runs on the calling stream. Under DBO it
|
||||
runs on the comm stream so the peer ubatch can compute while it is in
|
||||
flight, using the same compute/comm handoff as DeepEP-HT. Graph
|
||||
capture keeps every collective on the capture stream, so overlap is
|
||||
limited to eager steps (prefill).
|
||||
"""
|
||||
if not dbo_enabled() or torch.cuda.is_current_stream_capturing():
|
||||
yield
|
||||
return
|
||||
# Release the CPU to the peer ubatch so it can enqueue compute while
|
||||
# this ubatch's collective runs; the compute stream waits on exit.
|
||||
dbo_yield_and_switch_from_compute_to_comm()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
dbo_switch_to_compute_sync()
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
@@ -178,13 +215,14 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
extra_tensors = []
|
||||
extra_tensors.append(local_token_lora_mapping)
|
||||
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
with self._comm_region():
|
||||
res = get_ep_group().dispatch(
|
||||
a1q,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
|
||||
if extra_tensors is None:
|
||||
assert len(res) == 3
|
||||
@@ -238,11 +276,12 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
with self._comm_region():
|
||||
get_ep_group().combine_into_output(
|
||||
out,
|
||||
output,
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
)
|
||||
|
||||
|
||||
class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
|
||||
|
||||
@@ -700,12 +700,29 @@ class RoutedExperts(PluggableLayer):
|
||||
param.data[:, :dim1, :dim2].copy_(loaded_weight)
|
||||
return True if return_success else None
|
||||
|
||||
quant_method_name = self.quant_method.__class__.__name__
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
|
||||
base_quant_method = self.quant_method
|
||||
|
||||
# In case online MOE re-quantization is used, loading is handled
|
||||
# by the original quant method.
|
||||
if (
|
||||
isinstance(base_quant_method, OnlineMoEMethodBase)
|
||||
and base_quant_method.requantization_source is not None
|
||||
):
|
||||
assert isinstance(
|
||||
base_quant_method.requantization_source, FusedMoEMethodBase
|
||||
)
|
||||
base_quant_method = base_quant_method.requantization_source
|
||||
|
||||
quant_method_name = base_quant_method.__class__.__name__
|
||||
global_expert_id = expert_id
|
||||
expert_id = self._map_global_expert_id_to_local_expert_id(global_expert_id)
|
||||
|
||||
use_global_sf = (
|
||||
getattr(self.quant_method, "use_global_sf", False)
|
||||
getattr(base_quant_method, "use_global_sf", False)
|
||||
and "input_scale" in weight_name
|
||||
)
|
||||
|
||||
@@ -842,7 +859,7 @@ class RoutedExperts(PluggableLayer):
|
||||
if "ModelOpt" in quant_method_name:
|
||||
# Determine per-tensor weight scale patterns based on variant
|
||||
# Use the dedicated method instead of brittle string matching
|
||||
uses_weight_scale_2 = self.quant_method.uses_weight_scale_2_pattern()
|
||||
uses_weight_scale_2 = base_quant_method.uses_weight_scale_2_pattern()
|
||||
quant_method = getattr(param, "quant_method", None)
|
||||
|
||||
# Call _load_per_tensor_weight_scale() to load per-tensor (scalar)
|
||||
@@ -921,7 +938,7 @@ class RoutedExperts(PluggableLayer):
|
||||
FusedMoeWeightScaleSupported.GROUP.value,
|
||||
FusedMoeWeightScaleSupported.BLOCK.value,
|
||||
]:
|
||||
scale_refine = getattr(self.quant_method, "weight_scale_refine", None)
|
||||
scale_refine = getattr(base_quant_method, "weight_scale_refine", None)
|
||||
if (
|
||||
quant_method == FusedMoeWeightScaleSupported.BLOCK.value
|
||||
and scale_refine is not None
|
||||
@@ -986,12 +1003,20 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
for expert_name, loaded_weight in weights:
|
||||
qual_name = f"{self.layer_name}.{expert_name}"
|
||||
is_fused = loaded_weight.dim() == 3
|
||||
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
|
||||
named_expert = (
|
||||
mapping_by_expert.get(expert_key)
|
||||
if qual_name.count("experts.") == 1
|
||||
else None
|
||||
)
|
||||
# Encoded per-expert [N, K / block_size, bytes_per_block] weights
|
||||
# have the same rank as fused [E, N, K] weights. An explicit expert
|
||||
# name identifies the per-expert layout without treating N as E.
|
||||
is_fused = loaded_weight.dim() == 3 and named_expert is None
|
||||
# Fused tensors and ambiguous names keep the full mapping.
|
||||
candidates = expert_mapping
|
||||
if not is_fused and qual_name.count("experts.") == 1:
|
||||
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
|
||||
candidates = mapping_by_expert.get(expert_key, expert_mapping)
|
||||
candidates = named_expert or expert_mapping
|
||||
|
||||
matched = False
|
||||
for param_name, weight_name, expert_id, shard_id in candidates:
|
||||
|
||||
@@ -31,7 +31,7 @@ def can_use_dsv4_topk(
|
||||
and correction_bias.is_contiguous()
|
||||
and topk == _TOPK
|
||||
and renormalize
|
||||
and indices_dtype in (torch.int32, torch.uint32, torch.int64)
|
||||
and indices_dtype in (torch.int32, torch.int64)
|
||||
)
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ if current_platform.is_cuda():
|
||||
def _dsv4_topk_kernel(
|
||||
gating_output_ptr,
|
||||
correction_bias_ptr,
|
||||
is_padding_ptr,
|
||||
topk_weights_ptr,
|
||||
topk_ids_ptr,
|
||||
routed_scaling_factor,
|
||||
@@ -49,6 +50,7 @@ if current_platform.is_cuda():
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
HAS_PADDING: tl.constexpr,
|
||||
HAS_VL: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
@@ -106,6 +108,11 @@ if current_platform.is_cuda():
|
||||
output_mask = topk_offsets < 6
|
||||
output_offsets = row * 6 + topk_offsets
|
||||
|
||||
if HAS_PADDING:
|
||||
is_padding = tl.load(is_padding_ptr + row)
|
||||
selected_weights = tl.where(is_padding, 0.0, selected_weights)
|
||||
selected_ids = tl.where(is_padding, -1, selected_ids)
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
@@ -118,11 +125,20 @@ def dsv4_topk(
|
||||
correction_bias: torch.Tensor,
|
||||
indices_dtype: torch.dtype,
|
||||
routed_scaling_factor: float,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
bias_vl: torch.Tensor | None = None,
|
||||
image_sentinel_lo: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens, num_experts = gating_output.shape
|
||||
if is_padding is not None:
|
||||
assert is_padding.dtype == torch.bool
|
||||
assert is_padding.shape == (num_tokens,)
|
||||
assert is_padding.device == gating_output.device
|
||||
assert is_padding.is_contiguous()
|
||||
assert indices_dtype in (torch.int32, torch.int64), (
|
||||
"Padding requires a signed indices dtype for the -1 sentinel."
|
||||
)
|
||||
has_vl = bias_vl is not None and image_sentinel_lo > 0
|
||||
if bias_vl is not None:
|
||||
assert input_ids is not None, "bias_vl routing requires input_ids"
|
||||
@@ -136,6 +152,7 @@ def dsv4_topk(
|
||||
_dsv4_topk_kernel[(num_tokens,)](
|
||||
gating_output,
|
||||
correction_bias,
|
||||
is_padding,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
routed_scaling_factor,
|
||||
@@ -144,6 +161,7 @@ def dsv4_topk(
|
||||
image_sentinel_lo,
|
||||
NUM_EXPERTS=num_experts,
|
||||
BLOCK_N=triton.next_power_of_2(num_experts),
|
||||
HAS_PADDING=is_padding is not None,
|
||||
HAS_VL=has_vl,
|
||||
num_warps=1,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
|
||||
@@ -176,6 +176,7 @@ def fused_topk_bias(
|
||||
)
|
||||
|
||||
output_indices_dtype = torch.int32 if indices_type is None else indices_type
|
||||
padding_mask = _get_padding_mask(gating_output.shape[0])
|
||||
if (
|
||||
scoring_func == "sqrtsoftplus"
|
||||
and hash_indices_table is None
|
||||
@@ -193,6 +194,7 @@ def fused_topk_bias(
|
||||
e_score_correction_bias,
|
||||
output_indices_dtype,
|
||||
routed_scaling_factor,
|
||||
is_padding=padding_mask,
|
||||
input_ids=input_tokens,
|
||||
bias_vl=bias_vl,
|
||||
image_sentinel_lo=image_sentinel_lo,
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -22,14 +23,16 @@ class GateLinear(ReplicatedLinear):
|
||||
"""MoE gate linear layer with multi-tier GEMM dispatch:
|
||||
|
||||
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
|
||||
K divisible by 8)
|
||||
any shape w/ K divisible by 8)
|
||||
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
|
||||
M<=32, model-specific shapes)
|
||||
3. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
4. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
|
||||
any shape)
|
||||
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
|
||||
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
|
||||
M>=2048)
|
||||
5. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
6. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
|
||||
7. F.linear via ReplicatedLinear (ultimate fallback)
|
||||
|
||||
The ``out_dtype`` attribute is mutable and can be set after init
|
||||
(e.g. when the required dtype depends on the expert quantization
|
||||
@@ -37,6 +40,9 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
A ``quant_config`` that actually quantizes the gate disables every
|
||||
specialized tier, leaving plain ``ReplicatedLinear`` behavior.
|
||||
|
||||
``VLLM_BATCH_INVARIANT`` also disables tiers 1-5: they pick kernels and
|
||||
split-K from the token count and skip the batch-invariant linear path.
|
||||
"""
|
||||
|
||||
# (hidden_size, num_experts) pairs with an instantiated fp32 kernel:
|
||||
@@ -77,8 +83,12 @@ class GateLinear(ReplicatedLinear):
|
||||
input_size,
|
||||
output_size,
|
||||
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
|
||||
batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
can_use_specialized_kernels = (
|
||||
current_platform.is_cuda() and (is_hopper or is_blackwell) and not bias
|
||||
current_platform.is_cuda()
|
||||
and (is_hopper or is_blackwell)
|
||||
and not bias
|
||||
and not batch_invariant
|
||||
)
|
||||
|
||||
# If fp32 compute is required and no specialized kernel is available,
|
||||
@@ -108,6 +118,7 @@ class GateLinear(ReplicatedLinear):
|
||||
# fp32 specialized kernel eligibility (exact dims, fp32 weight)
|
||||
self.allow_fp32_router_gemm = (
|
||||
self.is_unquantized
|
||||
and not batch_invariant
|
||||
and not bias
|
||||
and self.weight.dtype == torch.float32
|
||||
and (
|
||||
@@ -121,6 +132,7 @@ class GateLinear(ReplicatedLinear):
|
||||
)
|
||||
self.allow_bf16x3_router_gemm = (
|
||||
self.is_unquantized
|
||||
and not batch_invariant
|
||||
and not bias
|
||||
and self.weight.dtype == torch.float32
|
||||
and current_platform.is_cuda()
|
||||
@@ -137,8 +149,10 @@ class GateLinear(ReplicatedLinear):
|
||||
# (GB10 / DGX Spark), which this tier still covers. See #49921.
|
||||
self._router_gemm_no_bias = self.is_unquantized and not bias
|
||||
self._router_gemm_cublas_capable = (
|
||||
current_platform.is_cuda() or current_platform.is_rocm()
|
||||
) and self._router_gemm_no_bias
|
||||
(current_platform.is_cuda() or current_platform.is_rocm())
|
||||
and self._router_gemm_no_bias
|
||||
and not batch_invariant
|
||||
)
|
||||
self.allow_cublas_router_gemm = (
|
||||
self._router_gemm_cublas_capable
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
@@ -154,6 +168,7 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
self._rocm_bf16x3_weight_eligible = (
|
||||
self._router_gemm_no_bias
|
||||
and not batch_invariant
|
||||
and self.weight.dtype == torch.float32
|
||||
and bf16x3_router_gemm_rocm.platform_supported()
|
||||
)
|
||||
@@ -172,20 +187,50 @@ class GateLinear(ReplicatedLinear):
|
||||
self.quant_method = _GateLinearMethod()
|
||||
logger.info_once("Enabled ROCm BF16x3 router GEMM.")
|
||||
|
||||
# cuteDSL ll_bf16_gemm eligibility. Any dims supported, but SM90+ required bc:
|
||||
# cuteDSL ll_bf16_gemm/ll_fp32w_gemm eligibility. Any dims supported, but
|
||||
# SM90+ required because:
|
||||
# 1. PDL support. Both dot-product and split-K kernels.
|
||||
# 2. Thread Block Clusters. Split-K kernel for cross-CTA reduction.
|
||||
self.allow_ll_bf16_gemm = False
|
||||
self.allow_ll_fp32w_gemm = False
|
||||
if can_use_specialized_kernels:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available,
|
||||
is_available as is_ll_bf16_gemm_available,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
is_available as is_ll_fp32w_gemm_available,
|
||||
)
|
||||
|
||||
self.allow_ll_bf16_gemm = (
|
||||
self.weight.dtype == torch.bfloat16
|
||||
is_ll_bf16_gemm_available()
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
and self.out_dtype == torch.float32
|
||||
and is_available()
|
||||
)
|
||||
self.allow_ll_fp32w_gemm = (
|
||||
is_ll_fp32w_gemm_available()
|
||||
and self.weight.dtype == torch.float32
|
||||
and self.out_dtype == torch.float32
|
||||
)
|
||||
|
||||
self._register_ll_fp32w_warmup()
|
||||
|
||||
def _register_ll_fp32w_warmup(self) -> None:
|
||||
# Based on multi-tier GEMM dispatch. See above.
|
||||
# This avoids compiling LL_FP32W_GEMM_KERNEL if not used.
|
||||
if not self.allow_ll_fp32w_gemm or self.allow_fp32_router_gemm:
|
||||
return
|
||||
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
LL_FP32W_GEMM_KERNEL,
|
||||
)
|
||||
|
||||
# FP16 remains runtime-supported; JIT monitoring reports unexpected use.
|
||||
# The kernel supports it, but there is no FP16 activation use case yet.
|
||||
LL_FP32W_GEMM_KERNEL.register_warmup(
|
||||
shapes=((self.weight.shape[1], self.weight.shape[0]),),
|
||||
m_values=range(1, self.FP32_MAX_TOKENS + 1),
|
||||
a_dtypes=(torch.bfloat16, torch.float32),
|
||||
)
|
||||
|
||||
def set_out_dtype(self, out_dtype: torch.dtype) -> None:
|
||||
"""Set output dtype for the router logits after init.
|
||||
@@ -211,14 +256,23 @@ class GateLinear(ReplicatedLinear):
|
||||
|
||||
if self.allow_specialized_router_gemm:
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import (
|
||||
is_available,
|
||||
is_available as is_ll_bf16_gemm_available,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
is_available as is_ll_fp32w_gemm_available,
|
||||
)
|
||||
|
||||
self.allow_ll_bf16_gemm = (
|
||||
self.weight.dtype == torch.bfloat16
|
||||
is_ll_bf16_gemm_available()
|
||||
and self.weight.dtype == torch.bfloat16
|
||||
and out_dtype == torch.float32
|
||||
and is_available()
|
||||
)
|
||||
self.allow_ll_fp32w_gemm = (
|
||||
is_ll_fp32w_gemm_available()
|
||||
and self.weight.dtype == torch.float32
|
||||
and out_dtype == torch.float32
|
||||
)
|
||||
self._register_ll_fp32w_warmup()
|
||||
|
||||
def _return(
|
||||
self, output: torch.Tensor
|
||||
@@ -228,7 +282,8 @@ class GateLinear(ReplicatedLinear):
|
||||
def forward(
|
||||
self, x: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, Parameter | None]:
|
||||
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, any dims)
|
||||
# Tier 1: cuteDSL ll_bf16_gemm (SM90+, bf16 in, fp32 out,
|
||||
# any dims w/ K divisible by 8)
|
||||
if (
|
||||
self.allow_ll_bf16_gemm
|
||||
and x.shape[0] <= self.LL_BF16_MAX_TOKENS
|
||||
@@ -241,7 +296,7 @@ class GateLinear(ReplicatedLinear):
|
||||
output = ll_bf16_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 4: ROCm bf16x3 router GEMM. Checked before tier 2 because both
|
||||
# Tier 5: ROCm bf16x3 router GEMM. Checked before tier 2 because both
|
||||
# tiers claim the same gfx950 shapes and tier 2 returns
|
||||
# unconditionally; the custom op below picks between them on the
|
||||
# runtime num_tokens.
|
||||
@@ -270,7 +325,20 @@ class GateLinear(ReplicatedLinear):
|
||||
)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 3: bf16x3 CuteDSL kernel for fp32 router weights
|
||||
# Tier 3: cuteDSL ll_fp32w_gemm (SM90+, bf16/fp32 in, fp32 out)
|
||||
if (
|
||||
self.allow_ll_fp32w_gemm
|
||||
and x.shape[0] <= 32
|
||||
and x.dtype in (torch.bfloat16, torch.float16, torch.float32)
|
||||
):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
LL_FP32W_GEMM_KERNEL,
|
||||
)
|
||||
|
||||
output = LL_FP32W_GEMM_KERNEL(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 4: bf16x3 CuteDSL kernel for fp32 router weights
|
||||
if self.allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
|
||||
from vllm.model_executor.layers.fused_moe.router.bf16x3_router_gemm_cutedsl import ( # noqa: E501
|
||||
bf16x3_router_gemm,
|
||||
@@ -279,12 +347,12 @@ class GateLinear(ReplicatedLinear):
|
||||
output = bf16x3_router_gemm(x, self.weight)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 5: cuBLAS bf16→fp32
|
||||
# Tier 6: cuBLAS bf16→fp32
|
||||
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
|
||||
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
|
||||
return self._return(output)
|
||||
|
||||
# Tier 6: F.linear (ReplicatedLinear)
|
||||
# Tier 7: F.linear (ReplicatedLinear)
|
||||
if (
|
||||
self.out_dtype is not None
|
||||
and self.is_unquantized
|
||||
|
||||
@@ -301,7 +301,10 @@ class MoERunner(MoERunnerInterface):
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
# routed_experts reports names relative to itself, but callers such as
|
||||
# AutoWeightsLoader qualify what this returns with the runner's prefix.
|
||||
for name in self.routed_experts.load_weights(weights):
|
||||
yield f"routed_experts.{name}"
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu():
|
||||
|
||||
@@ -235,6 +235,73 @@ def count_expert_num_tokens(
|
||||
return expert_num_tokens
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _remap_topk_to_local_kernel(
|
||||
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
|
||||
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
|
||||
out_ptr, # [n] local expert IDs (-1 for invalid/non-local)
|
||||
n_elements,
|
||||
BLOCK: tl.constexpr,
|
||||
OUT_INT32: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < n_elements
|
||||
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
|
||||
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
|
||||
# rows don't read OOB, then select -1 for them. Matches
|
||||
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
|
||||
# plain expert_map[-1] would wrap to a valid local id and misroute).
|
||||
valid = tid >= 0
|
||||
idx = tl.where(valid, tid, 0)
|
||||
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
|
||||
if OUT_INT32:
|
||||
out = tl.where(valid, local, -1)
|
||||
else:
|
||||
out = tl.where(valid, local.to(tl.int64), -1)
|
||||
tl.store(out_ptr + offs, out, mask=mask)
|
||||
|
||||
|
||||
def remap_topk_to_local(
|
||||
topk_ids: torch.Tensor,
|
||||
expert_map: torch.Tensor,
|
||||
*,
|
||||
out_dtype: torch.dtype = torch.int64,
|
||||
) -> torch.Tensor:
|
||||
"""Map global expert ids to local ids; non-local or invalid routes become -1.
|
||||
|
||||
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
|
||||
with one kernel. Returns a NEW tensor -- the caller keeps the original
|
||||
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
|
||||
|
||||
Args:
|
||||
topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous;
|
||||
the Triton kernel indexes by flat ``numel()`` offsets.
|
||||
expert_map: Global-to-local mapping; -1 marks non-local experts.
|
||||
out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``.
|
||||
|
||||
Returns:
|
||||
Remapped local expert ids with the same shape as ``topk_ids``.
|
||||
|
||||
"""
|
||||
if out_dtype not in (torch.int32, torch.int64):
|
||||
raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}")
|
||||
assert topk_ids.is_contiguous()
|
||||
out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device)
|
||||
n = topk_ids.numel()
|
||||
BLOCK = 1024
|
||||
grid = (triton.cdiv(n, BLOCK),)
|
||||
_remap_topk_to_local_kernel[grid](
|
||||
topk_ids,
|
||||
expert_map,
|
||||
out,
|
||||
n,
|
||||
BLOCK=BLOCK,
|
||||
OUT_INT32=out_dtype == torch.int32,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
|
||||
"""Shrink the given tensor and apply the given view to it. This is
|
||||
used to resize the intermediate fused_moe caches.
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm import envs, ir
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.determinism.batch_invariant import rms_norm_batch_invariant
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -63,6 +64,9 @@ class RMSNorm(CustomOp):
|
||||
self.weight = torch.ones(hidden_size, dtype=weight_dtype)
|
||||
if self.has_weight:
|
||||
self.weight = nn.Parameter(self.weight)
|
||||
# Dummy weights keep norms neutral: random ones would shrink every
|
||||
# activation and skew data-dependent kernels (e.g. sparse top-k).
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
|
||||
# When has_weight=False, pass weight=None so implementations that
|
||||
# support a weightless path can skip the per-channel multiply.
|
||||
@@ -154,6 +158,7 @@ class GemmaRMSNorm(CustomOp):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(hidden_size))
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 0.0})
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward_native(
|
||||
@@ -239,8 +244,10 @@ class LayerNorm(CustomOp):
|
||||
self.bias: nn.Parameter | None = None
|
||||
if elementwise_affine:
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
if bias:
|
||||
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))
|
||||
set_weight_attrs(self.bias, {"dummy_weight_value": 0.0})
|
||||
|
||||
def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
|
||||
if self.weight is None:
|
||||
@@ -328,6 +335,7 @@ class RMSNormGated(CustomOp):
|
||||
self.eps = eps
|
||||
self.activation = activation
|
||||
self.weight = nn.Parameter(torch.empty(hidden_size, **factory_kwargs))
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
self.register_parameter("bias", None)
|
||||
self.group_size = group_size
|
||||
self.norm_before_gate = norm_before_gate
|
||||
|
||||
@@ -516,6 +516,20 @@ class ColumnParallelLinear(LinearBase):
|
||||
self._maybe_allow_fp8_block_shape_mismatch()
|
||||
self.gather_output = gather_output
|
||||
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
quant_method = self.quant_method
|
||||
if (
|
||||
isinstance(quant_method, OnlineLinearBase)
|
||||
and quant_method.requantization_source is not None
|
||||
):
|
||||
quant_method = quant_method.requantization_source
|
||||
weight_loader = (
|
||||
self.weight_loader_v2
|
||||
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
)
|
||||
|
||||
self.quant_method.create_weights(
|
||||
layer=self,
|
||||
input_size_per_partition=self.input_size_per_partition,
|
||||
@@ -523,11 +537,7 @@ class ColumnParallelLinear(LinearBase):
|
||||
input_size=self.input_size,
|
||||
output_size=self.output_size,
|
||||
params_dtype=self.params_dtype,
|
||||
weight_loader=(
|
||||
self.weight_loader_v2
|
||||
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
if bias:
|
||||
@@ -1697,6 +1707,20 @@ class RowParallelLinear(LinearBase):
|
||||
self.input_is_parallel = input_is_parallel
|
||||
self.reduce_results = reduce_results
|
||||
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
quant_method = self.quant_method
|
||||
if (
|
||||
isinstance(quant_method, OnlineLinearBase)
|
||||
and quant_method.requantization_source is not None
|
||||
):
|
||||
quant_method = quant_method.requantization_source
|
||||
weight_loader = (
|
||||
self.weight_loader_v2
|
||||
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
)
|
||||
|
||||
self.quant_method.create_weights(
|
||||
layer=self,
|
||||
input_size_per_partition=self.input_size_per_partition,
|
||||
@@ -1704,11 +1728,7 @@ class RowParallelLinear(LinearBase):
|
||||
input_size=self.input_size,
|
||||
output_size=self.output_size,
|
||||
params_dtype=self.params_dtype,
|
||||
weight_loader=(
|
||||
self.weight_loader_v2
|
||||
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
if not reduce_results and (bias and not skip_bias_add):
|
||||
raise ValueError(
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Iterable
|
||||
from math import prod
|
||||
from math import lcm, prod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
@@ -62,6 +62,13 @@ class MambaBase(AttentionLayerBase):
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
pass
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
"""Token alignment of the checkpoint offsets this layer can export from
|
||||
inside a prefill, if any; in align mode, the platform makes Mamba blocks
|
||||
multiples of it.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
mamba_block_size = vllm_config.cache_config.mamba_block_size
|
||||
assert mamba_block_size is not None
|
||||
@@ -86,3 +93,21 @@ class MambaBase(AttentionLayerBase):
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
"""Get the attention backend class for this Mamba layer."""
|
||||
return get_mamba_attn_backend(self.mamba_type)
|
||||
|
||||
|
||||
def get_mamba_prefill_checkpoint_alignment(vllm_config: VllmConfig) -> int | None:
|
||||
"""LCM of the Mamba layers' prefill checkpoint alignments.
|
||||
|
||||
None unless every Mamba layer declares one: the scheduler takes internal
|
||||
checkpoints only when every Mamba group can export them.
|
||||
"""
|
||||
alignment = 1
|
||||
for layer in get_layers_from_vllm_config(
|
||||
vllm_config,
|
||||
MambaBase, # type: ignore[type-abstract]
|
||||
).values():
|
||||
layer_alignment = layer.get_prefill_checkpoint_alignment()
|
||||
if layer_alignment is None:
|
||||
return None
|
||||
alignment = lcm(alignment, layer_alignment)
|
||||
return alignment
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
@@ -71,9 +71,11 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
# Host copies of ``checkpoint_offsets`` and of each row's query length, for
|
||||
# backends that lay out the checkpoint before any tensor exists (Mamba2's
|
||||
# SSD chunk split, FlashInfer GDN's checkpoint plan).
|
||||
offsets: list[int] | None = None
|
||||
query_lens: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -150,20 +152,134 @@ class MambaPrefillCheckpointBuilder:
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
query_lens=query_lens,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
|
||||
class MambaPrefillCheckpointExporter(ABC):
|
||||
"""Export a mid-prefill checkpoint into backend-specific paged states."""
|
||||
@dataclass(frozen=True)
|
||||
class MambaPrefillCheckpointExporter:
|
||||
"""Store each checkpoint's pre-convolution window and recurrent state.
|
||||
|
||||
The recurrent states are rows of a buffer the prefill kernel emitted;
|
||||
``recurrent_rows`` picks each checkpoint row's state out of it, and by
|
||||
default row ``i`` holds checkpoint ``i``'s.
|
||||
"""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
@abstractmethod
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*args,
|
||||
**kwargs,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
recurrent_rows: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Write checkpoint state into the paged cache."""
|
||||
raise NotImplementedError
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
recurrent_rows if recurrent_rows is not None else checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
recurrent_rows is not None,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
recurrent_rows_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
HAS_RECURRENT_ROWS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
if HAS_RECURRENT_ROWS:
|
||||
recurrent_row = tl.load(recurrent_rows_ptr + seq_idx).to(tl.int64)
|
||||
else:
|
||||
recurrent_row = seq_idx.to(tl.int64)
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + recurrent_row * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
@@ -524,6 +525,7 @@ def _make_fused_conv1d_weight_loader(dims, tp_size, tp_rank):
|
||||
return weight_loader
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def olmo_hybrid_gdn_full_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Qwen3-Next/Qwen3.5 model."""
|
||||
|
||||
import functools
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
@@ -15,6 +19,7 @@ from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
)
|
||||
@@ -30,6 +35,12 @@ from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.abstract import (
|
||||
get_mamba_prefill_checkpoint_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import mamba_v2_sharded_weight_loader
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
@@ -59,13 +70,19 @@ from vllm.third_party.flash_linear_attention.ops import (
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
FLASHINFER_GDN_CHECKPOINT_ALIGNMENT,
|
||||
GDNAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
# Optional ROCm AITER Triton kernels for the GDN decode path.
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
@@ -90,6 +107,23 @@ MAX_FUSED_GDN_MTP_TOKENS = 8
|
||||
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
|
||||
|
||||
|
||||
def _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config: VllmConfig, head_k_dim: int | None, head_v_dim: int | None
|
||||
) -> list[str]:
|
||||
"""Every reason the AITER FlyDSL GDN prefill kernels cannot serve this
|
||||
model, or an empty list if they can."""
|
||||
reasons = []
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
reasons.append(
|
||||
f"linear head dims are K={head_k_dim} V={head_v_dim}; "
|
||||
"FlyDSL requires 128/128"
|
||||
)
|
||||
dtype = vllm_config.model_config.dtype
|
||||
if dtype != torch.bfloat16:
|
||||
reasons.append(f"model dtype is {dtype}; FlyDSL requires bfloat16")
|
||||
return reasons
|
||||
|
||||
|
||||
def _resolve_gdn_prefill_backend(
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[str, Literal["triton", "flashinfer", "cutedsl", "aiter_flydsl"]]:
|
||||
@@ -108,7 +142,7 @@ def _resolve_gdn_prefill_backend(
|
||||
* Blackwell (SM10.x) with ``head_k_dim == 128``;
|
||||
|
||||
AITER FlyDSL GDN prefill kernels are chosen when:
|
||||
* "aiter_flydsl" is requested; (opt-in only)
|
||||
* "aiter_flydsl" is requested, or "auto" is requested on gfx942/gfx950;
|
||||
* ROCm AITER exposes the optimized VK prefill API and FlyDSL kernels;
|
||||
* the model uses BF16 activations with ``head_k_dim == head_v_dim == 128``.
|
||||
"""
|
||||
@@ -136,24 +170,28 @@ def _resolve_gdn_prefill_backend(
|
||||
"that VLLM_ROCM_USE_AITER=1, that this is a CDNA 3 or "
|
||||
"newer GPU, and that the installed AITER exports them."
|
||||
)
|
||||
if head_k_dim != 128 or head_v_dim != 128:
|
||||
reasons = _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config, head_k_dim, head_v_dim
|
||||
)
|
||||
if reasons:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"linear head dims are K=%s V=%s; FlyDSL requires 128/128. "
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but %s. "
|
||||
"Falling back to Triton/FLA.",
|
||||
head_k_dim,
|
||||
head_v_dim,
|
||||
)
|
||||
return backend, "triton"
|
||||
if vllm_config.model_config.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"GDN prefill backend 'aiter_flydsl' was requested but "
|
||||
"model dtype is %s; FlyDSL requires bfloat16. "
|
||||
"Falling back to Triton/FLA.",
|
||||
vllm_config.model_config.dtype,
|
||||
"; ".join(reasons),
|
||||
)
|
||||
return backend, "triton"
|
||||
return backend, "aiter_flydsl"
|
||||
if backend == "auto":
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
if (
|
||||
on_mi3xx()
|
||||
and not _aiter_flydsl_unsupported_reasons(
|
||||
vllm_config, head_k_dim, head_v_dim
|
||||
)
|
||||
and rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
):
|
||||
return backend, "aiter_flydsl"
|
||||
return backend, "triton"
|
||||
|
||||
if backend == "aiter_flydsl":
|
||||
@@ -241,6 +279,10 @@ def fi_chunk_gated_delta_rule(
|
||||
output_final_state: bool,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = True,
|
||||
state_checkpoints: torch.Tensor | None = None,
|
||||
checkpoint_cu_starts: torch.Tensor | None = None,
|
||||
checkpoint_every_n_tokens: int = 0,
|
||||
use_cp: Literal["auto"] | bool = "auto",
|
||||
):
|
||||
from flashinfer.gdn_prefill import (
|
||||
chunk_gated_delta_rule as chunk_gated_delta_rule_fi,
|
||||
@@ -271,6 +313,10 @@ def fi_chunk_gated_delta_rule(
|
||||
initial_state=fi_state,
|
||||
output_final_state=output_final_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=checkpoint_cu_starts,
|
||||
checkpoint_every_n_tokens=checkpoint_every_n_tokens,
|
||||
use_cp=use_cp,
|
||||
backend="flashinfer",
|
||||
)
|
||||
# FlashInfer returns (output, state) when output_final_state=True,
|
||||
@@ -283,6 +329,20 @@ def fi_chunk_gated_delta_rule(
|
||||
return result.unsqueeze(0), None
|
||||
|
||||
|
||||
def _checkpoint_grid(cache_config: CacheConfig) -> int:
|
||||
"""Token grid that FlashInfer GDN checkpoint offsets are kept on.
|
||||
|
||||
Prefix-cache boundaries, made multiples of FlashInfer's checkpoint
|
||||
alignment; the platform already keeps blocks on it.
|
||||
"""
|
||||
if cache_config.prefix_match_unit is not None:
|
||||
return math.lcm(
|
||||
cache_config.prefix_match_unit, FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
|
||||
)
|
||||
assert cache_config.mamba_block_size is not None
|
||||
return cache_config.mamba_block_size
|
||||
|
||||
|
||||
@CustomOp.register("chunk_gated_delta_rule")
|
||||
class ChunkGatedDeltaRule(CustomOp):
|
||||
def __init__(self) -> None:
|
||||
@@ -622,6 +682,21 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
|
||||
self.gdn_prefill_backend = self.chunk_gated_delta_rule.gdn_prefill_backend
|
||||
self._prefill_kernels_warmed_up = False
|
||||
# FlashInfer's prefill emits mid-prefill states, so an align-mode
|
||||
# checkpoint needs no split prefill.
|
||||
self._checkpoint_alignment: int | None = None
|
||||
if (
|
||||
vllm_config.cache_config.mamba_cache_mode == "align"
|
||||
and self.gdn_prefill_backend == "flashinfer"
|
||||
):
|
||||
self._checkpoint_alignment = FLASHINFER_GDN_CHECKPOINT_ALIGNMENT
|
||||
self._vllm_config = vllm_config
|
||||
self._max_num_batched_tokens = (
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self._checkpoint_exporter = MambaPrefillCheckpointExporter(
|
||||
state_len=self.conv_kernel_size - 1
|
||||
)
|
||||
self.enable_packed_recurrent_decode = (
|
||||
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
|
||||
)
|
||||
@@ -666,6 +741,46 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_prefill_checkpoint_alignment(self) -> int | None:
|
||||
return self._checkpoint_alignment
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not self._exports_checkpoints or not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# FlashInfer emits states at one stride for the whole batch, the gcd of
|
||||
# the offsets; keeping every offset on the prefix-cache grid keeps that
|
||||
# stride a multiple of the grid, which bounds the states emitted.
|
||||
return replace(
|
||||
spec,
|
||||
num_prefill_checkpoint_blocks=1,
|
||||
prefill_checkpoint_alignment=_checkpoint_grid(vllm_config.cache_config),
|
||||
)
|
||||
|
||||
@functools.cached_property
|
||||
def _exports_checkpoints(self) -> bool:
|
||||
# Known only once every layer is built: a Mamba layer that cannot
|
||||
# checkpoint (e.g. Qwen4Exp's PLE short conv) keeps the scheduler
|
||||
# splitting the prefill instead.
|
||||
return (
|
||||
self._checkpoint_alignment is not None
|
||||
and get_mamba_prefill_checkpoint_alignment(self._vllm_config) is not None
|
||||
)
|
||||
|
||||
@functools.cached_property
|
||||
def _checkpoint_buffer_spec(self) -> tuple[tuple[int, ...], torch.dtype]:
|
||||
# Offsets on the grid emit at most one state per grid step of a batch.
|
||||
# Read lazily: the block size is final only after the model is built.
|
||||
return (
|
||||
(
|
||||
cdiv(self._max_num_batched_tokens, _checkpoint_grid(self.cache_config)),
|
||||
self.num_v_heads // self.tp_size,
|
||||
self.head_v_dim,
|
||||
self.head_k_dim,
|
||||
),
|
||||
torch.float32,
|
||||
)
|
||||
|
||||
def _fused_gdn_decode_unsupported_reason(
|
||||
self, vllm_config: VllmConfig
|
||||
) -> str | None:
|
||||
@@ -1228,6 +1343,9 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
if self._prefill_kernels_warmed_up:
|
||||
return
|
||||
self._prefill_kernels_warmed_up = True
|
||||
if self._exports_checkpoints:
|
||||
# Size the workspace while profiling; it is locked afterwards.
|
||||
current_workspace_manager().get_simultaneous(self._checkpoint_buffer_spec)
|
||||
|
||||
device = qkv_or_qkvz.device
|
||||
dtype = qkv_or_qkvz.dtype
|
||||
@@ -1303,6 +1421,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=aiter_prefill_metadata,
|
||||
)
|
||||
if self.gdn_prefill_backend == "flashinfer":
|
||||
self._warmup_flashinfer_prefill_variants(q, k, v, g, beta, state)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"GDN prefill kernel warmup (T=%d) failed for "
|
||||
@@ -1337,6 +1457,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
def _warmup_flashinfer_prefill_variants(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
g: torch.Tensor,
|
||||
beta: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
) -> None:
|
||||
"""Compile every FlashInfer GDN prefill kernel a batch can dispatch to.
|
||||
|
||||
FlashInfer builds separate kernels for its context-parallel route (few
|
||||
sequences) and its regular route, each with and without checkpoint
|
||||
output. A single sequence under ``use_cp="auto"`` takes the CP route
|
||||
wherever the heuristic can pick it; ``use_cp=False`` forces the other.
|
||||
"""
|
||||
checkpoint_buffers: list[torch.Tensor | None] = [None]
|
||||
if self._exports_checkpoints:
|
||||
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
|
||||
self._checkpoint_buffer_spec
|
||||
)
|
||||
checkpoint_buffers.append(state_checkpoints[:1])
|
||||
cu_seqlens = torch.tensor([0, q.shape[1]], device=q.device, dtype=torch.int32)
|
||||
checkpoint_cu_starts = torch.tensor([0, 1], device=q.device)
|
||||
for use_cp, state_checkpoints in itertools.product(
|
||||
("auto", False), checkpoint_buffers
|
||||
):
|
||||
has_checkpoint = state_checkpoints is not None
|
||||
fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
use_cp=use_cp,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=checkpoint_cu_starts if has_checkpoint else None,
|
||||
checkpoint_every_n_tokens=q.shape[1] if has_checkpoint else 0,
|
||||
)
|
||||
|
||||
def _forward_core_rocm(
|
||||
self,
|
||||
qkvz: torch.Tensor,
|
||||
@@ -1508,6 +1672,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
)
|
||||
|
||||
# 1.2: Process the remaining part
|
||||
conv_input_non_spec = mixed_qkv_non_spec
|
||||
if attn_metadata.num_prefills > 0:
|
||||
assert mixed_qkv_non_spec is not None
|
||||
mixed_qkv_non_spec_T = mixed_qkv_non_spec.transpose(0, 1)
|
||||
@@ -1709,23 +1874,59 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
assert prefill_has_initial_state is not None
|
||||
initial_state = ssm_state[prefill_state_indices]
|
||||
initial_state[~prefill_has_initial_state, ...] = 0
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = self.chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
plan = attn_metadata.flashinfer_checkpoint_plan
|
||||
if plan is None:
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = self.chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
chunk_indices=attn_metadata.chunk_indices,
|
||||
chunk_offsets=attn_metadata.chunk_offsets,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
aiter_prefill_metadata=attn_metadata.aiter_prefill_metadata,
|
||||
)
|
||||
else:
|
||||
# Only FlashInfer's prefill emits checkpoints.
|
||||
(state_checkpoints,) = current_workspace_manager().get_simultaneous(
|
||||
self._checkpoint_buffer_spec
|
||||
)
|
||||
state_checkpoints = state_checkpoints[: plan.num_checkpoints]
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = fi_chunk_gated_delta_rule(
|
||||
q=query_non_spec,
|
||||
k=key_non_spec,
|
||||
v=value_non_spec,
|
||||
g=g_non_spec,
|
||||
beta=beta_non_spec,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=attn_metadata.prefill_query_start_loc,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=plan.cu_starts,
|
||||
checkpoint_every_n_tokens=plan.every_n_tokens,
|
||||
)
|
||||
assert attn_metadata.checkpoint is not None
|
||||
assert non_spec_query_start_loc is not None
|
||||
self._checkpoint_exporter.export(
|
||||
attn_metadata.checkpoint,
|
||||
raw_qkv=conv_input_non_spec,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=state_checkpoints,
|
||||
recurrent_state=ssm_state,
|
||||
cu_seqlens=non_spec_query_start_loc,
|
||||
recurrent_rows=plan.rows,
|
||||
)
|
||||
# Init cache
|
||||
ssm_state[prefill_state_indices] = last_recurrent_state.to(ssm_state.dtype)
|
||||
|
||||
|
||||
@@ -1,130 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
|
||||
def kda_prefill_checkpoint_alignment(backend: str) -> int | None:
|
||||
return 16 if backend == "flashkda" else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashKDAPrefillCheckpointExporter(MambaPrefillCheckpointExporter):
|
||||
"""Store FlashKDA recurrent and convolution checkpoint states."""
|
||||
|
||||
state_len: int | None = None
|
||||
|
||||
def export(
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
recurrent_checkpoint: torch.Tensor,
|
||||
recurrent_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> None:
|
||||
state_len = (
|
||||
self.state_len if self.state_len is not None else conv_state.shape[-1]
|
||||
)
|
||||
width = raw_qkv.shape[-1]
|
||||
recurrent_row_size = recurrent_checkpoint[0].numel()
|
||||
block_size = 256
|
||||
store_cache_checkpoints_kernel[
|
||||
(
|
||||
checkpoint.checkpoint_offsets.numel(),
|
||||
triton.cdiv(max(width * state_len, recurrent_row_size), block_size),
|
||||
)
|
||||
](
|
||||
raw_qkv,
|
||||
conv_state,
|
||||
recurrent_checkpoint,
|
||||
recurrent_state,
|
||||
cu_seqlens,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
raw_qkv.stride(0),
|
||||
raw_qkv.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
recurrent_checkpoint.stride(0),
|
||||
recurrent_state.stride(0),
|
||||
checkpoint.checkpoint_offsets.stride(0),
|
||||
state_len,
|
||||
width,
|
||||
recurrent_row_size,
|
||||
NULL_BLOCK_ID,
|
||||
block_size,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def store_cache_checkpoints_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
recurrent_checkpoint_ptr,
|
||||
recurrent_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
checkpoint_state_indices_ptr,
|
||||
x_stride_0: tl.constexpr,
|
||||
x_stride_1: tl.constexpr,
|
||||
state_stride_0: tl.constexpr,
|
||||
state_stride_1: tl.constexpr,
|
||||
state_stride_2: tl.constexpr,
|
||||
checkpoint_stride_0: tl.constexpr,
|
||||
recurrent_state_stride_0: tl.constexpr,
|
||||
checkpoint_offset_stride: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
RECURRENT_ROW_SIZE: tl.constexpr,
|
||||
NULL_STATE_IDX: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
seq_idx = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
state_idx = tl.load(checkpoint_state_indices_ptr + seq_idx).to(tl.int64)
|
||||
checkpoint_offset = tl.load(
|
||||
checkpoint_offsets_ptr + seq_idx * checkpoint_offset_stride
|
||||
)
|
||||
valid_checkpoint = (state_idx != NULL_STATE_IDX) & (checkpoint_offset > 0)
|
||||
valid_conv = (
|
||||
(cols < WIDTH * STATE_LEN) & valid_checkpoint & (checkpoint_offset >= STATE_LEN)
|
||||
)
|
||||
width_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
checkpoint_end = tl.load(query_start_loc_ptr + seq_idx) + checkpoint_offset
|
||||
token_idx = checkpoint_end - STATE_LEN + history_idx
|
||||
values = tl.load(
|
||||
x_ptr + token_idx * x_stride_0 + width_idx * x_stride_1,
|
||||
mask=valid_conv,
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ state_idx * state_stride_0
|
||||
+ width_idx * state_stride_1
|
||||
+ history_idx * state_stride_2,
|
||||
values,
|
||||
mask=valid_conv,
|
||||
)
|
||||
|
||||
valid_recurrent = (cols < RECURRENT_ROW_SIZE) & valid_checkpoint
|
||||
recurrent = tl.load(
|
||||
recurrent_checkpoint_ptr + seq_idx * checkpoint_stride_0 + cols,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
tl.store(
|
||||
recurrent_state_ptr + state_idx * recurrent_state_stride_0 + cols,
|
||||
recurrent,
|
||||
mask=valid_recurrent,
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
@@ -325,6 +326,7 @@ class MiniMaxText01LinearAttention(LinearAttention):
|
||||
output[:num_actual_tokens], _ = self.out_proj(hidden)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def linear_attention(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -465,6 +466,7 @@ def split_batch_to_prefill_and_decode(
|
||||
)
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -8,6 +8,7 @@ from dataclasses import replace
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
@@ -811,10 +812,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - It will read the initial states for every sequence,
|
||||
# that has "has_initial_states_p" == True,
|
||||
# from "cache_indices", using "state_indices_tensor_p".
|
||||
# - It updates the "conv_state" cache in positions pointed
|
||||
# to by "state_indices_tensor_p".
|
||||
# In particular, it will always write the state at the
|
||||
# sequence end.
|
||||
# - It writes the state at the sequence end to the "conv_state"
|
||||
# cache in positions pointed to by "state_indices_tensor_p".
|
||||
x = hidden_states_B_C_p.transpose(
|
||||
0, 1
|
||||
) # this is the form that causal-conv see
|
||||
@@ -891,6 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
state_len=self.conv_kernel_size - 1,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
@@ -1214,6 +1214,7 @@ def share_replayssm_ring_trackers(
|
||||
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def mamba_mixer2(
|
||||
projected_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -238,13 +238,14 @@ class MambaStateShapeCalculator:
|
||||
"""Append the physical ReplaySSM ring shapes.
|
||||
|
||||
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
|
||||
B_cache uses the un-extended ``n_groups``.
|
||||
B_cache extends ``n_groups`` for head shards as in ``mamba2_state_shape``.
|
||||
"""
|
||||
ring_buffer_len = logical_window
|
||||
if backend == MambaBackendEnum.FLASHINFER:
|
||||
# FlashInfer keeps the live window and current verify window together.
|
||||
ring_buffer_len += 1 + num_speculative_tokens
|
||||
local_nheads, head_dim, state_size = base_shapes[1]
|
||||
n_groups = n_groups + cls.extra_groups_for_head_shards(n_groups, tp_world_size)
|
||||
local_ngroups = divide(n_groups, tp_world_size)
|
||||
return (
|
||||
*base_shapes,
|
||||
|
||||
@@ -20,17 +20,11 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
w_ptr, # (dim, width)
|
||||
bias_ptr,
|
||||
initial_states_ptr, # conv_states_ptr
|
||||
cache_indices_ptr, # (batch, n_blocks + padding) The second dimension contains
|
||||
# the block indices relevant for each sequence
|
||||
# plus potential 0-padding at the beginning and at the end
|
||||
cache_indices_ptr, # (batch,)
|
||||
has_initial_states_ptr,
|
||||
query_start_loc_ptr,
|
||||
batch_ptr,
|
||||
token_chunk_offset_ptr,
|
||||
block_idx_first_scheduled_token, # (batch,)
|
||||
block_idx_last_scheduled_token, # (batch,)
|
||||
initial_state_idx, # (batch,)
|
||||
num_computed_tokens, # (batch,)
|
||||
o_ptr, # (dim, seqlen) - actually pointing to x_ptr
|
||||
# Matrix dimensions
|
||||
dim: tl.constexpr,
|
||||
@@ -46,7 +40,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
stride_cache_indices: tl.constexpr,
|
||||
stride_o_dim: tl.constexpr,
|
||||
stride_o_token: tl.int64,
|
||||
stride_block_m: tl.constexpr, # Stride block to align divided by BLOCK_M
|
||||
# others
|
||||
pad_slot_id: tl.constexpr,
|
||||
null_block_id: tl.constexpr,
|
||||
@@ -54,7 +47,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
HAS_BIAS: tl.constexpr,
|
||||
KERNEL_WIDTH: tl.constexpr,
|
||||
SILU_ACTIVATION: tl.constexpr,
|
||||
IS_APC_ENABLED: tl.constexpr,
|
||||
HAS_NULL_BLOCK: tl.constexpr,
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
@@ -93,40 +85,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
# find the actual sequence length
|
||||
seqlen = sequence_end_index - sequence_start_index
|
||||
|
||||
B_size: tl.constexpr = stride_block_m * BLOCK_M
|
||||
|
||||
if IS_APC_ENABLED:
|
||||
# Handle the case if prefix caching is enabled.
|
||||
# In particular, if prefix caching is enabled, the program write additional cache states to "cache_indices_ptr"
|
||||
|
||||
# Get the length of the completed sequence so far and compute the offset.
|
||||
current_first_index = tl.load(block_idx_first_scheduled_token + idx_seq)
|
||||
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
|
||||
sequence_completed_index = tl.load(num_computed_tokens + idx_seq)
|
||||
|
||||
# Compute the offset where the first stride_block_m-aligned first full block is
|
||||
# Value in "token-space"
|
||||
sequence_completed_offset_token = sequence_completed_index % B_size
|
||||
seq_completed_offset = B_size - sequence_completed_offset_token
|
||||
seq_end_offset = (seqlen - seq_completed_offset) % B_size
|
||||
last_full_block_token_index = sequence_end_index - seq_end_offset
|
||||
# If the sequence without the sequence_offset_index is stride_cache_chunk-aligned, then the last full chunk is the second-to-last one
|
||||
if seq_end_offset == 0:
|
||||
last_full_block_token_index = last_full_block_token_index - B_size
|
||||
|
||||
# Get the number of blocks to be filled for the current sequence
|
||||
# If n_block_to_fill = 0, then only the state at the sequence end is stored
|
||||
n_block_to_fill = current_last_index - current_first_index
|
||||
|
||||
# Get the index of the init block
|
||||
conv_state_init_index = tl.load(initial_state_idx + idx_seq)
|
||||
else:
|
||||
n_block_to_fill = 0
|
||||
current_last_index = 0
|
||||
conv_state_init_index = 0
|
||||
current_first_index = 0
|
||||
last_full_block_token_index = 0
|
||||
|
||||
token_offset = BLOCK_M * chunk_offset
|
||||
segment_len = min(BLOCK_M, seqlen - token_offset)
|
||||
|
||||
@@ -137,7 +95,7 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
|
||||
# cache_idx
|
||||
conv_states_input_coord = tl.load(
|
||||
conv_state_indices_ptr + idx_seq * stride_cache_indices + conv_state_init_index
|
||||
conv_state_indices_ptr + idx_seq * stride_cache_indices
|
||||
).to(tl.int64)
|
||||
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
@@ -224,20 +182,10 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
|
||||
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
|
||||
|
||||
# Compute the offset where the last block should be written in the conv_states
|
||||
conv_states_output_coord = tl.load(
|
||||
conv_state_indices_ptr
|
||||
+ idx_seq * stride_cache_indices
|
||||
+ current_last_index
|
||||
).to(tl.int64)
|
||||
|
||||
conv_states_ptrs_target = (
|
||||
conv_states_ptr
|
||||
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
|
||||
+ (idx_feats * stride_conv_state_dim)
|
||||
)[None, :] + ( # [BLOCK_N,]
|
||||
idx_tokens_conv * stride_conv_state_tok
|
||||
)[:, None]
|
||||
conv_states_base[None, :]
|
||||
+ (idx_tokens_conv * stride_conv_state_tok)[:, None]
|
||||
) # [BLOCK_M, BLOCK_N]
|
||||
|
||||
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
|
||||
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
|
||||
@@ -346,52 +294,6 @@ def _causal_conv1d_fwd_kernel( # continuous batching
|
||||
conv_states_ptrs = prior_tokens - 3 * stride_x_token # [BLOCK_N]
|
||||
col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca")
|
||||
|
||||
# Store intermediate states aligned with stride_block_m
|
||||
# The additional states are cached starting from the last stride_block_m.
|
||||
# For example:
|
||||
# If n_block_to_fill = 0, then only the state at the sequence end is cached and the process below is not involved.
|
||||
# If n_block_to_fill > 0, then the states at the sequence end and at the n_block_to_fill-last
|
||||
# stride_block_m are cached.
|
||||
# For example chunk_offset = n_block_to_fill stores the state at last_full_block
|
||||
if (chunk_offset - 1) < n_block_to_fill:
|
||||
# Store the states at the chunk boundaries from the start of the sequence
|
||||
idx_tokens_last = (
|
||||
last_full_block_token_index
|
||||
- (n_block_to_fill - chunk_offset) * B_size
|
||||
- state_len
|
||||
) + tl.arange(0, NP2_STATELEN) # [BLOCK_M]
|
||||
x_ptrs = (
|
||||
x_ptr
|
||||
+ (idx_tokens_last * stride_x_token)[:, None]
|
||||
+ (idx_feats * stride_x_dim)[None, :]
|
||||
) # [BLOCK_M,BLOCK_N,]
|
||||
|
||||
mask_x = (idx_tokens_last >= 0)[:, None] & (idx_feats < dim)[
|
||||
None, :
|
||||
] # token-index # token-index # feature-index
|
||||
loaded_x = tl.load(x_ptrs, mask_x, 0.0)
|
||||
idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M]
|
||||
|
||||
# cache_idx
|
||||
conv_states_output_coord = tl.load(
|
||||
conv_state_indices_ptr
|
||||
+ idx_seq * stride_cache_indices
|
||||
+ current_first_index
|
||||
+ (chunk_offset - 1)
|
||||
).to(tl.int64)
|
||||
|
||||
conv_states_ptrs_target = (
|
||||
conv_states_ptr
|
||||
+ (conv_states_output_coord * stride_conv_state_seq) # Offset from seq
|
||||
+ (idx_feats * stride_conv_state_dim)
|
||||
)[None, :] + ( # [BLOCK_N,]
|
||||
idx_tokens_conv * stride_conv_state_tok
|
||||
)[:, None]
|
||||
|
||||
mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :]
|
||||
tl.debug_barrier() # NOTE: use this due to bug in Triton compiler
|
||||
tl.store(conv_states_ptrs_target, loaded_x, mask)
|
||||
|
||||
if HAS_BIAS:
|
||||
bias = bias_ptr + idx_feats
|
||||
mask_bias = idx_feats < dim
|
||||
@@ -489,11 +391,6 @@ def causal_conv1d_fn(
|
||||
activation: str | None = "silu",
|
||||
pad_slot_id: int = PAD_SLOT_ID,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
block_idx_first_scheduled_token: torch.Tensor | None = None,
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
num_computed_tokens: torch.Tensor | None = None,
|
||||
block_size_to_align=0,
|
||||
metadata=None,
|
||||
validate_data=False,
|
||||
):
|
||||
@@ -536,16 +433,6 @@ def causal_conv1d_fn(
|
||||
for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id]
|
||||
in this case, the kernel will not process entries at
|
||||
indices 0 and 3
|
||||
block_idx_first_scheduled_token: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the first cache block to be filled is located.
|
||||
block_idx_last_scheduled_token: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the last cache block to be filled is located.
|
||||
initial_state_idx: (batch,), dtype int32
|
||||
The pointer into cache_indices, where the cache block containing the initial state is located.
|
||||
num_computed_tokens: (batch,), dtype int32
|
||||
The number of tokens already completed for each sequence
|
||||
block_size_to_align: int
|
||||
The block size to align the cached states to
|
||||
out: same shape as `x`
|
||||
"""
|
||||
if isinstance(activation, bool) and activation:
|
||||
@@ -631,12 +518,6 @@ def causal_conv1d_fn(
|
||||
assert weight.stride(1) == 1
|
||||
assert (dim, width) == weight.shape
|
||||
assert is_channel_last, "Need to run in channel-last layout"
|
||||
if block_size_to_align is not None and block_size_to_align > 0:
|
||||
assert (block_size_to_align % BLOCK_M) == 0, (
|
||||
"The mamba block size needs to be divisible by the BLOCK_M"
|
||||
)
|
||||
else:
|
||||
block_size_to_align = BLOCK_M
|
||||
|
||||
if metadata is None:
|
||||
|
||||
@@ -720,10 +601,6 @@ def causal_conv1d_fn(
|
||||
query_start_loc,
|
||||
batch_ptr,
|
||||
token_chunk_offset_ptr,
|
||||
block_idx_first_scheduled_token,
|
||||
block_idx_last_scheduled_token,
|
||||
initial_state_idx,
|
||||
num_computed_tokens,
|
||||
out,
|
||||
# Matrix dimensions
|
||||
dim,
|
||||
@@ -739,7 +616,6 @@ def causal_conv1d_fn(
|
||||
stride_cache_indices,
|
||||
stride_o_dim,
|
||||
stride_o_token,
|
||||
block_size_to_align // BLOCK_M,
|
||||
# others
|
||||
pad_slot_id,
|
||||
null_block_id,
|
||||
@@ -747,7 +623,6 @@ def causal_conv1d_fn(
|
||||
HAS_BIAS=bias is not None,
|
||||
KERNEL_WIDTH=width,
|
||||
SILU_ACTIVATION=activation in ["silu", "swish"],
|
||||
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
|
||||
HAS_NULL_BLOCK=null_block_id is not None,
|
||||
NP2_STATELEN=np2_statelen,
|
||||
# launch_cooperative_grid=True
|
||||
@@ -769,8 +644,6 @@ def _causal_conv1d_update_kernel(
|
||||
conv_state_indices_ptr,
|
||||
num_accepted_tokens_ptr,
|
||||
query_start_loc_ptr, # (batch + 1)
|
||||
block_idx_last_scheduled_token, # (batch,)
|
||||
initial_state_idx, # (batch,)
|
||||
o_ptr, # (batch, dim, seqlen)
|
||||
# Matrix dimensions
|
||||
batch: int,
|
||||
@@ -798,7 +671,6 @@ def _causal_conv1d_update_kernel(
|
||||
KERNEL_WIDTH: tl.constexpr,
|
||||
SILU_ACTIVATION: tl.constexpr,
|
||||
IS_VARLEN: tl.constexpr,
|
||||
IS_APC_ENABLED: tl.constexpr,
|
||||
IS_SPEC_DECODING: tl.constexpr,
|
||||
NP2_STATELEN: tl.constexpr,
|
||||
HAS_NULL_BLOCK: tl.constexpr,
|
||||
@@ -818,17 +690,9 @@ def _causal_conv1d_update_kernel(
|
||||
# [BLOCK_N,] elements along the feature-dimension (channel)
|
||||
idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
|
||||
if IS_APC_ENABLED:
|
||||
# Get the state from the initial_state_idx
|
||||
conv_state_init = tl.load(initial_state_idx + idx_seq)
|
||||
current_last_index = tl.load(block_idx_last_scheduled_token + idx_seq)
|
||||
else:
|
||||
conv_state_init = 0
|
||||
current_last_index = 0
|
||||
|
||||
# cache_idx
|
||||
conv_states_input_coord = tl.load(
|
||||
conv_state_indices_ptr + idx_seq * stride_state_indices + conv_state_init
|
||||
conv_state_indices_ptr + idx_seq * stride_state_indices
|
||||
).to(tl.int64)
|
||||
|
||||
if HAS_NULL_BLOCK: # noqa
|
||||
@@ -941,14 +805,9 @@ def _causal_conv1d_update_kernel(
|
||||
|
||||
new_conv_state = tl.where(mask, conv_state, loaded_x)
|
||||
|
||||
# Get the state from the initial_state_idx
|
||||
# cache_idx
|
||||
conv_states_offset = tl.load(
|
||||
conv_state_indices_ptr + idx_seq * stride_state_indices + current_last_index
|
||||
).to(tl.int64)
|
||||
conv_state_ptrs_target = (
|
||||
conv_state_ptr
|
||||
+ (conv_states_offset * stride_conv_state_seq) # Offset from seq
|
||||
+ (conv_states_input_coord * stride_conv_state_seq) # Offset from seq
|
||||
+ (idx_feats * stride_conv_state_dim)
|
||||
)[None, :] + ( # [BLOCK_N,]
|
||||
idx_tokens * stride_conv_state_tok
|
||||
@@ -1104,8 +963,6 @@ def causal_conv1d_update(
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
max_query_len: int = -1,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
block_idx_last_scheduled_token: torch.Tensor | None = None,
|
||||
initial_state_idx: torch.Tensor | None = None,
|
||||
validate_data=False,
|
||||
out: torch.Tensor | None = None,
|
||||
):
|
||||
@@ -1123,10 +980,6 @@ def causal_conv1d_update(
|
||||
If not None, the conv_state is a larger tensor along the batch dim,
|
||||
and we are selecting the batch coords specified by conv_state_indices.
|
||||
Useful for a continuous batching scenario.
|
||||
block_idx_last_scheduled_token: (batch,), dtype int32
|
||||
The pointer into conv_state_indices, where the last cache block to be filled is located.
|
||||
initial_state_idx: (batch,), dtype int32
|
||||
The pointer into conv_state_indices, where the cache block containing the initial state is located.
|
||||
num_accepted_tokens: (batch,), dtype int32
|
||||
If not None, it indicates the number of accepted tokens for each
|
||||
sequence in the batch.
|
||||
@@ -1237,8 +1090,6 @@ def causal_conv1d_update(
|
||||
conv_state_indices,
|
||||
num_accepted_tokens,
|
||||
query_start_loc,
|
||||
block_idx_last_scheduled_token,
|
||||
initial_state_idx,
|
||||
out,
|
||||
# Matrix dimensions
|
||||
batch,
|
||||
@@ -1266,7 +1117,6 @@ def causal_conv1d_update(
|
||||
KERNEL_WIDTH=width,
|
||||
SILU_ACTIVATION=activation in ["silu", "swish"],
|
||||
IS_VARLEN=query_start_loc is not None,
|
||||
IS_APC_ENABLED=block_idx_last_scheduled_token is not None,
|
||||
IS_SPEC_DECODING=num_accepted_tokens is not None,
|
||||
NP2_STATELEN=np2_statelen,
|
||||
HAS_NULL_BLOCK=null_block_id is not None,
|
||||
|
||||
@@ -69,6 +69,8 @@ def store_prefill_checkpoint(
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
*,
|
||||
state_len: int,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
@@ -78,14 +80,15 @@ def store_prefill_checkpoint(
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len + num_spec)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
state_len: ``conv_kernel_size - 1``, not the spec-widened conv width.
|
||||
Readers take the first ``state_len`` slots as the newest inputs.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -360,6 +361,7 @@ class ShortConv(MambaBase, PluggableLayer):
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
|
||||
@eager_break_during_capture
|
||||
def short_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import array
|
||||
import contextlib
|
||||
import ctypes
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
@@ -16,6 +17,7 @@ except ImportError:
|
||||
from cuda import cudart
|
||||
|
||||
_ALIGN = 1 << 21 # 2 MiB — CUDA IPC allocation alignment
|
||||
_IPC_HANDLE_SIZE = 64 # sizeof(cudaIpcMemHandle_t)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -30,6 +32,20 @@ def _check(error):
|
||||
raise RuntimeError(f"CUDA runtime error: {error}")
|
||||
|
||||
|
||||
# cuda-bindings >= 13.4 no longer exposes cudaIpcMemHandle_t.reserved, so
|
||||
# copy the raw handle bytes through getPtr() instead. This is safe because the
|
||||
# CUDA ABI fixes the handle as an opaque 64-byte struct (char reserved[64]).
|
||||
def _ipc_handle_to_bytes(handle) -> bytes:
|
||||
return ctypes.string_at(handle.getPtr(), _IPC_HANDLE_SIZE)
|
||||
|
||||
|
||||
def _ipc_handle_from_bytes(raw: bytes):
|
||||
assert len(raw) == _IPC_HANDLE_SIZE, len(raw)
|
||||
handle = cudart.cudaIpcMemHandle_t()
|
||||
ctypes.memmove(handle.getPtr(), raw, _IPC_HANDLE_SIZE)
|
||||
return handle
|
||||
|
||||
|
||||
def _cuda_malloc(size: int):
|
||||
aligned = ((size + _ALIGN - 1) >> 21) << 21
|
||||
err, ptr = cudart.cudaMalloc(aligned)
|
||||
@@ -84,15 +100,16 @@ class IpcBuffer:
|
||||
|
||||
all_handles: list[bytes | None] = [None] * world_size
|
||||
torch.distributed.all_gather_object(
|
||||
all_handles, bytes(local_handle.reserved), group=process_group
|
||||
all_handles, _ipc_handle_to_bytes(local_handle), group=process_group
|
||||
)
|
||||
|
||||
for r in range(world_size):
|
||||
if r == rank:
|
||||
self.peer_ptrs[r] = self.local_ptr
|
||||
else:
|
||||
handle = cudart.cudaIpcMemHandle_t()
|
||||
handle.reserved = all_handles[r]
|
||||
raw = all_handles[r]
|
||||
assert raw is not None
|
||||
handle = _ipc_handle_from_bytes(raw)
|
||||
err, ptr = cudart.cudaIpcOpenMemHandle(
|
||||
handle, cudart.cudaIpcMemLazyEnablePeerAccess
|
||||
)
|
||||
|
||||
@@ -198,6 +198,11 @@ class BOSEOSFilter(Pooler):
|
||||
for i, (prompt_len, token_ids) in enumerate(
|
||||
zip(pooling_metadata.prompt_lens, prompt_token_ids)
|
||||
):
|
||||
# Late chunking returns one row per chunk, with special tokens
|
||||
# already included in the means. Skip BOS/EOS row removal for these
|
||||
# requests; ordinary token outputs still use the filtering below.
|
||||
if pooling_metadata.pooling_params[i].late_chunking_params is not None:
|
||||
continue
|
||||
pooled_data = pooled_outputs[i]
|
||||
assert (
|
||||
isinstance(pooled_data, torch.Tensor)
|
||||
|
||||
@@ -17,6 +17,23 @@ from .methods import TokenPoolingMethodOutputItem
|
||||
TokenPoolerHeadOutputItem: TypeAlias = torch.Tensor | None
|
||||
|
||||
|
||||
def _mean_pool_chunks(data: torch.Tensor, chunk_size: int) -> torch.Tensor:
|
||||
"""Mean-pool complete contextual states, accumulating in FP32."""
|
||||
chunk_size = min(chunk_size, len(data))
|
||||
num_full, remainder = divmod(len(data), chunk_size)
|
||||
full_end = num_full * chunk_size
|
||||
full_chunks = data[:full_end].reshape(num_full, chunk_size, data.shape[-1])
|
||||
if not remainder:
|
||||
return full_chunks.mean(dim=1, dtype=torch.float32)
|
||||
|
||||
return torch.cat(
|
||||
[
|
||||
full_chunks.mean(dim=1, dtype=torch.float32),
|
||||
data[full_end:].mean(dim=0, keepdim=True, dtype=torch.float32),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TokenPoolerHead(nn.Module, ABC):
|
||||
@abstractmethod
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
@@ -80,6 +97,12 @@ class TokenEmbeddingPoolerHead(TokenPoolerHead):
|
||||
if pooled_data is None:
|
||||
return None
|
||||
|
||||
if (late_chunking := pooling_param.late_chunking_params) is not None:
|
||||
means = _mean_pool_chunks(pooled_data, late_chunking.chunk_size)
|
||||
pooled_data = (
|
||||
means.to(pooled_data.dtype) if self.head_dtype is None else means
|
||||
)
|
||||
|
||||
if self.head_dtype is not None:
|
||||
pooled_data = pooled_data.to(self.head_dtype)
|
||||
# pooled_data shape: [n_tokens, hidden_size]
|
||||
|
||||
@@ -84,11 +84,14 @@ class QuantizeMethodBase(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
|
||||
def dequantize_weight(
|
||||
self, layer: nn.Module
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Materialize a serialized quantized weight for requantization."""
|
||||
raise NotImplementedError(
|
||||
f"The quantization method {type(self)} does not implement "
|
||||
"dequantize_weight. Please open an issue."
|
||||
"dequantize_weight, so re-quantization from this checkpoint is "
|
||||
"not supported. Please open an issue."
|
||||
)
|
||||
|
||||
|
||||
@@ -302,6 +305,9 @@ def resolve_quant_method(
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
@@ -328,11 +334,6 @@ def resolve_quant_method(
|
||||
# The checkpoint quant method is applied as there is no online override.
|
||||
return base_quant_method
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
|
||||
online_quant_method = quant_config.online_quantization_config.get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
@@ -341,7 +342,7 @@ def resolve_quant_method(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
|
||||
assert isinstance(online_quant_method, OnlineLinearBase)
|
||||
assert isinstance(online_quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
online_quant_method.set_requantization_source(base_quant_method)
|
||||
|
||||
# The online method dequantizes the checkpoint method before requantizing.
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsW4A4Mxfp4,
|
||||
CompressedTensorsW4A8Fp8,
|
||||
CompressedTensorsW4A8Int,
|
||||
CompressedTensorsW4A8Mxfp4,
|
||||
CompressedTensorsW8A8Fp8,
|
||||
CompressedTensorsW8A8Int8,
|
||||
CompressedTensorsW8A8Mxfp8,
|
||||
@@ -63,6 +64,11 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
create_fp8_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
@@ -449,6 +455,39 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
and is_symmetric
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _is_mxfp4_w4a8_fp8(
|
||||
cls, weight_quant: QuantizationArgs, input_quant: QuantizationArgs
|
||||
) -> bool:
|
||||
if not cls._is_mxfp4(weight_quant) or input_quant is None:
|
||||
return False
|
||||
|
||||
is_float_type = input_quant.type == QuantizationType.FLOAT
|
||||
is_8_bits = input_quant.num_bits == 8
|
||||
is_static_weight = not weight_quant.dynamic
|
||||
is_dynamic_input = bool(input_quant.dynamic)
|
||||
is_token_or_group = input_quant.strategy in (
|
||||
QuantizationStrategy.TOKEN,
|
||||
QuantizationStrategy.GROUP,
|
||||
)
|
||||
|
||||
return (
|
||||
is_float_type
|
||||
and is_8_bits
|
||||
and is_static_weight
|
||||
and is_dynamic_input
|
||||
and is_token_or_group
|
||||
and input_quant.symmetric
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fp8_dynamic_input_key(input_quant: QuantizationArgs) -> QuantKey:
|
||||
if input_quant.strategy == QuantizationStrategy.GROUP:
|
||||
group_shape = GroupShape(1, input_quant.group_size)
|
||||
else:
|
||||
group_shape = GroupShape.PER_TOKEN
|
||||
return create_fp8_quant_key(static=False, group_shape=group_shape)
|
||||
|
||||
@staticmethod
|
||||
def _is_mxfp8(quant_args: QuantizationArgs) -> bool:
|
||||
if quant_args is None:
|
||||
@@ -764,6 +803,11 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
)
|
||||
return CompressedTensorsW4A4Fp4()
|
||||
|
||||
if self._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
|
||||
return CompressedTensorsW4A8Mxfp4(
|
||||
activation_quant_key=self._fp8_dynamic_input_key(input_quant)
|
||||
)
|
||||
|
||||
if self._is_mxfp4(weight_quant):
|
||||
return CompressedTensorsW4A4Mxfp4()
|
||||
|
||||
|
||||
+16
-2
@@ -34,8 +34,12 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
# make sure quantization config for Linear can target it
|
||||
quant_config._add_fused_moe_to_target_scheme_map()
|
||||
unfused_names = [
|
||||
layer_name + proj_name
|
||||
for proj_name in [".0.gate_proj", ".0.up_proj", ".0.down_proj"]
|
||||
f"{layer_name}.0.{proj_name}"
|
||||
for proj_name in (
|
||||
layer.ckpt_gate_proj_name,
|
||||
layer.ckpt_up_proj_name,
|
||||
layer.ckpt_down_proj_name,
|
||||
)
|
||||
]
|
||||
# TODO: refactor this to use expert_mapping and check all layer numbers
|
||||
all_scheme_dicts = [
|
||||
@@ -59,6 +63,16 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
input_quant = scheme_dict.get("input_activations")
|
||||
format = scheme_dict.get("format")
|
||||
|
||||
if quant_config._is_mxfp4_w4a8_fp8(weight_quant, input_quant):
|
||||
from .compressed_tensors_moe_w4a8_mxfp4 import (
|
||||
CompressedTensorsW4A8Mxfp4MoEMethod,
|
||||
)
|
||||
|
||||
return CompressedTensorsW4A8Mxfp4MoEMethod(
|
||||
layer.moe_config,
|
||||
activation_key=quant_config._fp8_dynamic_input_key(input_quant),
|
||||
)
|
||||
|
||||
if quant_config._is_mxfp4(weight_quant):
|
||||
from .compressed_tensors_moe_w4a4_mxfp4 import (
|
||||
CompressedTensorsW4A4Mxfp4MoEMethod,
|
||||
|
||||
+19
@@ -26,6 +26,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_nvfp4_moe_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
@@ -185,6 +188,22 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
)
|
||||
set_weight_attrs(w2_input_scale, extra_weight_attrs)
|
||||
|
||||
def dequantize_weight(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Materialize serialized NVFP4 expert weights for requantization."""
|
||||
return dequantize_nvfp4_moe_weights(
|
||||
layer.w13_weight_packed,
|
||||
layer.w13_weight_scale,
|
||||
layer.w13_weight_global_scale,
|
||||
layer.w2_weight_packed,
|
||||
layer.w2_weight_scale,
|
||||
layer.w2_weight_global_scale,
|
||||
layer.params_dtype,
|
||||
group_size=self.group_size,
|
||||
invert_global_scales=True,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
|
||||
# NOTE(rob): wN_weight_packed -> wN_weight is because ModularKernelMethod
|
||||
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
|
||||
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
|
||||
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
|
||||
|
||||
Only the checkpoint naming and the declared activation scheme differ from
|
||||
the native MXFP4 path, so backend selection, weight conversion and
|
||||
execution are inherited.
|
||||
"""
|
||||
|
||||
# The packed -> unpacked rename means a processed state dict no longer
|
||||
# matches the parameters registered by create_weights.
|
||||
supports_pre_processed_weights = False
|
||||
|
||||
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
|
||||
super().create_weights(layer, *args, **kwargs)
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, name)
|
||||
delattr(layer, name)
|
||||
layer.register_parameter(f"{name}_packed", param)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, f"{name}_packed")
|
||||
delattr(layer, f"{name}_packed")
|
||||
layer.register_parameter(name, param)
|
||||
super().process_weights_after_loading(layer)
|
||||
@@ -6,6 +6,7 @@ from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
|
||||
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
|
||||
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
|
||||
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
|
||||
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
|
||||
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
|
||||
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
|
||||
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
|
||||
@@ -23,6 +24,7 @@ __all__ = [
|
||||
"CompressedTensorsW8A8Int8",
|
||||
"CompressedTensorsW8A8Fp8",
|
||||
"CompressedTensorsW4A4Mxfp4",
|
||||
"CompressedTensorsW4A8Mxfp4",
|
||||
"CompressedTensorsW4A4Fp4",
|
||||
"CompressedTensorsW4A8Int",
|
||||
"CompressedTensorsW4A8Fp8",
|
||||
|
||||
+3
-2
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
CompressedTensorsScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
@@ -35,10 +36,10 @@ class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
|
||||
Otherwise: W4A16 weight-only via Marlin.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, activation_quant_key: QuantKey = kMxfp4Dynamic):
|
||||
self.group_size = 32
|
||||
self.kernel = init_mxfp4_linear_kernel(
|
||||
activation_quant_key=kMxfp4Dynamic,
|
||||
activation_quant_key=activation_quant_key,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxfp4 import ( # noqa: E501
|
||||
CompressedTensorsW4A4Mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = ["CompressedTensorsW4A8Mxfp4"]
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Mxfp4(CompressedTensorsW4A4Mxfp4):
|
||||
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
|
||||
|
||||
The weight layout matches W4A4; only the activation quantization differs.
|
||||
The checkpoint's FP8 activations are used when a kernel supports them,
|
||||
otherwise this falls back to the W4A4 activation format.
|
||||
"""
|
||||
|
||||
def __init__(self, activation_quant_key: QuantKey):
|
||||
try:
|
||||
super().__init__(activation_quant_key=activation_quant_key)
|
||||
except ValueError as e:
|
||||
logger.warning_once(
|
||||
"No MXFP4 linear kernel supports %s activations on this "
|
||||
"platform; falling back to MXFP4 activations. %s",
|
||||
activation_quant_key,
|
||||
e,
|
||||
)
|
||||
super().__init__()
|
||||
+55
-10
@@ -6,11 +6,15 @@ from collections.abc import Callable
|
||||
from fractions import Fraction
|
||||
|
||||
import torch
|
||||
from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.utils import verify_group_size_divides_partition
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.determinism.batch_invariant import matmul_batch_invariant
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MarlinLinearKernel,
|
||||
MPLinearKernel,
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
@@ -28,6 +32,7 @@ from vllm.model_executor.parameter import (
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -71,6 +76,11 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
self.symmetric = symmetric
|
||||
self.group_size = -1 if group_size is None else group_size
|
||||
self.layer_name = layer_name
|
||||
# Marlin picks its tiling and K-split from the number of tokens, so its
|
||||
# results depend on the batch size. In batch-invariant mode keep the
|
||||
# checkpoint layout, unpack the weight and use the batch-invariant matmul.
|
||||
self.use_batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
self.kernel: MPLinearKernel | None = None
|
||||
|
||||
if self.group_size == -1 and self.strategy != "channel":
|
||||
raise ValueError(
|
||||
@@ -132,11 +142,16 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
zero_points=not self.symmetric,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
if self.use_batch_invariant:
|
||||
kernel_type = None
|
||||
backend_name = "dequantize + batch-invariant matmul"
|
||||
else:
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
backend_name = kernel_type.__name__
|
||||
|
||||
if kernel_type.__name__ not in self._kernel_backends_being_used:
|
||||
logger.info("Using %s for CompressedTensorsWNA16", kernel_type.__name__)
|
||||
self._kernel_backends_being_used.add(kernel_type.__name__)
|
||||
if backend_name not in self._kernel_backends_being_used:
|
||||
logger.info("Using %s for CompressedTensorsWNA16", backend_name)
|
||||
self._kernel_backends_being_used.add(backend_name)
|
||||
|
||||
if kernel_type is MarlinLinearKernel:
|
||||
input_dtype = get_marlin_input_dtype(self.layer_name)
|
||||
@@ -227,19 +242,49 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
if not self.symmetric:
|
||||
layer.register_parameter("weight_zero_point", qzeros)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_linear_kernel_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
)
|
||||
if kernel_type is not None:
|
||||
self.kernel = kernel_type(
|
||||
mp_linear_kernel_config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
)
|
||||
|
||||
# Checkpoints are serialized in compressed-tensors format, which is
|
||||
# different from the format the kernel may want. Handle repacking here.
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if self.use_batch_invariant:
|
||||
# Keep the checkpoint layout: Marlin's reduction depends on M.
|
||||
for name in ("weight_packed", "weight_scale", "weight_zero_point"):
|
||||
if hasattr(layer, name):
|
||||
replace_parameter(layer, name, getattr(layer, name).data)
|
||||
return
|
||||
assert self.kernel is not None
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
if self.use_batch_invariant:
|
||||
shape = torch.Size(
|
||||
(layer.output_size_per_partition, layer.input_size_per_partition)
|
||||
)
|
||||
weight = unpack_from_int32(layer.weight_packed, self.num_bits, shape)
|
||||
weight = weight.to(layer.weight_scale.dtype).reshape(
|
||||
shape[0], layer.weight_scale.shape[1], -1
|
||||
)
|
||||
if not self.symmetric:
|
||||
zero_point = unpack_from_int32(
|
||||
layer.weight_zero_point,
|
||||
self.num_bits,
|
||||
layer.weight_scale.shape,
|
||||
packed_dim=0,
|
||||
)
|
||||
weight = weight - zero_point.unsqueeze(-1)
|
||||
weight = (weight * layer.weight_scale.unsqueeze(-1)).reshape(shape)
|
||||
output = matmul_batch_invariant(x, weight.t().to(x.dtype))
|
||||
if bias is not None:
|
||||
output.add_(bias)
|
||||
return output
|
||||
assert self.kernel is not None
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
|
||||
@@ -150,16 +150,16 @@ def scaled_mm_kernel(
|
||||
scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1))
|
||||
accumulator = scale_b.T * accumulator.to(tl.float32)
|
||||
|
||||
# Convert to output format.
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
|
||||
# Add bias, it's already in output format, so add it after conversion.
|
||||
# Add bias in accumulator dtype for less rounding error.
|
||||
if bias_ptr:
|
||||
offsets_bias = offsets_bn
|
||||
bias_ptrs = bias_ptr + offsets_bias
|
||||
bias_mask = offsets_bias < N
|
||||
bias = tl.load(bias_ptrs, bias_mask)
|
||||
c += bias
|
||||
accumulator += bias
|
||||
|
||||
# Convert to output format.
|
||||
c = accumulator.to(c_ptr.type.element_ty)
|
||||
|
||||
# Save output
|
||||
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
|
||||
@@ -224,6 +224,8 @@ def triton_scaled_mm(
|
||||
tile_shape = (64, 128, 128)
|
||||
else:
|
||||
tile_shape = (128, 128, 128)
|
||||
else:
|
||||
tile_shape = (block_size_m, block_size_n, block_size_k)
|
||||
|
||||
block_size_m, block_size_n, block_size_k = tile_shape
|
||||
|
||||
|
||||
@@ -9,10 +9,10 @@ import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
fp8_scaled_mm_is_batch_invariant,
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -432,7 +432,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
bias,
|
||||
)
|
||||
else:
|
||||
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
|
||||
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
# per-tensor/channel: dequant to BF16 and run GEMM
|
||||
|
||||
@@ -584,7 +584,9 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
param_dtype=layer.param_dtype,
|
||||
allow_fallback=self.quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
self.layer_config = _hm.prepare_layer_config(
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
self.layer_config, tensors = _hm.prepare_and_transform_humming_tensors(
|
||||
tensors=source_tensors,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
weight_schema=self.weight_schema,
|
||||
@@ -595,8 +597,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
torch_dtype=layer.param_dtype,
|
||||
device=layer.weight.device,
|
||||
)
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
|
||||
name = self.input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
@@ -610,7 +610,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
setattr(layer, name, param)
|
||||
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = _hm.create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -244,10 +244,12 @@ class QuantFP8(CustomOp):
|
||||
|
||||
x_grouped = x.view(-1, num_groups, self.group_size)
|
||||
absmax = x_grouped.abs().max(dim=-1, keepdim=True)[0].float()
|
||||
scales_raw = absmax / _FP8_MAX
|
||||
if self.use_ue8m0:
|
||||
scales_raw = torch.exp2(torch.ceil(torch.log2(scales_raw)))
|
||||
scales = (scales_raw).clamp(min=_FP8_MIN_SCALING_FACTOR)
|
||||
scales_raw = (absmax / _FP8_MAX).clamp(min=_FP8_MIN_SCALING_FACTOR)
|
||||
scales = (
|
||||
torch.exp2(torch.ceil(torch.log2(scales_raw)))
|
||||
if self.use_ue8m0
|
||||
else scales_raw
|
||||
)
|
||||
|
||||
x_scaled = x_grouped / scales
|
||||
x_quant = x_scaled.clamp(_FP8_MIN, _FP8_MAX).to(_FP8_DTYPE)
|
||||
|
||||
@@ -81,6 +81,9 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_VALUE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_nvfp4_moe_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
QuantKey,
|
||||
@@ -99,7 +102,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
requantize_with_max_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
@@ -111,6 +117,7 @@ from vllm.model_executor.utils import (
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.utils.b12x import B12X_BLOCK_CODECS
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -885,6 +892,22 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
"""FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales."""
|
||||
return True
|
||||
|
||||
def dequantize_weight(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Materialize serialized ModelOpt NVFP4 expert weights."""
|
||||
return dequantize_nvfp4_moe_weights(
|
||||
layer.w13_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w13_weight_scale_2,
|
||||
layer.w2_weight,
|
||||
layer.w2_weight_scale,
|
||||
layer.w2_weight_scale_2,
|
||||
layer.params_dtype,
|
||||
group_size=self.quant_config.group_size,
|
||||
invert_global_scales=False,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -1559,6 +1582,19 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
super().__init__(exclude_modules)
|
||||
self.kv_cache_quant_method = kv_cache_quant_method
|
||||
self.quantized_layers = quantized_layers
|
||||
for prefix, recipe in quantized_layers.items():
|
||||
codec = recipe.get("quant_algo", "").lower()
|
||||
if codec in B12X_BLOCK_CODECS and any(
|
||||
recipe.get(key) != value
|
||||
for key, value in {
|
||||
"group_size": B12X_BLOCK_CODECS[codec][0],
|
||||
"block_payload_bytes": B12X_BLOCK_CODECS[codec][1],
|
||||
"packing": "ggml",
|
||||
}.items()
|
||||
):
|
||||
raise ValueError(
|
||||
f"unsupported {codec.upper()} block contract for {prefix}: {recipe}"
|
||||
)
|
||||
self.fp8_config = fp8_config
|
||||
self.nvfp4_config = nvfp4_config
|
||||
self.w4a16_nvfp4_config = w4a16_nvfp4_config
|
||||
@@ -1811,6 +1847,22 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
|
||||
quant_algo = self._resolve_quant_algo(prefix)
|
||||
|
||||
if quant_algo is not None and quant_algo.lower() in B12X_BLOCK_CODECS:
|
||||
from .modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
ModelOptBlockQuantMoEMethod,
|
||||
)
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return ModelOptBlockQuantMoEMethod(
|
||||
layer.moe_config, codec=quant_algo.lower()
|
||||
)
|
||||
if isinstance(layer, (LinearBase, VocabParallelEmbedding)):
|
||||
return ModelOptBlockQuantLinearMethod(codec=quant_algo.lower())
|
||||
raise ValueError(
|
||||
f"{quant_algo} requires a linear, embedding or routed experts"
|
||||
)
|
||||
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
# Per-prefix algo -> its sub-config, then the generic linear method.
|
||||
# resolve() reads group_size off whichever sub-config it is handed.
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ModelOpt safetensors block codecs with BF16 activations."""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.b12x import B12xExperts
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearMethodBase,
|
||||
register_weight_loader_v2_supported_method,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
from vllm.utils.b12x import B12X_BLOCK_CODECS, B12xWarmupUnit, get_b12x_blockscaled
|
||||
|
||||
|
||||
@torch.library.custom_op(
|
||||
"vllm::modelopt_block_embedding_chunked", mutates_args=("out",)
|
||||
)
|
||||
def _modelopt_block_embedding_chunked(
|
||||
weight: torch.Tensor, ids: torch.Tensor, out: torch.Tensor, plan_handle: int
|
||||
) -> None:
|
||||
from b12x.preparation.types import plan_from_handle
|
||||
from b12x.sequence import embedding
|
||||
|
||||
plan = plan_from_handle(plan_handle)
|
||||
capacity = plan.query.max_rows
|
||||
for start in range(0, ids.numel(), capacity):
|
||||
embedding.run(
|
||||
weight,
|
||||
ids[start : start + capacity],
|
||||
out=out[start : start + capacity],
|
||||
plan=plan,
|
||||
)
|
||||
|
||||
|
||||
@_modelopt_block_embedding_chunked.register_fake
|
||||
def _modelopt_block_embedding_chunked_fake(
|
||||
weight: torch.Tensor, ids: torch.Tensor, out: torch.Tensor, plan_handle: int
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class ModelOptBlockQuantMoEMethod(FusedMoEMethodBase):
|
||||
"""Load native blocks and retain compact b12x prepared expert storage."""
|
||||
|
||||
def __init__(self, moe_config: FusedMoEConfig, codec: str = "iq2_xs"):
|
||||
super().__init__(moe_config)
|
||||
self.codec = codec.lower()
|
||||
self.block_size, self.block_bytes = B12X_BLOCK_CODECS[self.codec]
|
||||
if moe_config.moe_backend not in ("auto", "b12x"):
|
||||
raise ValueError(
|
||||
"Block-quantized routed experts require the b12x MoE backend"
|
||||
)
|
||||
if not B12xExperts._supports_current_device():
|
||||
raise ValueError("Block-quantized routed experts require b12x on SM12x")
|
||||
if not B12xExperts._supports_parallel_config(moe_config.moe_parallel_config):
|
||||
raise ValueError(
|
||||
"Block-quantized supports tensor parallelism without EP or EPLB"
|
||||
)
|
||||
if moe_config.in_dtype != torch.bfloat16:
|
||||
raise ValueError("Block-quantized routed experts require BF16 activations")
|
||||
if (
|
||||
moe_config.activation
|
||||
not in (MoEActivation.SILU, MoEActivation.RELU2_NO_MUL)
|
||||
or moe_config.has_bias
|
||||
):
|
||||
raise ValueError(
|
||||
"Block-quantized routed experts require bias-free SiLU or ReLU2"
|
||||
)
|
||||
self.gated = moe_config.activation == MoEActivation.SILU
|
||||
if any(
|
||||
value is not None
|
||||
for value in (
|
||||
moe_config.swiglu_limit,
|
||||
moe_config.swiglu_alpha,
|
||||
moe_config.swiglu_beta,
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Block-quantized routed experts require standard SiLU parameters"
|
||||
)
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
act_dtype,
|
||||
moe_parallel_config,
|
||||
):
|
||||
alignment = max(128, self.block_size)
|
||||
if hidden_size % alignment or intermediate_size_per_partition % alignment:
|
||||
raise ValueError(
|
||||
f"{self.codec.upper()} hidden and per-rank intermediate sizes "
|
||||
f"must align to {alignment}"
|
||||
)
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer,
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
self.maybe_roundup_sizes(
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
params_dtype,
|
||||
self.moe.moe_parallel_config,
|
||||
)
|
||||
for name, shape in (
|
||||
(
|
||||
"w13_weight",
|
||||
(
|
||||
num_experts,
|
||||
(2 if self.gated else 1) * intermediate_size_per_partition,
|
||||
hidden_size // self.block_size,
|
||||
self.block_bytes,
|
||||
),
|
||||
),
|
||||
(
|
||||
"w2_weight",
|
||||
(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.block_size,
|
||||
self.block_bytes,
|
||||
),
|
||||
),
|
||||
):
|
||||
layer.register_parameter(
|
||||
name,
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(shape, dtype=torch.uint8),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=self.weight_loader,
|
||||
),
|
||||
)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success=False,
|
||||
):
|
||||
if shard_id not in ("w1", "w2", "w3"):
|
||||
raise ValueError(f"invalid Block-quantized expert projection: {shard_id}")
|
||||
if not self.gated and shard_id == "w3":
|
||||
raise ValueError("non-gated Block-quantized experts have no w3 projection")
|
||||
local_i = self.moe.intermediate_size_per_partition
|
||||
global_i = local_i * self.moe.moe_parallel_config.tp_size
|
||||
hidden = self.moe.hidden_dim
|
||||
expected = (
|
||||
(hidden, global_i // self.block_size, self.block_bytes)
|
||||
if shard_id == "w2"
|
||||
else (global_i, hidden // self.block_size, self.block_bytes)
|
||||
)
|
||||
if loaded_weight.dtype != torch.uint8 or tuple(loaded_weight.shape) != expected:
|
||||
raise ValueError(
|
||||
f"invalid packed tensor {weight_name}: expected uint8{expected}, "
|
||||
f"got {loaded_weight.dtype}{tuple(loaded_weight.shape)}"
|
||||
)
|
||||
if not 0 <= expert_id < param.shape[0]:
|
||||
raise ValueError(f"invalid Block-quantized expert index: {expert_id}")
|
||||
destination = param.data[expert_id]
|
||||
if shard_id == "w2":
|
||||
source = loaded_weight.narrow(
|
||||
1,
|
||||
self.moe.tp_rank * (local_i // self.block_size),
|
||||
local_i // self.block_size,
|
||||
)
|
||||
else:
|
||||
source = loaded_weight.narrow(0, self.moe.tp_rank * local_i, local_i)
|
||||
destination = destination.narrow(
|
||||
0, 0 if shard_id == "w1" else local_i, local_i
|
||||
)
|
||||
destination.copy_(source)
|
||||
return True if return_success else None
|
||||
|
||||
def get_fused_moe_quant_config(self, layer):
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(),
|
||||
_a2=FusedMoEQuantDesc(),
|
||||
_w1=FusedMoEQuantDesc(dtype=self.codec),
|
||||
_w2=FusedMoEQuantDesc(dtype=self.codec),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
moe=self.moe,
|
||||
quant_config=self.moe_quant_config,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
allow_new_interface=True,
|
||||
)
|
||||
assert prepare_finalize is not None
|
||||
experts = B12xExperts(self.moe, self.moe_quant_config)
|
||||
self.moe_kernel = mk.FusedMoEKernel(prepare_finalize, experts)
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer,
|
||||
x,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
shared_experts,
|
||||
shared_experts_input,
|
||||
):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
|
||||
ModelOptBlockQuantMoEMethod.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@register_weight_loader_v2_supported_method
|
||||
class ModelOptBlockQuantLinearMethod(LinearMethodBase):
|
||||
"""Load native block payloads and execute the prepared b12x dense API."""
|
||||
|
||||
def __init__(self, codec: str = "iq2_xs"):
|
||||
self.codec = codec.lower()
|
||||
self.block_size, self.block_bytes = B12X_BLOCK_CODECS[self.codec]
|
||||
self.is_embedding = False
|
||||
from vllm.model_executor.kernels.linear import _get_linear_backend
|
||||
from vllm.utils.b12x import get_b12x_blockscaled
|
||||
|
||||
if _get_linear_backend(quantization=self.codec) not in ("auto", "b12x"):
|
||||
raise ValueError(
|
||||
"Block-quantized dense weights require the b12x linear backend"
|
||||
)
|
||||
api = get_b12x_blockscaled()
|
||||
if (
|
||||
api is None
|
||||
or not hasattr(api, "BlockQuantLinearWeight")
|
||||
or not api.is_supported()
|
||||
):
|
||||
raise ValueError("Block-quantized dense weights require b12x on SM12x")
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
n = sum(output_partition_sizes)
|
||||
self.is_embedding = isinstance(
|
||||
layer, VocabParallelEmbedding
|
||||
) and not isinstance(layer, ParallelLMHead)
|
||||
if self.is_embedding and self.codec != "q8_0":
|
||||
raise ValueError("packed embedding currently requires Q8_0")
|
||||
if (
|
||||
params_dtype != torch.bfloat16
|
||||
or input_size_per_partition % self.block_size
|
||||
or n % 8
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.codec.upper()} dense weights require BF16, "
|
||||
f"K{self.block_size} and N8"
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
(n, input_size_per_partition // self.block_size, self.block_bytes),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=extra_weight_attrs["weight_loader"],
|
||||
),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
from b12x.sequence import embedding
|
||||
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
api = get_b12x_blockscaled()
|
||||
assert api is not None
|
||||
config = get_current_vllm_config()
|
||||
scheduler = config.scheduler_config
|
||||
capture_sizes = config.compilation_config.cudagraph_capture_sizes or []
|
||||
capacity = max(
|
||||
scheduler.max_num_batched_tokens,
|
||||
scheduler.max_num_scheduled_tokens or 0,
|
||||
*capture_sizes,
|
||||
)
|
||||
weight = layer.weight.data
|
||||
if self.is_embedding:
|
||||
bases = weight[..., :2].contiguous().view(torch.float16)
|
||||
if not torch.isfinite(bases).all().item():
|
||||
raise ValueError("Q8_0 embedding scales must be finite")
|
||||
layer.b12x_embedding_plans = {}
|
||||
width = weight.shape[1] * self.block_size
|
||||
if config.model_config is not None:
|
||||
capacity = max(capacity, config.model_config.max_model_len)
|
||||
|
||||
for dtype in (torch.int32, torch.int64):
|
||||
plan = embedding.plan(
|
||||
embedding.EmbeddingQuery(
|
||||
max_rows=capacity,
|
||||
table_rows=weight.shape[0],
|
||||
width=width,
|
||||
row_stride=weight.stride(0),
|
||||
weight_dtype="uint8",
|
||||
id_dtype=str(dtype).removeprefix("torch."),
|
||||
),
|
||||
device=weight.device,
|
||||
)
|
||||
layer.b12x_embedding_plans[dtype] = plan
|
||||
else:
|
||||
packed = api.pack_weight(weight, recipe=self.codec)
|
||||
query = api.BlockscaledQuery(
|
||||
recipe=self.codec,
|
||||
num_tokens=capacity,
|
||||
in_features=packed.in_features,
|
||||
padded_in_features=packed.in_features,
|
||||
out_features=packed.out_features,
|
||||
activation_mode="a16",
|
||||
)
|
||||
plan = api.plan_regimes(
|
||||
query,
|
||||
exact_m=tuple(sorted({m for m in capture_sizes if 0 < m < capacity})),
|
||||
)
|
||||
layer.b12x_block_weight = packed
|
||||
layer.b12x_block_plan = plan
|
||||
if not self.is_embedding:
|
||||
replace_parameter(
|
||||
layer, "weight", torch.empty(0, dtype=torch.uint8, device=weight.device)
|
||||
)
|
||||
|
||||
layer.b12x_warmup_provider = self
|
||||
|
||||
def get_b12x_warmup_unit(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
token_counts: tuple[int, ...],
|
||||
output_dtype: torch.dtype,
|
||||
) -> B12xWarmupUnit:
|
||||
if self.is_embedding:
|
||||
plans = tuple(layer.b12x_embedding_plans.values())
|
||||
else:
|
||||
plans = (layer.b12x_block_plan,)
|
||||
|
||||
def compile() -> None:
|
||||
if self.is_embedding:
|
||||
for dtype, plan in layer.b12x_embedding_plans.items():
|
||||
ids = torch.zeros(
|
||||
plan.query.max_rows, dtype=dtype, device=layer.weight.device
|
||||
)
|
||||
self.embedding(layer, ids)
|
||||
else:
|
||||
packed = layer.b12x_block_weight
|
||||
for tokens in sorted(set(token_counts) | set(plans[0].token_counts)):
|
||||
source = torch.zeros(
|
||||
(tokens, packed.in_features),
|
||||
dtype=output_dtype,
|
||||
device=packed.values.device,
|
||||
)
|
||||
self.apply(layer, source)
|
||||
|
||||
return B12xWarmupUnit(
|
||||
name=(
|
||||
self.codec.upper() + (" embedding" if self.is_embedding else " linear")
|
||||
),
|
||||
# Every layer owns a plan that must be prepared before capture.
|
||||
key=(type(self), *(plan.handle for plan in plans)),
|
||||
compile=compile,
|
||||
)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
if x.dtype != torch.bfloat16:
|
||||
raise ValueError(
|
||||
"Block-quantized dense execution requires BF16 activations"
|
||||
)
|
||||
api = get_b12x_blockscaled()
|
||||
assert api is not None
|
||||
output = api.mm(
|
||||
x.reshape(-1, x.shape[-1]).contiguous(),
|
||||
layer.b12x_block_weight,
|
||||
plan=layer.b12x_block_plan,
|
||||
bias=bias,
|
||||
)
|
||||
return output.view(*x.shape[:-1], layer.b12x_block_weight.out_features)
|
||||
|
||||
def embedding(self, layer, input_):
|
||||
plan = layer.b12x_embedding_plans[input_.dtype]
|
||||
width = layer.weight.shape[1] * self.block_size
|
||||
out = torch.empty(
|
||||
(*input_.shape, width), dtype=torch.bfloat16, device=layer.weight.device
|
||||
)
|
||||
ids, rows = input_.reshape(-1), out.view(-1, width)
|
||||
_modelopt_block_embedding_chunked(layer.weight, ids, rows, plan.handle)
|
||||
return out
|
||||
@@ -38,7 +38,10 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
@@ -541,9 +544,12 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
def __init__(self, moe: FusedMoEConfig, activation_key: QuantKey | None = None):
|
||||
super().__init__(moe)
|
||||
self.weight_dtype = "mxfp4"
|
||||
# Checkpoint-declared activation quantization. Only backends that pick
|
||||
# their activation format from the layer (Humming) consume it.
|
||||
self.activation_key = activation_key
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
@@ -822,6 +828,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
activation=self.moe.activation,
|
||||
use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4,
|
||||
activation_key=self.activation_key,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -15,9 +15,11 @@ if TYPE_CHECKING:
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
fp8_scaled_mm_is_batch_invariant,
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -32,10 +34,12 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.requantization import (
|
||||
OnlineRequantizationMixin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
@@ -116,7 +120,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
return is_row_parallel or (reduces_output_dim and is_column_parallel)
|
||||
|
||||
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
class OnlineLinearBase(OnlineRequantizationMixin, LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
@@ -124,15 +128,17 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
activation_quant_key: QuantKey | None
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
self.requantization_source: QuantizeMethodBase | None = None
|
||||
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Configure serialized-weight conversion before online quantization."""
|
||||
self.requantization_source = source_method
|
||||
self.uses_meta_device = False
|
||||
def get_weight_for_quantization(self, layer: torch.nn.Module) -> torch.Tensor:
|
||||
if self.requantization_source is None:
|
||||
return layer.weight
|
||||
weight = self.requantization_source.dequantize_weight(layer)
|
||||
if not isinstance(weight, torch.Tensor):
|
||||
raise TypeError("Linear requantization requires a single weight tensor.")
|
||||
return weight
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -144,22 +150,15 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
if self.requantization_source is not None:
|
||||
existing_parameter_names = set(layer._parameters)
|
||||
self.requantization_source.create_weights(
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
self.requantization_source_parameters = {
|
||||
name: parameter
|
||||
for name, parameter in layer._parameters.items()
|
||||
if name not in existing_parameter_names and parameter is not None
|
||||
}
|
||||
if self.create_requantization_source_weights(
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
return
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
@@ -185,19 +184,6 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Release checkpoint parameters after successful requantization."""
|
||||
for name, source_parameter in self.requantization_source_parameters.items():
|
||||
if layer._parameters.get(name) is source_parameter:
|
||||
delattr(layer, name)
|
||||
self.requantization_source_parameters.clear()
|
||||
|
||||
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
|
||||
"""Return checkpoint weights materialized for online quantization."""
|
||||
if self.requantization_source is None:
|
||||
return layer.weight
|
||||
return self.requantization_source.dequantize_weight(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
@@ -266,6 +252,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
|
||||
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
@@ -278,7 +265,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
) -> torch.Tensor:
|
||||
# if batch invariant mode is enabled, use BF16 dequant
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
|
||||
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
weight_fp8 = layer.weight.to(torch.bfloat16)
|
||||
@@ -366,6 +353,7 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
del layer.weight_scale
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
@@ -466,7 +454,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
# if batch invariant mode is enabled dequant
|
||||
if (
|
||||
envs.VLLM_BATCH_INVARIANT
|
||||
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
|
||||
and not fp8_scaled_mm_is_batch_invariant(self.fp8_linear)
|
||||
and not isinstance(x, QuantizedActivation)
|
||||
):
|
||||
weight_dequant = (
|
||||
@@ -622,27 +610,29 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
|
||||
# If checkpoint is fp16, quantize in place.
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
|
||||
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
moe_tp_size = self.moe.tp_size
|
||||
w13_amax = weight_amax(layer.w13_weight.flatten(1), dim=-1)
|
||||
w13_amax = weight_amax(w13_weight.flatten(1), dim=-1)
|
||||
w13_amax = amax_for_moe_weight_quant(w13_amax, moe_tp_size)
|
||||
w13_scale = _fp8_scale(w13_amax)
|
||||
w2_amax = weight_amax(layer.w2_weight.flatten(1), dim=-1)
|
||||
w2_amax = weight_amax(w2_weight.flatten(1), dim=-1)
|
||||
w2_amax = amax_for_moe_weight_quant(w2_amax, moe_tp_size)
|
||||
w2_scale = _fp8_scale(w2_amax)
|
||||
|
||||
for expert in range(layer.local_num_experts):
|
||||
w13[expert, :, :], _ = ops.scaled_fp8_quant(
|
||||
layer.w13_weight[expert, :, :], scale=w13_scale[expert]
|
||||
w13_weight[expert, :, :], scale=w13_scale[expert]
|
||||
)
|
||||
w2[expert, :, :], _ = ops.scaled_fp8_quant(
|
||||
layer.w2_weight[expert, :, :], scale=w2_scale[expert]
|
||||
w2_weight[expert, :, :], scale=w2_scale[expert]
|
||||
)
|
||||
|
||||
# Shuffle weights to runtime format and setup kernel.
|
||||
@@ -655,6 +645,7 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
w13_input_scale=layer.w13_input_scale,
|
||||
w2_input_scale=layer.w2_input_scale,
|
||||
)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
@@ -698,11 +689,12 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
self._zero_padding(layer)
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
self._zero_padding(layer, w13_weight, w2_weight)
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
|
||||
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
|
||||
|
||||
block_size = self.weight_block_size
|
||||
assert block_size is not None
|
||||
@@ -711,8 +703,8 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
# Create block-shaped scales (computed here rather than in
|
||||
# create_weights because online quant doesn't need them until now).
|
||||
num_experts = layer.local_num_experts
|
||||
_, w13_out, w13_in = layer.w13_weight.shape
|
||||
_, w2_out, w2_in = layer.w2_weight.shape
|
||||
_, w13_out, w13_in = w13_weight.shape
|
||||
_, w2_out, w2_in = w2_weight.shape
|
||||
|
||||
w13_scale = torch.ones(
|
||||
num_experts,
|
||||
@@ -731,18 +723,22 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
|
||||
for expert in range(num_experts):
|
||||
w13[expert], w13_scale[expert] = per_block_cast_to_fp8(
|
||||
layer.w13_weight[expert],
|
||||
w13_weight[expert],
|
||||
block_size=block_size,
|
||||
use_ue8m0=False,
|
||||
)
|
||||
w2[expert], w2_scale[expert] = per_block_cast_to_fp8(
|
||||
layer.w2_weight[expert],
|
||||
w2_weight[expert],
|
||||
block_size=block_size,
|
||||
use_ue8m0=False,
|
||||
)
|
||||
|
||||
layer.weight_block_size = block_size
|
||||
|
||||
# Activation quantization is dynamic (including in re-quantization)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
# Shuffle weights to runtime format and setup kernel.
|
||||
self._setup_kernel(
|
||||
layer,
|
||||
@@ -750,9 +746,10 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
layer.w13_input_scale,
|
||||
layer.w2_input_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
@@ -804,32 +801,32 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
|
||||
w13 = torch.empty_like(w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(w2_weight, dtype=fp8_dtype)
|
||||
# Scale's leading dim is taken from the fp8 weight tensor by
|
||||
# construction, so it cannot drift from the weight's expert count
|
||||
# under EP / padded MoE.
|
||||
n_w13 = layer.w13_weight.shape[1]
|
||||
n_w13 = w13_weight.shape[1]
|
||||
w13_scale = torch.ones(
|
||||
w13.shape[0], n_w13, 1, device=w13.device, dtype=torch.float32
|
||||
)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
w2_amax = weight_amax(layer.w2_weight, dim=-1, keepdim=True)
|
||||
w2_amax = weight_amax(w2_weight, dim=-1, keepdim=True)
|
||||
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
|
||||
w2_scale = _fp8_channel_scale(w2_amax)
|
||||
|
||||
for expert in range(layer.local_num_experts):
|
||||
w13[expert], w13_scale[expert] = ops.scaled_fp8_quant(
|
||||
layer.w13_weight[expert],
|
||||
w13_weight[expert],
|
||||
scale=None,
|
||||
use_per_token_if_dynamic=True,
|
||||
)
|
||||
w2[expert] = _fp8_quant_per_channel(
|
||||
layer.w2_weight[expert], w2_scale[expert]
|
||||
)
|
||||
w2[expert] = _fp8_quant_per_channel(w2_weight[expert], w2_scale[expert])
|
||||
|
||||
self._setup_kernel(
|
||||
layer,
|
||||
@@ -840,5 +837,6 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -52,42 +52,51 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
self._quantize_weights(layer)
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
self._quantize_weights(layer, w13_weight, w2_weight)
|
||||
self._setup_kernel(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
def _quantize_weights(self, layer: Module) -> None:
|
||||
def _quantize_weights(
|
||||
self,
|
||||
layer: Module,
|
||||
w13_weight: torch.Tensor | None = None,
|
||||
w2_weight: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
w13_weight = layer.w13_weight if w13_weight is None else w13_weight
|
||||
w2_weight = layer.w2_weight if w2_weight is None else w2_weight
|
||||
vmax = torch.iinfo(torch.int8).max
|
||||
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=torch.int8)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=torch.int8)
|
||||
w13 = torch.empty_like(w13_weight, dtype=torch.int8)
|
||||
w2 = torch.empty_like(w2_weight, dtype=torch.int8)
|
||||
w13_scale = torch.zeros(
|
||||
layer.num_experts,
|
||||
layer.w13_weight.shape[1],
|
||||
w13_weight.shape[1],
|
||||
device=w13.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
w2_scale = torch.zeros(
|
||||
layer.num_experts,
|
||||
layer.w2_weight.shape[1],
|
||||
w2_weight.shape[1],
|
||||
device=w2.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
w2_amax = weight_amax(layer.w2_weight, dim=-1)
|
||||
w2_amax = weight_amax(w2_weight, dim=-1)
|
||||
w2_amax = amax_for_moe_weight_quant(w2_amax, self.moe.tp_size)
|
||||
|
||||
for expert in range(layer.local_num_experts):
|
||||
# w13: per-row quantization over hidden_size dim
|
||||
w = layer.w13_weight[expert, :, :]
|
||||
w = w13_weight[expert, :, :]
|
||||
scales = w.abs().amax(dim=1) / vmax
|
||||
q = w.div(scales.unsqueeze(1)).round().clamp(-vmax, vmax)
|
||||
w13[expert, :, :] = q.to(torch.int8)
|
||||
w13_scale[expert, :] = scales
|
||||
|
||||
# w2: per-row quantization over intermediate_size dim
|
||||
w = layer.w2_weight[expert, :, :]
|
||||
w = w2_weight[expert, :, :]
|
||||
scales = w2_amax[expert] / vmax
|
||||
q = w.div(scales.unsqueeze(1)).round().clamp(-vmax, vmax)
|
||||
w2[expert, :, :] = q.to(torch.int8)
|
||||
|
||||
@@ -10,26 +10,26 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.layers.quantization.online.requantization import (
|
||||
OnlineRequantizationMixin,
|
||||
)
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
initialize_online_processing,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
|
||||
class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
class OnlineMoEMethodBase(OnlineRequantizationMixin, FusedMoEMethodBase):
|
||||
"""Base for MoE methods that load full-precision weights on meta device
|
||||
and quantize them after loading via the QeRL layerwise processing system.
|
||||
"""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Reject requantization from a checkpoint-quantized MoE method."""
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -40,10 +40,22 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
# Target conversion needs this metadata even when the source creates weights.
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
if self.requantization_source is not None:
|
||||
self.create_requantization_source_weights(
|
||||
layer,
|
||||
num_experts,
|
||||
layer.moe_config.hidden_dim_unpadded,
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
return
|
||||
|
||||
# Fused gate_up_proj (column parallel) — full precision on meta device
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
@@ -103,22 +115,95 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def _zero_padding(self, layer: torch.nn.Module) -> None:
|
||||
def get_weights_for_quantization(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.requantization_source is None:
|
||||
return layer.w13_weight, layer.w2_weight
|
||||
|
||||
weights = self.requantization_source.dequantize_weight(layer)
|
||||
if not isinstance(weights, tuple):
|
||||
raise TypeError("MoE requantization requires w13 and w2 weight tensors.")
|
||||
|
||||
w13, w2 = weights
|
||||
|
||||
config = layer.moe_config
|
||||
hidden_size = config.hidden_dim_unpadded
|
||||
intermediate_size = config.intermediate_size_per_partition_unpadded
|
||||
target_hidden_size = config.hidden_dim
|
||||
target_intermediate_size = config.intermediate_size_per_partition
|
||||
|
||||
num_experts = layer.local_num_experts
|
||||
num_shards = self.moe.w13_num_shards
|
||||
|
||||
if w13.shape != (num_experts, num_shards * intermediate_size, hidden_size):
|
||||
raise ValueError("Requantization requires unpadded source w13 weights.")
|
||||
if w2.shape != (num_experts, hidden_size, intermediate_size):
|
||||
raise ValueError("Requantization requires unpadded source w2 weights.")
|
||||
|
||||
if (
|
||||
target_hidden_size < hidden_size
|
||||
or target_intermediate_size < intermediate_size
|
||||
):
|
||||
raise ValueError("Requantization target dimensions cannot shrink weights.")
|
||||
|
||||
# No padding required.
|
||||
if (hidden_size, intermediate_size) == (
|
||||
target_hidden_size,
|
||||
target_intermediate_size,
|
||||
):
|
||||
return w13, w2
|
||||
|
||||
if any(
|
||||
getattr(layer, name, None) is not None for name in ("w13_bias", "w2_bias")
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Padded requantization with biases is unsupported."
|
||||
)
|
||||
|
||||
def pad_moe_weights(
|
||||
w13: torch.Tensor, w2: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
padded_w13 = w13.new_zeros(
|
||||
(num_experts, num_shards * target_intermediate_size, target_hidden_size)
|
||||
)
|
||||
padded_w2 = w2.new_zeros(
|
||||
(num_experts, target_hidden_size, target_intermediate_size)
|
||||
)
|
||||
for shard in range(num_shards):
|
||||
source_start = shard * intermediate_size
|
||||
target_start = shard * target_intermediate_size
|
||||
padded_w13[
|
||||
:, target_start : target_start + intermediate_size, :hidden_size
|
||||
] = w13[:, source_start : source_start + intermediate_size, :]
|
||||
padded_w2[:, :hidden_size, :intermediate_size] = w2
|
||||
return padded_w13, padded_w2
|
||||
|
||||
return pad_moe_weights(w13, w2)
|
||||
|
||||
def _zero_padding(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
w13_weight: torch.Tensor | None = None,
|
||||
w2_weight: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
w13_weight = layer.w13_weight if w13_weight is None else w13_weight
|
||||
w2_weight = layer.w2_weight if w2_weight is None else w2_weight
|
||||
hidden_size = layer.moe_config.hidden_dim_unpadded
|
||||
intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
|
||||
w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards
|
||||
w13_shard = w13_weight.shape[1] // self.moe.w13_num_shards
|
||||
if w13_shard > intermediate_size:
|
||||
for shard in range(self.moe.w13_num_shards):
|
||||
start = shard * w13_shard + intermediate_size
|
||||
layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
|
||||
if layer.w13_weight.shape[2] > hidden_size:
|
||||
layer.w13_weight[:, :, hidden_size:] = 0
|
||||
w13_weight[:, start : (shard + 1) * w13_shard, :] = 0
|
||||
if w13_weight.shape[2] > hidden_size:
|
||||
w13_weight[:, :, hidden_size:] = 0
|
||||
|
||||
if layer.w2_weight.shape[1] > hidden_size:
|
||||
layer.w2_weight[:, hidden_size:, :] = 0
|
||||
if layer.w2_weight.shape[2] > intermediate_size:
|
||||
layer.w2_weight[:, :, intermediate_size:] = 0
|
||||
if w2_weight.shape[1] > hidden_size:
|
||||
w2_weight[:, hidden_size:, :] = 0
|
||||
if w2_weight.shape[2] > intermediate_size:
|
||||
w2_weight[:, :, intermediate_size:] = 0
|
||||
|
||||
if getattr(layer, "w13_bias", None) is not None:
|
||||
w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards
|
||||
|
||||
@@ -127,6 +127,7 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -295,7 +296,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
self._zero_padding(layer)
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
self._zero_padding(layer, w13_weight, w2_weight)
|
||||
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
@@ -304,8 +306,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
w13, w13_scale = _quantize_mxfp4_moe_weight(layer.w13_weight)
|
||||
w2, w2_scale = _quantize_mxfp4_moe_weight(layer.w2_weight)
|
||||
w13, w13_scale = _quantize_mxfp4_moe_weight(w13_weight)
|
||||
w2, w2_scale = _quantize_mxfp4_moe_weight(w2_weight)
|
||||
|
||||
self._setup_kernel(
|
||||
layer,
|
||||
@@ -316,5 +318,6 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
getattr(layer, "w13_bias", None),
|
||||
getattr(layer, "w2_bias", None),
|
||||
)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -34,7 +34,6 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
@@ -86,6 +85,7 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -237,14 +237,12 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
w13, w13_scale = self._quantize_mxfp8_moe_weight(layer.w13_weight)
|
||||
w2, w2_scale = self._quantize_mxfp8_moe_weight(layer.w2_weight)
|
||||
w13, w13_scale = self._quantize_mxfp8_moe_weight(w13_weight)
|
||||
w2, w2_scale = self._quantize_mxfp8_moe_weight(w2_weight)
|
||||
|
||||
self._setup_kernel(
|
||||
layer,
|
||||
@@ -255,5 +253,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
layer.w13_input_scale,
|
||||
layer.w2_input_scale,
|
||||
)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -116,19 +116,26 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
self._quantize_weights(layer)
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
self._quantize_weights(layer, w13_weight, w2_weight)
|
||||
self._setup_kernel(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
def _quantize_weights(self, layer: Module) -> None:
|
||||
def _quantize_weights(
|
||||
self,
|
||||
layer: Module,
|
||||
w13_weight: torch.Tensor | None = None,
|
||||
w2_weight: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
w13_weight = layer.w13_weight if w13_weight is None else w13_weight
|
||||
w2_weight = layer.w2_weight if w2_weight is None else w2_weight
|
||||
moe_tp_size = self.moe.tp_size
|
||||
w13, w13_scale, w13_scale_2 = _quantize_moe_weight_to_nvfp4(
|
||||
layer.w13_weight, moe_tp_size
|
||||
)
|
||||
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(
|
||||
layer.w2_weight, moe_tp_size
|
||||
w13_weight, moe_tp_size
|
||||
)
|
||||
w2, w2_scale, w2_scale_2 = _quantize_moe_weight_to_nvfp4(w2_weight, moe_tp_size)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
|
||||
|
||||
class OnlineRequantizationMixin:
|
||||
"""Shared lifecycle for converting serialized weights before quantization."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.requantization_source: QuantizeMethodBase | None = None
|
||||
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Configure serialized-weight conversion before online quantization."""
|
||||
self.requantization_source = source_method
|
||||
self.uses_meta_device = False
|
||||
|
||||
def create_requantization_source_weights(
|
||||
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
|
||||
) -> bool:
|
||||
"""Delegate weight creation to the source and track its parameters."""
|
||||
if self.requantization_source is None:
|
||||
return False
|
||||
|
||||
existing_parameter_names = set(layer._parameters)
|
||||
self.requantization_source.create_weights(
|
||||
layer, *weight_args, **extra_weight_attrs
|
||||
)
|
||||
self.requantization_source_parameters = {
|
||||
name: parameter
|
||||
for name, parameter in layer._parameters.items()
|
||||
if name not in existing_parameter_names and parameter is not None
|
||||
}
|
||||
return True
|
||||
|
||||
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Release serialized parameters after successful requantization."""
|
||||
for name, source_parameter in self.requantization_source_parameters.items():
|
||||
if layer._parameters.get(name) is source_parameter:
|
||||
delattr(layer, name)
|
||||
self.requantization_source_parameters.clear()
|
||||
@@ -69,6 +69,9 @@ from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_nvfp4_moe_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
@@ -2110,6 +2113,22 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
layer.register_parameter("w2_input_scale_2", w2_input_scale_2)
|
||||
set_weight_attrs(w2_input_scale_2, extra_weight_attrs)
|
||||
|
||||
def dequantize_weight(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Dequantize Quark NVFP4 expert weights for requantization."""
|
||||
return dequantize_nvfp4_moe_weights(
|
||||
layer.w13_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w13_weight_scale_2,
|
||||
layer.w2_weight,
|
||||
layer.w2_weight_scale,
|
||||
layer.w2_weight_scale_2,
|
||||
layer.params_dtype,
|
||||
group_size=self.group_size,
|
||||
invert_global_scales=False,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
|
||||
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
|
||||
|
||||
@@ -1041,7 +1041,10 @@ def w8a8_triton_block_scaled_mm(
|
||||
C_shape = A.shape[:-1] + (N,)
|
||||
C = A.new_empty(C_shape, dtype=output_dtype)
|
||||
|
||||
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
configs = None # use the default config
|
||||
else:
|
||||
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
|
||||
if configs:
|
||||
# Get the optimal config if there is one
|
||||
config = configs[min(configs.keys(), key=lambda x: abs(x - M))]
|
||||
|
||||
@@ -59,13 +59,13 @@ def prepare_humming_linear_layer_config(
|
||||
input_quant_config: dict | None = None,
|
||||
*,
|
||||
input_schema: "BaseInputSchema | None" = None,
|
||||
allow_input_schema_fallback: bool = True,
|
||||
) -> "LayerConfig":
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
prepare_and_transform_humming_tensors,
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
@@ -120,6 +120,7 @@ def prepare_humming_linear_layer_config(
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
param_dtype=layer.params_dtype,
|
||||
allow_fallback=allow_input_schema_fallback,
|
||||
)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
@@ -127,7 +128,8 @@ def prepare_humming_linear_layer_config(
|
||||
input_tensors.pop(name, None)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding.
|
||||
config = prepare_layer_config(
|
||||
config, tensors = prepare_and_transform_humming_tensors(
|
||||
tensors=tensors,
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
@@ -138,7 +140,6 @@ def prepare_humming_linear_layer_config(
|
||||
torch_dtype=layer.params_dtype,
|
||||
device=tensors["weight"].device,
|
||||
)
|
||||
tensors = transform_humming_tensors(config, tensors)
|
||||
tensors.update(input_tensors)
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
|
||||
@@ -397,11 +397,15 @@ def _prepare_and_transform_sublayer(
|
||||
) -> "LayerConfig":
|
||||
"""Prepare Humming configuration and transform one sublayer's tensors."""
|
||||
from vllm.utils.humming import (
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
prepare_and_transform_humming_tensors,
|
||||
)
|
||||
|
||||
config = prepare_layer_config(
|
||||
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
source_tensors.pop(name, None)
|
||||
config, tensors = prepare_and_transform_humming_tensors(
|
||||
tensors=source_tensors,
|
||||
shape_n=shape_n,
|
||||
shape_k=shape_k,
|
||||
pad_n_to_multiple=256,
|
||||
@@ -413,11 +417,6 @@ def _prepare_and_transform_sublayer(
|
||||
torch_dtype=param_dtype,
|
||||
device=getattr(layer, f"{sublayer_name}_weight").device,
|
||||
)
|
||||
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
source_tensors.pop(name, None)
|
||||
tensors = transform_humming_tensors(config, source_tensors)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name in source_tensors:
|
||||
tensors[name] = source_tensors[name]
|
||||
|
||||
@@ -322,22 +322,6 @@ def check_and_fallback_input_schema(
|
||||
input_group_size = input_schema.input_scale_group_size
|
||||
input_scale_dtype = input_schema.input_scale_dtype
|
||||
|
||||
def is_compatible(candidate: "HummingInputSchema") -> bool:
|
||||
check_weight_schema = weight_schema
|
||||
if (
|
||||
candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
|
||||
and weight_schema.b_dtype == humming_dtypes.float4e2m1
|
||||
and weight_group_size == 16
|
||||
and weight_schema.bs_dtype
|
||||
in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
|
||||
):
|
||||
# Humming rejects folded NVFP4 scales despite A16 software dequant
|
||||
# supporting them. Relax only this check, preserving kernel metadata.
|
||||
check_weight_schema = dataclasses.replace(
|
||||
weight_schema, bs_dtype=humming_dtypes.float8e4m3
|
||||
)
|
||||
return candidate.is_compatible_with(check_weight_schema, param_dtype)
|
||||
|
||||
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
|
||||
if dtype is None or dtype.num_bits == 16:
|
||||
return False
|
||||
@@ -349,7 +333,7 @@ def check_and_fallback_input_schema(
|
||||
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
|
||||
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
|
||||
|
||||
if is_compatible(input_schema):
|
||||
if input_schema.is_compatible_with(weight_schema, param_dtype):
|
||||
if not allow_fallback:
|
||||
if is_deprecated(a_dtype):
|
||||
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
|
||||
@@ -406,7 +390,7 @@ def check_and_fallback_input_schema(
|
||||
input_scale_dtype=scale_dtype,
|
||||
input_quant_mode=quant_mode,
|
||||
)
|
||||
if is_compatible(candidate):
|
||||
if candidate.is_compatible_with(weight_schema, param_dtype):
|
||||
return candidate
|
||||
|
||||
raise ValueError(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user