[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests

- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm
  target path; drop the rocm requirement files
- Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules
  and retarget the linear kernel registries away from them
- Reduce platforms/rocm.py to the GCN architecture predicates that live
  code still imports, and delete the RocmPlatform class and plugin
- Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc,
  server_info and sleep routes plus the log_error_stack flag
- Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins;
  the live server process maps none of them
- Delete tests that only build architectures this fork no longer
  registers, and test directories left without any test file
- Restore the tests/v1/kv_connector mock helpers that scheduler tests
  import at module level

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 19:31:07 +07:00
co-authored by Hinata
parent 2066d3143f
commit 5f3c7bd8ab
162 changed files with 599 additions and 27490 deletions
@@ -50,12 +50,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.machete import (
from vllm.model_executor.kernels.linear.mixed_precision.marlin import (
MarlinLinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import (
RDNA3W4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import (
RDNAHybridW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
@@ -122,9 +116,6 @@ from vllm.model_executor.kernels.linear.mxfp8.marlin import (
from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
TorchMxFp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4 import (
NvFp4LinearKernel,
NvFp4LinearLayerConfig,
@@ -210,9 +201,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
PerTensorTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
ROCmFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
TritonFp8BlockScaledMMKernel,
TritonFP8ScaledMMLinearKernel,
@@ -439,7 +427,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
AiterPerTokenFp8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
@@ -502,8 +489,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
HummingLinearKernel,
],
PlatformEnum.ROCM: [
RDNA3W4A16LinearKernel,
RDNAHybridW4A16LinearKernel,
TritonW4A16LinearKernel,
ConchLinearKernel,
ExllamaLinearKernel,
@@ -528,9 +513,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
TorchMxFp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
# falls through to BF16 emulation (hipBLASLt) elsewhere / on regression.
RocmDotScaledMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
],
}
@@ -1283,7 +1265,6 @@ __all__ = [
"ChannelWiseTorchFP8ScaledMMLinearKernel",
"PerTensorTorchFP8ScaledMMLinearKernel",
"RowWiseTorchFP8ScaledMMLinearKernel",
"ROCmFP8ScaledMMLinearKernel",
"TritonInt8ScaledMMLinearKernel",
"ZentorchInt8ScaledMMLinearKernel",
"ZentorchWNA16LinearKernel",
@@ -1294,7 +1275,6 @@ __all__ = [
"CutlassW4A8LinearKernel",
"Dynamic4bitLinearKernel",
"ExllamaLinearKernel",
"RDNAHybridW4A16LinearKernel",
"MacheteLinearKernel",
"MarlinLinearKernel",
"TritonW4A16LinearKernel",
@@ -29,12 +29,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearKernel,
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import (
RDNA3W4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import (
RDNAHybridW4A16LinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
TritonW4A16LinearKernel,
)
@@ -53,8 +47,6 @@ __all__ = [
"HummingLinearKernel",
"MacheteLinearKernel",
"MarlinLinearKernel",
"RDNA3W4A16LinearKernel",
"RDNAHybridW4A16LinearKernel",
"TritonW4A16LinearKernel",
"XPUwNa16LinearKernel",
"ZentorchWNA16LinearKernel",
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""W4A16 GPTQ kernel for AMD RDNA3 (gfx1100) — fp16 + bf16.
Drop-in replacement for ExllamaLinearKernel on RDNA3 that adds native bf16
support. The HIP kernel lives in ``csrc/rocm/q_gemm_rdna3.cu``
and is exposed via ``torch.ops._rocm_C.gptq_gemm_rdna3``.
Registered ahead of TritonW4A16LinearKernel for the ROCm-RDNA3 path; falls
through to the Triton kernel on non-RDNA3 ROCm devices (e.g. CDNA/MI300).
"""
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.quant_utils import (
pack_quantized_values_into_int32,
)
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
class RDNA3W4A16LinearKernel(MPLinearKernel):
SUPPORTED_QUANT_TYPES = [scalar_types.uint4b8]
@classmethod
def get_min_capability(cls) -> int:
# ROCm gates via on_gfx1100() in can_implement.
return 60
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "RDNA3 W4A16 kernel is ROCm-only"
from vllm.platforms.rocm import on_gfx1100
if not on_gfx1100():
return False, "RDNA3 W4A16 kernel requires gfx1100"
# The HIP op is registered by the C++ extension; if a user is running
# against a vLLM build that doesn't include it (e.g. partial rebuild),
# fall through gracefully to the next kernel in the registry.
if not (
hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
):
return (
False,
"torch.ops._rocm_C.gptq_gemm_rdna3 missing — rebuild C++ extension",
)
if c.act_type not in (torch.float16, torch.bfloat16):
return False, "RDNA3 W4A16 kernel only supports fp16 and bf16"
if c.weight_type not in cls.SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
f"RDNA3 W4A16 kernel; supported: {cls.SUPPORTED_QUANT_TYPES}",
)
if c.group_size <= 0:
return (
False,
"RDNA3 W4A16 kernel does not support channelwise quantization",
)
if c.full_weight_shape[0] % c.group_size != 0:
return (
False,
f"Group size ({c.group_size}) does not evenly divide K "
f"({c.full_weight_shape[0]})",
)
# Output features must be a multiple of the pack factor (8 nibbles per
# int32) and of 8 so that qzeros (packed 4-bit per col) align cleanly
# against the BLOCK_KN_SIZE*4 = 512 N-stride and per-thread 4 columns.
if c.partition_weight_shape[1] % 8 != 0:
return (
False,
"Output features must be a multiple of 8 for the RDNA3 "
"W4A16 kernel (qzeros packing)",
)
return True, None
# ----- Weight prep (identical layout/shuffle as ExllamaLinearKernel) -----
def process_weights_after_loading(self, layer: torch.nn.Module):
c = self.config
device = getattr(layer, self.w_q_name).device
# Synthesize zero points if the checkpoint doesn't carry them.
if not c.zero_points:
self.w_zp_name = "qzeros"
groups = c.partition_weight_shape[0] // c.group_size
out_features = c.partition_weight_shape[1]
if c.weight_type.has_bias():
# GPTQv1 quirk: the kernel adds 1 to the stored zero, so we
# encode (bias - 1) here. See exllama.py for the link to the
# documentation of this checkpoint-format wart.
zeros = torch.full(
(groups, out_features),
c.weight_type.bias - 1,
dtype=torch.int32,
device=device,
)
else:
raise NotImplementedError(
"RDNA3 W4A16 kernel: zero-bias 4-bit quant requires "
"explicit zero points (GPTQv1 +1 quirk)."
)
zeros = pack_quantized_values_into_int32(zeros, c.weight_type, packed_dim=1)
setattr(
layer, self.w_zp_name, torch.nn.Parameter(zeros, requires_grad=False)
)
def transform_w_q(x):
assert isinstance(x, BasevLLMParameter)
permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0)
x_cont = x.data.contiguous()
# Same 4-bit shuffle as exllama. The RDNA3 kernel reads weights in
# the same shuffled int32 layout and uses the (qa & 0x000F000F)
# bit-trick on top.
ops.gptq_shuffle(x_cont, c.weight_type.size_bits)
return x_cont
def transform_w_s(x):
assert isinstance(x, BasevLLMParameter)
permute_param_layout_(x, input_dim=0, output_dim=1)
x.data = x.data.contiguous()
# Keep scales in the activation dtype (fp16 OR bf16) — the kernel
# branches on dtype internally.
return x.to(dtype=c.act_type)
self._transform_param(layer, self.w_q_name, transform_w_q)
self._transform_param(layer, self.w_s_name, transform_w_s)
# ----- Forward --------------------------------------------------------
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
c = self.config
x_2d = x.reshape(-1, x.shape[-1])
out_shape = x.shape[:-1] + (c.partition_weight_shape[1],)
w_q, w_s, w_zp = self._get_weight_params(layer)
assert w_zp is not None, "Zero points are required by RDNA3 W4A16"
output = ops.gptq_gemm_rdna3(x_2d, w_q, w_zp, w_s, False)
if bias is not None:
output.add_(bias)
return output.reshape(out_shape)
@@ -1,651 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Hybrid W4A16 kernel: Triton for prefill, HIP skinny for decode.
Routes based on the activation shape:
M <= MAX_SKINNY_BATCH_SIZE and K*M <= MEDIUM_SKINNY_LIMIT_ELEMENTS:
HIP skinny GEMM (wvSplitK_int4_g)
otherwise: Triton W4A16 fused dequant GEMM
Stores the weights ONCE as int8 [N, K//2] (ExLlama shuffle packed). Both
paths read this single buffer: the HIP skinny kernel uses it directly, and
the triton kernel reinterprets it as int32 [N, K//8] via a view (and
transposes tiles in-register). No dual weight storage.
"""
from contextlib import nullcontext
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
unpack_quantized_values_into_int32,
)
from vllm.model_executor.parameter import (
permute_param_layout_,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
SUPPORTED_GROUP_SIZES = [32, 64, 128]
def _on_gfx12x() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx12x
return on_gfx12x()
def _on_gfx1x() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx1x
return on_gfx1x()
def _on_gfx1151() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx1151
return on_gfx1151()
def _on_gfx115x() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx115x
return on_gfx115x()
# Maximum activation rows supported by HIP skinny (Python M maps to C++ N_in).
# When K*M exceeds regular LDS capacity, the C++ medium kernel caches the
# prefix in LDS and reads the remaining activation elements from global memory.
# On gfx115x the K*M term is dropped entirely: deep-K shapes stage every
# activation row in LDS one K chunk at a time, and only when N is not a
# multiple of the kernel's row tile does the medium kernel fall back to reading
# the rows that do not fit from global memory.
MAX_SKINNY_BATCH_SIZE = 5
# 64 KiB per-workgroup LDS limit expressed in fp16 elements.
# (AMD RDNA has 128 KiB total LDS per CU, but 64 KiB per workgroup.)
LDS_CAPACITY_ELEMENTS = 64 * 1024 // 2 # 32768 fp16 elements
MEDIUM_SKINNY_LIMIT_ELEMENTS = int(LDS_CAPACITY_ELEMENTS * 1.2)
# ---------------------------------------------------------------------------
# Triton kernel for the prefill path (reads skinny-format weights [N, K//8])
# ---------------------------------------------------------------------------
@triton.jit
def _triton_w4a16_skinny_fmt_kernel(
# Pointers
a_ptr, # [M, K] fp16/bf16 activations
b_ptr, # [N, K//8] int32 packed (ExLlama shuffle, K is packed dim)
scales_ptr, # [N, K//G] fp16/bf16 scales (skinny layout)
zp_ptr, # [N//8, K//G] int32 zero-points (when HAS_ZP=True)
c_ptr, # [M, N] fp16/bf16 output
# Dimensions
M,
N,
K,
K8, # K // 8
num_groups, # K // group_size
stride_sn, # row stride of scales_ptr and zp_ptr, in groups
stride_bn, # b_ptr row stride; >= K8, the rows may be padded
stride_am, # a_ptr row stride in elements; >= K, rows may be padded
# Quantization parameters
group_size,
ZP_BIAS: tl.constexpr,
HAS_ZP: tl.constexpr,
# Block sizes
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
"""Fused W4A16 GEMM reading weights from skinny format [N, K//8].
B is stored as [N, K//8] int32 using ExLlama shuffle packing:
each int32 packs 8 K-values with interleave [0,2,4,6,1,3,5,7]:
packed = val[0] | (val[2]<<4) | (val[4]<<8) | (val[6]<<12)
| (val[1]<<16) | (val[3]<<20) | (val[5]<<24) | (val[7]<<28)
Scales are [N, K//G] (skinny layout, NOT transposed).
When HAS_ZP=True, zp_ptr holds [N//8, K//G] int32 with row n's raw
zero-point at word[n//8] bits 4*(n%8), and dequant is
(nibble - zp_raw) * scale.
When HAS_ZP=False, only the constant ZP_BIAS is subtracted (symmetric).
"""
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
# ExLlama unshuffle shifts: shift[j] = (j//2)*4 + (j%2)*16
# For 8 values: [0, 16, 4, 20, 8, 24, 12, 28]
exllama_shifts_row = (tl.arange(0, 8) // 2) * 4 + (tl.arange(0, 8) % 2) * 16
shifts_1d = tl.reshape(
tl.broadcast_to(exllama_shifts_row[None, :], (BLOCK_K // 8, 8)),
(BLOCK_K,),
)
shifts_full = tl.broadcast_to(shifts_1d[None, :], (BLOCK_N, BLOCK_K))
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for k_start in range(0, tl.cdiv(K, BLOCK_K)):
offs_k = k_start * BLOCK_K + tl.arange(0, BLOCK_K)
mask_k = offs_k < K
a_ptrs = a_ptr + offs_m[:, None] * stride_am + offs_k[None, :]
mask_a = (offs_m[:, None] < M) & mask_k[None, :]
a = tl.load(a_ptrs, mask=mask_a, other=0.0)
offs_k8 = k_start * (BLOCK_K // 8) + tl.arange(0, BLOCK_K // 8)
b_ptrs = b_ptr + offs_n[:, None] * stride_bn + offs_k8[None, :]
mask_b = (offs_n[:, None] < N) & (offs_k8[None, :] < K8)
b_packed = tl.load(b_ptrs, mask=mask_b, other=0)
b = tl.interleave(b_packed, b_packed)
b = tl.interleave(b, b)
b = tl.interleave(b, b)
b = (b >> shifts_full) & 0xF
group_idx = (k_start * BLOCK_K) // group_size
scale_ptrs = scales_ptr + offs_n * stride_sn + group_idx
scale_mask = offs_n < N
scales = tl.load(scale_ptrs, mask=scale_mask, other=1.0)
if HAS_ZP:
zp_ptrs = zp_ptr + (offs_n // 8) * stride_sn + group_idx
zp_word = tl.load(zp_ptrs, mask=scale_mask, other=0)
zp_raw = (zp_word >> (4 * (offs_n % 8))) & 0xF
b_fp = (b - zp_raw[:, None]).to(scales.dtype) * scales[:, None]
else:
b_fp = (b - ZP_BIAS).to(scales.dtype) * scales[:, None]
b_fp_t = tl.trans(b_fp)
accumulator += tl.dot(a, b_fp_t, out_dtype=tl.float32)
c = accumulator.to(c_ptr.type.element_ty)
c_ptrs = c_ptr + offs_m[:, None] * N + offs_n[None, :]
mask_c = (offs_m[:, None] < M) & (offs_n[None, :] < N)
tl.store(c_ptrs, c, mask=mask_c)
# Per-shape (group_size, K, N) -> (BLOCK_M, BLOCK_N, BLOCK_K, num_warps,
# num_stages) tile-config overrides for prefill (M <= 128) on gfx1x.
# Picked by sweeping benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py + a
# per-config sweep script; only added when better than the generic heuristic
# by > 20% at M=128. Re-run benchmarks after edits.
_GFX1X_PREFILL_OVERRIDES: dict[tuple[int, int, int], tuple[int, int, int, int, int]] = {
# SmolLM2-1.7B-Instruct-AWQ (gs=32, K=2048; gs forces BLOCK_K to 32 so
# widen BLOCK_M and let Triton pipeline 4 stages to amortize the small
# K-tile).
(32, 2048, 6144): (128, 32, 32, 4, 4), # qkv_proj
(32, 2048, 2048): (128, 32, 32, 4, 4), # o_proj
(32, 2048, 16384): (128, 32, 32, 4, 4), # gate_up_proj
(32, 8192, 2048): (128, 64, 32, 8, 2), # down_proj
# Qwen3-8B-quantized.w4a16 (gs=128, K=4096 / 12288). For these K's a
# full BLOCK_K=group_size with no software pipelining beats the generic
# 64x64x64 — single-stage keeps register pressure down.
(128, 4096, 6144): (128, 64, 128, 8, 1), # qkv_proj
(128, 4096, 4096): (128, 32, 128, 4, 1), # o_proj
(128, 4096, 24576): (64, 32, 128, 2, 1), # gate_up_proj
(128, 12288, 4096): (128, 64, 128, 8, 1), # down_proj
}
def triton_w4a16_skinny_fmt_gemm(
a: torch.Tensor, # [M, K] fp16/bf16
b_q: torch.Tensor, # [N, K//8] int32 (ExLlama shuffle packed)
scales: torch.Tensor, # [N, K//G] fp16/bf16
group_size: int,
zp_bias: int = 8,
zp: torch.Tensor | None = None, # [N//8, K//G] int32 zero-points
) -> torch.Tensor:
"""Fused W4A16 GEMM reading from skinny weight format [N, K//8].
Args:
a: Activation matrix [M, K], float16 or bfloat16.
b_q: Packed weight matrix [N, K//8], int32 (ExLlama shuffle).
scales: Per-group scales [N, K//G], same dtype as a.
group_size: Quantization group size (resolved from -1 to K by caller).
zp_bias: Constant zero bias (default 8 for unsigned int4).
zp: Raw per-group zero-points [N//8, K//G] int32, row n at
word[n//8] bits 4*(n%8) (asymmetric). When provided,
dequant is (nibble - zp_raw) * scale.
Returns:
Output matrix [M, N], same dtype as a.
"""
assert a.stride(1) == 1, "Activation rows must be contiguous"
assert b_q.stride(1) == 1, "Weight rows must be contiguous"
assert scales.stride(1) == 1, "Scale rows must be contiguous"
stride_sn = scales.stride(0)
M, K = a.shape
N = b_q.shape[0]
K8 = K // 8
num_groups = K // group_size
stride_bn = b_q.stride(0)
stride_am = a.stride(0)
assert b_q.shape == (N, K8), f"b_q shape mismatch: {b_q.shape} vs ({N}, {K8})"
assert scales.shape == (N, num_groups), (
f"scales shape mismatch: {scales.shape} vs ({N}, {num_groups})"
)
if zp is not None:
assert zp.stride(1) == 1, "Zero-point rows must be contiguous"
assert zp.stride(0) == stride_sn, (
f"zp row stride {zp.stride(0)} must match scales {stride_sn}"
)
assert N % 8 == 0, f"N must be divisible by 8 for packed zp, got {N}"
assert zp.shape == (N // 8, num_groups), (
f"zp shape mismatch: {zp.shape} vs ({N // 8}, {num_groups})"
)
has_zp = zp is not None
c = torch.empty((M, N), dtype=a.dtype, device=a.device)
# num_stages stays None unless a per-shape override sets it, so the
# generic heuristics fall back to Triton's default pipeline depth.
num_stages: int | None = None
if _on_gfx12x():
# Tuned on gfx1201 (Radeon AI PRO R9700, 32 CUs, 32-wide wavefronts)
# using Llama-3.1-8B AWQ weight shapes with group_size=128.
if M <= 32:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 16, 16, 128, 4
elif M <= 64:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 32, 128, 8
elif N > K: # wide N (e.g. qkv_proj, gate_up_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 32, 64, 8
else: # N ~= K (e.g. o_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 32, 64, 128, 4
elif M <= 128:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 16, 64, 1
elif N >= 2 * K: # very wide N (e.g. gate_up_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 128, 64, 8
else: # N ~= K (e.g. o_proj, qkv_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 64, 8
elif M <= 512:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
elif N >= 4 * K: # very wide N (e.g. gate_up_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 128, 64, 8
else:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 128, 64, 8
else:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
elif N >= 4 * K: # very wide N (e.g. gate_up_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 256, 64, 64, 8
else:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 128, 32, 8
elif _on_gfx1151():
# Tuned on gfx1151 (Strix Halo, 40 CUs, 32-wide wavefronts)
# using Qwen3-4B weight shapes with group_size=128.
# Per-shape overrides for known prefill regressions live in a small
# lookup table — see _GFX1X_PREFILL_OVERRIDES below. Re-run
# benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py after edits.
override = (
_GFX1X_PREFILL_OVERRIDES.get((group_size, K, N)) if M <= 128 else None
)
if override is not None:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages = override
elif M <= 32:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 32, 32, 128, 4
elif M <= 64:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 32, 4
elif M <= 128:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 16, 64, 1
elif N > K: # wide N (e.g. qkv_proj, gate_up_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 64, 4
else: # N ~= K (e.g. o_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 32, 64, 4
elif M <= 1024:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 64, 64, 4
elif N >= 4 * K: # very wide N (e.g. gate_up_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
else:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 64, 128, 32, 4
else:
if K >= 2 * N: # tall K (e.g. down_proj)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 512, 32, 16
else:
BLOCK_M, BLOCK_N, BLOCK_K, num_warps = 128, 64, 64, 8
else:
num_warps = 4
if M <= 32:
BLOCK_M, BLOCK_N, BLOCK_K = 32, 64, 32
elif M <= 64:
BLOCK_M, BLOCK_N, BLOCK_K = 64, 64, 32
else:
BLOCK_M, BLOCK_N, BLOCK_K = 128, 128, 32
# The kernel loads one scale per BLOCK_K tile, so BLOCK_K must not
# exceed group_size — otherwise elements in the tile that belong to
# a different group would get the wrong scale.
BLOCK_K = min(BLOCK_K, group_size)
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
extra_kwargs = {} if num_stages is None else {"num_stages": num_stages}
_triton_w4a16_skinny_fmt_kernel[grid](
a,
b_q,
scales,
zp if has_zp else scales, # dummy pointer when no zp (unused)
c,
M,
N,
K,
K8,
num_groups,
stride_sn,
stride_bn,
stride_am,
group_size=group_size,
ZP_BIAS=zp_bias,
HAS_ZP=has_zp,
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
num_warps=num_warps,
**extra_kwargs,
)
return c
# ---------------------------------------------------------------------------
# Weight packing
# ---------------------------------------------------------------------------
def pack_int4_exllama_shuffle(w_uint4: torch.Tensor) -> torch.Tensor:
"""Pack uint4 values into ExLlama shuffle format: [N, K] -> [N, K//8] int32.
Each int32 packs 8 K-values with interleave order [0,2,4,6,1,3,5,7].
"""
N_dim, K_dim = w_uint4.shape
assert K_dim % 8 == 0
g = w_uint4.to(torch.uint8).view(N_dim, K_dim // 8, 8).to(torch.int32)
return (
g[:, :, 0]
| (g[:, :, 2] << 4)
| (g[:, :, 4] << 8)
| (g[:, :, 6] << 12)
| (g[:, :, 1] << 16)
| (g[:, :, 3] << 20)
| (g[:, :, 5] << 24)
| (g[:, :, 7] << 28)
)
# gfx11 row strides alias in the vector caches when they land on a power-of-two
# multiple.
_WEIGHT_CLIFF_BYTES = 1024 # packed weight row, K/2 B
_ACT_CLIFF_BYTES = 2048 # activation row, K*2 B
_STRIDE_PAD_BYTES = 128 # one cache line
def _weight_pad_bytes(row_bytes: int) -> int:
"""Bytes to add to a packed weight row stride to move it off the cliff.
Only strides that are a multiple of ``_WEIGHT_CLIFF_BYTES`` are moved, and
only by one cache line; padding a stride that is already off the cliff
costs memory and measures slower.
"""
if row_bytes % _WEIGHT_CLIFF_BYTES:
return 0
return _STRIDE_PAD_BYTES
def _act_pad_bytes(row_bytes: int) -> int:
"""Bytes to add to an activation row stride to move it off the cliff."""
if row_bytes % _ACT_CLIFF_BYTES:
return 0
return _STRIDE_PAD_BYTES
def pack_skinny_int4(unpacked: torch.Tensor) -> torch.Tensor:
"""Pack [N, K] uint4 into the skinny weight layout the kernels consume.
ExLlama shuffle to [N, K//8] int32, viewed as int8 [N, K//2]. On gfx1151
the row stride is nudged off the cliff (see ``_weight_pad_bytes``).
"""
shuffled = pack_int4_exllama_shuffle(unpacked)
n_rows, k8 = shuffled.shape
pad_int32 = _weight_pad_bytes(k8 * 4) // 4
if not (pad_int32 and _on_gfx1151()):
return shuffled.contiguous().view(torch.int8)
padded = torch.empty(
(n_rows, k8 + pad_int32), dtype=torch.int32, device=shuffled.device
)
padded[:, :k8].copy_(shuffled)
# The int8 view keeps stride(0) = 4 * (k8 + pad_int32) bytes, and
# .view(torch.int32) at apply time recovers the [N, K//8] int32 view.
return padded.view(torch.int8)[:, : k8 * 4]
# ---------------------------------------------------------------------------
# Hybrid dispatch logic
# ---------------------------------------------------------------------------
def _pad_activation_rows(x_2d: torch.Tensor) -> torch.Tensor:
"""Copy ``x_2d`` into a row-padded buffer when its row stride is on the cliff.
The packed weight is padded once at load time, but activations are produced
fresh every step, so this materialises a padded copy.
"""
pad_bytes = _act_pad_bytes(x_2d.shape[1] * x_2d.element_size())
if not (pad_bytes and _on_gfx1151()):
return x_2d
pad_elems = pad_bytes // x_2d.element_size()
buf = torch.empty(
(x_2d.shape[0], x_2d.shape[1] + pad_elems),
dtype=x_2d.dtype,
device=x_2d.device,
)
padded = buf[:, : x_2d.shape[1]]
padded.copy_(x_2d)
return padded
def _rdna_hybrid_w4a16_apply_impl(
x_2d: torch.Tensor,
w_q: torch.Tensor,
w_s: torch.Tensor,
w_zp: torch.Tensor | None,
bias: torch.Tensor | None,
cu_count: int,
group_size: int,
) -> torch.Tensor:
"""Dispatch between skinny GEMM and Triton based on batch size M.
``w_zp`` is [N//8, K//G] int32 for asymmetric layers, None for symmetric.
"""
import vllm._custom_ops as ops
M = x_2d.shape[0]
K = x_2d.shape[1]
N = w_q.shape[0]
if M <= MAX_SKINNY_BATCH_SIZE and (
K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS or _on_gfx115x()
):
# record_function is not torch.compile-safe; use nullcontext when
# compiling to keep the op traceable.
ctx = (
nullcontext()
if torch.compiler.is_compiling()
else torch.profiler.record_function(f"wvsplitk_int4 {M}x{N}x{K}")
)
with ctx:
return ops.wvSplitK_int4_g(w_q, x_2d, w_s, cu_count, group_size, w_zp, bias)
ctx = (
nullcontext()
if torch.compiler.is_compiling()
else torch.profiler.record_function(f"hybrid_triton_w4a16 {M}x{N}x{K}")
)
with ctx:
output = triton_w4a16_skinny_fmt_gemm(
a=_pad_activation_rows(x_2d),
b_q=w_q.view(torch.int32),
scales=w_s,
group_size=group_size,
zp=w_zp,
)
if bias is not None:
output.add_(bias)
return output
def _rdna_hybrid_w4a16_apply_fake(
x_2d: torch.Tensor,
w_q: torch.Tensor,
w_s: torch.Tensor,
w_zp: torch.Tensor | None,
bias: torch.Tensor | None,
cu_count: int,
group_size: int,
) -> torch.Tensor:
M = x_2d.size(0)
N = w_q.size(0)
return torch.empty((M, N), dtype=x_2d.dtype, device=x_2d.device)
direct_register_custom_op(
op_name="rdna_hybrid_w4a16_apply",
op_func=_rdna_hybrid_w4a16_apply_impl,
mutates_args=[],
fake_impl=_rdna_hybrid_w4a16_apply_fake,
)
class RDNAHybridW4A16LinearKernel(MPLinearKernel):
"""Hybrid W4A16 kernel: HIP skinny for decode, Triton for prefill.
Stores the weights once as int8 [N, K//2] (ExLlama shuffle packed). The
HIP skinny kernel reads it directly; the triton kernel reinterprets the
same buffer as int32 [N, K//8] via a view, so there is no dual weight
storage.
"""
SUPPORTED_QUANT_TYPES = [
scalar_types.uint4b8, # symmetric GPTQ (bias=8)
scalar_types.uint4, # asymmetric (zero_points)
]
@classmethod
def get_min_capability(cls) -> int:
# Arch filtering is handled by can_implement (_on_gfx1x check)
return 0
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "RDNAHybridW4A16LinearKernel only targets ROCm"
if not _on_gfx1x():
return False, "RDNAHybridW4A16LinearKernel only targets gfx11/gfx12"
if c.weight_type not in cls.SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type {c.weight_type} not supported; "
f"supported: {cls.SUPPORTED_QUANT_TYPES}",
)
if c.act_type not in (torch.float16, torch.bfloat16):
return False, "requires float16 or bfloat16 activations"
gs = c.group_size
if gs not in SUPPORTED_GROUP_SIZES:
return (
False,
f"Group size {gs} not supported; supported: {SUPPORTED_GROUP_SIZES}",
)
K = c.partition_weight_shape[0]
if K % 16 != 0:
return False, f"K={K} must be divisible by 16"
if K % gs != 0:
return (
False,
f"K={K} must be divisible by group_size={gs}",
)
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
c = self.config
w_q_raw = getattr(layer, self.w_q_name)
w_s_raw = getattr(layer, self.w_s_name)
unpacked = unpack_quantized_values_into_int32(
w_q_raw.data, c.weight_type, packed_dim=w_q_raw.packed_dim
)
# AWQ weights arrive as (K, N) with output_dim=1;
# compressed-tensors arrive as (N, K) with output_dim=0.
if getattr(w_q_raw, "output_dim", 0) != 0:
unpacked = unpacked.t().contiguous()
# Store as int8; Triton reinterprets via .view(torch.int32) at apply time.
w_q_skinny = pack_skinny_int4(unpacked)
permute_param_layout_(w_s_raw, input_dim=1, output_dim=0)
w_s_skinny = w_s_raw.data.contiguous()
if c.zero_points:
assert self.w_zp_name is not None
w_zp_raw = getattr(layer, self.w_zp_name)
permute_param_layout_(w_zp_raw, input_dim=1, output_dim=0, packed_dim=0)
w_zp = w_zp_raw.data.contiguous()
self._transform_param(layer, self.w_zp_name, lambda x: w_zp)
self._transform_param(layer, self.w_q_name, lambda x: w_q_skinny)
self._transform_param(layer, self.w_s_name, lambda x: w_s_skinny)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.platform_utils import num_compute_units
c = self.config
w_q, w_s, w_zp = self._get_weight_params(layer)
x_2d = x.reshape(-1, x.shape[-1])
N = w_q.shape[0]
out_shape = x.shape[:-1] + (N,)
cu_count = num_compute_units()
output = torch.ops.vllm.rdna_hybrid_w4a16_apply(
x_2d,
w_q,
w_s,
w_zp,
bias,
cu_count,
c.group_size,
)
return output.reshape(out_shape)
@@ -1,737 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Portions adapted from ROCm/aiter (https://github.com/ROCm/aiter/pull/5750):
# SPDX-License-Identifier: MIT
# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``).
``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per
row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in
the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being
expanded to every row. Each weight scale byte is then read once per 32 output
rows, and small-M shapes can use the packed kernel below.
Two kernels, picked per shape from a table tuned on MI355X:
* a tiled kernel for larger M;
* a packed kernel for small M, which fills the MFMA's rows with K panels
instead of tokens and keeps only the matching-panel (block-diagonal)
products, so a handful of tokens still streams the weight at full width.
Either can split K. The partials are normally summed in the same launch by
the last program of each output tile to finish, in split order, so the result
does not depend on scheduling. Otherwise a second launch reduces them.
The packed kernel and the in-launch split-K reduction on one XCD
(``_split_tile``, ``_sum_splits``, ``_split_counters``, ``_k_partition``) are
adapted from the group32 GEMM in ROCm/aiter#5750.
"""
from typing import NamedTuple
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
BLOCK_ROWS = 32
# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is
# a multiple of the XCD count in every partition mode. _split_tile hardcodes
# the same stride.
_XCD_STRIDE = 8
# One arrival counter per output tile of an in-launch split-K GEMM.
_SPLIT_COUNTERS = 1 << 16
@triton.jit
def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr):
"""(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on
one XCD, so they meet in that XCD's L2."""
pid = tl.program_id(0)
tile = pid // (8 * SPLITS) * 8 + pid % 8
grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N
return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS
@triton.jit
def _sum_splits(
acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr
):
"""Publish this split's partial; the tile's last arrival sums all of them
in split order and re-arms the counter."""
BM: tl.constexpr = acc.shape[0]
BN: tl.constexpr = acc.shape[1]
local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :]
tl.store(slot_ptr + split * BM * BN + local, acc)
# The partial must be in L2 before the arrival is counted. The splits
# share an XCD, so no device-scope release (an L2 writeback) is needed.
tl.inline_asm_elementwise(
"s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1
)
tl.debug_barrier()
if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1:
total = tl.zeros((BM, BN), dtype=tl.float32)
for s in tl.static_range(SPLITS):
total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv")
tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask)
tl.store(count_ptr, 0)
@triton.jit(do_not_specialize=["M"])
def _block32_tiled_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
slot_ptr,
count_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
stride_om,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
K_PER_SPLIT: tl.constexpr,
N_FIRST: tl.constexpr,
EVEN_K: tl.constexpr,
FUSED_SPLITS: tl.constexpr,
):
if FUSED_SPLITS > 1:
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
if pid_m * BLOCK_M >= M:
return
else:
# N_FIRST launches output-column tiles fastest, so consecutive
# programs share an activation tile.
if N_FIRST:
pid_n, pid_m = tl.program_id(0), tl.program_id(1)
else:
pid_m, pid_n = tl.program_id(0), tl.program_id(1)
pid_k = tl.program_id(2)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
m_mask = offs_m < M
n_mask = offs_n < N
k0 = pid_k * K_PER_SPLIT
offs_k = k0 + tl.arange(0, BLOCK_K)
offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32)
x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :]
xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :]
w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :]
ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for kk in range(0, K_PER_SPLIT, BLOCK_K):
if EVEN_K:
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
else:
k_ok = (offs_k + kk) < K
s_ok = (offs_sk + kk // 32) < K // 32
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=0)
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=0)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
x_ptrs += BLOCK_K
w_ptrs += BLOCK_K
xs_ptrs += BLOCK_K // 32
ws_ptrs += BLOCK_K // 32
out_mask = m_mask[:, None] & n_mask[None, :]
if FUSED_SPLITS > 1:
_sum_splits(
acc,
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
out_mask,
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
count_ptr + tile,
pid_k,
FUSED_SPLITS,
)
else:
out_ptrs = (
out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :]
)
tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
@triton.jit(do_not_specialize=["M"])
def _block32_packed_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
slot_ptr,
count_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
stride_om,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
K_PACK: tl.constexpr,
K_PER_SPLIT: tl.constexpr,
EVEN_K: tl.constexpr,
FUSED_SPLITS: tl.constexpr,
):
if FUSED_SPLITS > 1:
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
if pid_m * BLOCK_M >= M:
return
k_start = pid_k * K_PER_SPLIT
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
else:
pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
if K_PER_SPLIT >= K:
k_start = 0
k_stop = K
else:
k_start = pid_k * K_PER_SPLIT
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
# Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise
# for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices.
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK
x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK
w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK
ks = tl.arange(0, BLOCK_K)
gs = tl.arange(0, BLOCK_K // 32)
row_ok = rows < M
col_ok = cols < N
acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32)
for base in range(k_start, k_stop, BLOCK_K * K_PACK):
xk = base + x_panel[:, None] * BLOCK_K + ks[None, :]
wk = base + w_panel[:, None] * BLOCK_K + ks[None, :]
xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
x_mask = row_ok[:, None]
w_mask = col_ok[:, None]
xs_mask = row_ok[:, None]
ws_mask = col_ok[:, None]
if not EVEN_K:
x_mask = x_mask & (xk < K)
w_mask = w_mask & (wk < K)
xs_mask = xs_mask & (xg < K // 32)
ws_mask = ws_mask & (wg < K // 32)
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=0)
ws = tl.load(
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=0
)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
# Keep only products of matching K panels: the block diagonal.
blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3)
pair = tl.arange(0, K_PACK)
diagonal = tl.where(
pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0
)
y = tl.sum(tl.sum(diagonal, 3), 2)
out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :]
out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N)
if FUSED_SPLITS > 1:
_sum_splits(
y,
out_ptrs,
out_mask,
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
count_ptr + tile,
pid_k,
FUSED_SPLITS,
)
else:
tl.store(
out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask
)
@triton.jit(do_not_specialize=["M"])
def _block32_splitk_reduce_kernel(
partial_ptr,
out_ptr,
M,
N: tl.constexpr,
stride_om,
SPLIT_K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for split in tl.static_range(SPLIT_K):
acc += tl.load(
partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :],
mask=mask,
other=0.0,
)
tl.store(
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
acc.to(out_ptr.dtype.element_ty),
mask=mask,
)
class _Config(NamedTuple):
packed: bool
block_m: int
block_n: int
block_k: int
# K panels per MFMA tile (packed) or K partitions reduced by a second
# launch (tiled).
k_split: int
n_first: bool
num_warps: int
num_stages: int
matrix_instr_nonkdim: int
waves_per_eu: int
# K partitions summed within the launch.
fused_splits: int = 1
def _tiled(
bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1
) -> _Config:
return _Config(
False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused
)
def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config:
return _Config(
True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused
)
# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at
# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A
# tier serves every M up to its bound; larger M reuse the last tier.
_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = {
# shared expert gate_up, TP4
(1152, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)),
(2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)),
(4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)),
(8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)),
(16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)),
(24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)),
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)),
(48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)),
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)),
(96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)),
(128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)),
(160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
(192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)),
(256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)),
(384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
(512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)),
(1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)),
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
],
# fused wq_a|wkv (replicated)
(1792, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
(16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)),
(32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)),
(48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)),
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)),
(96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)),
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)),
(384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)),
],
# shared expert gate_up, TP2
(2304, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
(16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)),
(24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)),
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)),
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)),
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)),
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)),
(256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
(4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)),
],
# indexer wq_b
(4096, 1280): [
(1, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
(2, _packed(4, 16, 256, 4, 4, 2, 16, 0)),
(4, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
(8, _packed(8, 16, 256, 4, 4, 3, 16, 1)),
(16, _packed(16, 16, 512, 2, 2, 3, 16, 1)),
(32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)),
(64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)),
(128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)),
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)),
(256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)),
],
# shared expert down, TP4
(5120, 576): [
(8, _packed(8, 32, 256, 2, 2, 2, 16, 1)),
(16, _packed(16, 32, 512, 1, 2, 2, 16, 0)),
(32, _packed(32, 32, 256, 1, 8, 3, 16, 1)),
(64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
(96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)),
(128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)),
(192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)),
(256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)),
(768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
],
# shared expert down, TP2
(5120, 1152): [
(1, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
(32, _packed(32, 32, 512, 1, 2, 3, 16, 1)),
(128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)),
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)),
],
# wo_b, TP4
(5120, 2048): [
(1, _packed(4, 32, 256, 4, 2, 2, 16, 0)),
(4, _packed(4, 32, 256, 4, 2, 2, 16, 1)),
(8, _packed(8, 32, 256, 4, 2, 3, 16, 0)),
(16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)),
(32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)),
(64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
(96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)),
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)),
],
# wo_b, TP2
(5120, 4096): [
(4, _packed(4, 32, 512, 4, 2, 2, 16, 0)),
(8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)),
(16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)),
(24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)),
(32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)),
(48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)),
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)),
(128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
(3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
],
# wq_b, TP4
(8192, 1280): [
(1, _packed(8, 32, 256, 2, 4, 3, 16, 1)),
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
(16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)),
(32, _packed(32, 32, 512, 1, 2, 3, 16, 0)),
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)),
(64, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
],
# wq_b, TP2
(16384, 1280): [
(2, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
(4, _packed(4, 32, 128, 4, 4, 2, 16, 0)),
(8, _packed(8, 32, 256, 2, 4, 2, 16, 0)),
(32, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
(64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)),
(128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
],
}
def _default_config(M: int, N: int, K: int) -> _Config:
"""Untuned shapes: the tiers most tuned shapes settle on."""
if M <= 8:
return _packed(8, 16, 256, 2, 2, 3, 16, 0)
if M <= 32:
return _packed(32, 32, 256, 1, 2, 3, 16, 0)
if M <= 256:
return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0)
return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)
def _config(M: int, N: int, K: int) -> _Config:
tiers = _TUNED.get((N, K))
if tiers is None:
return _default_config(M, N, K)
for max_m, cfg in tiers:
if max_m >= M:
return cfg
return tiers[-1][1]
_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {}
def _split_counters(device: torch.device) -> torch.Tensor | None:
"""Zeroed arrival counters, one set per stream so concurrent launches never
share a tile's counter. Each launch leaves the counters it used at zero.
None when a stream first needs them inside a CUDA graph capture: that
allocation would come from the graph's pool, where it can take the address
of an intermediate that every replay overwrites. vLLM warms up on its
capture streams first, so this only affects captures without a warmup.
"""
key = (device.index, torch.accelerator.current_stream(device).stream_id)
counters = _split_counters_by_stream.get(key)
if counters is None:
if torch.cuda.is_current_stream_capturing():
return None
counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device)
_split_counters_by_stream[key] = counters
return counters
def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]:
"""K per split, rounded up to whole steps, and the number of non-empty
splits."""
size = triton.cdiv(triton.cdiv(K, splits), step) * step
return size, triton.cdiv(K, size)
def _launch(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out: torch.Tensor,
cfg: _Config,
) -> None:
M, K = x.shape
N = weight.shape[0]
opts = dict(
num_warps=cfg.num_warps,
num_stages=cfg.num_stages,
matrix_instr_nonkdim=cfg.matrix_instr_nonkdim,
waves_per_eu=cfg.waves_per_eu,
)
grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n)
tiles = grid_m * grid_n
step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k
splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split
k_per_split, splits = _k_partition(K, splits, step)
counters = None
if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS:
counters = _split_counters(out.device)
if counters is not None:
fused = splits
target = out
slots = torch.empty(
tiles * fused * cfg.block_m * cfg.block_n,
dtype=torch.float32,
device=out.device,
)
grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,)
else:
# Without counters, split K through a second launch.
fused = 1
target = (
out
if splits == 1
else torch.empty((splits, M, N), dtype=torch.float32, device=x.device)
)
slots = counters = target # Unused by the kernels.
if cfg.packed or not cfg.n_first:
grid = (grid_m, grid_n, splits)
else:
grid = (grid_n, grid_m, splits)
args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K)
stride = out.stride(0) if splits == 1 or fused > 1 else N
if cfg.packed:
_block32_packed_kernel[grid](
*args,
stride,
BLOCK_M=cfg.block_m,
BLOCK_N=cfg.block_n,
BLOCK_K=cfg.block_k,
K_PACK=cfg.k_split,
K_PER_SPLIT=k_per_split,
EVEN_K=K % step == 0,
FUSED_SPLITS=fused,
**opts,
)
else:
_block32_tiled_kernel[grid](
*args,
stride,
BLOCK_M=cfg.block_m,
BLOCK_N=cfg.block_n,
BLOCK_K=cfg.block_k,
K_PER_SPLIT=k_per_split,
N_FIRST=cfg.n_first,
EVEN_K=K % k_per_split == 0,
FUSED_SPLITS=fused,
**opts,
)
if fused == 1 and splits > 1:
_block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))](
target,
out,
M,
N,
out.stride(0),
SPLIT_K=splits,
BLOCK_M=32,
BLOCK_N=64,
)
def _rocm_mxfp8_block32_gemm_impl(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
M, K = x.shape
out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device)
if M > 0:
_launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K))
return out
def _rocm_mxfp8_block32_gemm_fake(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype)
# An opaque op: the kernel and tile choice branch on the token count, which a
# compiled caller must not freeze at its trace-time value.
direct_register_custom_op(
op_name="rocm_mxfp8_block32_gemm",
op_func=_rocm_mxfp8_block32_gemm_impl,
fake_impl=_rocm_mxfp8_block32_gemm_fake,
)
def rocm_mxfp8_block32_gemm(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight.
Args:
x: [M, K] e4m3 activation, contiguous.
x_scale: [M, K / 32] E8M0 (uint8) activation scales.
weight: [N, K] e4m3 weight, contiguous.
weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales.
out_dtype: Output dtype.
Returns:
The [M, N] product.
"""
return torch.ops.vllm.rocm_mxfp8_block32_gemm(
x, x_scale, weight, weight_scale, out_dtype
)
@@ -1,295 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Native MXFP8 linear GEMM for AMD CDNA4 (gfx950) via Triton ``tl.dot_scaled``.
Consumes the FP8 E4M3 weights + E8M0 block scales directly (no dequant-to-BF16);
activations are MXFP8-quantized per token. Uses the CDNA4 hardware microscaling
matrix cores. ``dot_scaled`` tiles K by 128; a weight whose K is not a multiple
of that is dequantized to BF16 once at load and served by a plain linear
instead, since ``can_implement`` does not filter on K.
"""
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm
@triton.jit
def _mxfp8_linear_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
M,
N,
K,
stride_xm,
stride_xk,
stride_xsm,
stride_xsk,
stride_wn,
stride_wk,
stride_wsn,
stride_wsk,
stride_om,
stride_on,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
offs_sk = tl.arange(0, BLOCK_K // 32)
m_mask = offs_m < M
n_mask = offs_n < N
x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk
xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(0, tl.cdiv(K, BLOCK_K)):
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3")
x_ptrs += BLOCK_K * stride_xk
w_ptrs += BLOCK_K * stride_wk
xs_ptrs += (BLOCK_K // 32) * stride_xsk
ws_ptrs += (BLOCK_K // 32) * stride_wsk
o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on
tl.store(
o_ptrs, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None] & n_mask[None, :]
)
def _mxfp8_dot_scaled_linear(
x: torch.Tensor, # [M, K] bf16/fp16
w: torch.Tensor, # [N, K] fp8 e4m3
w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0)
) -> torch.Tensor:
M, K = x.shape
N = w.shape[0]
x_q, x_scale = mxfp8_e4m3_quantize(x)
out = torch.empty((M, N), dtype=x.dtype, device=x.device)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages = _select_cfg(M, N, K)
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
_mxfp8_linear_kernel[grid](
x_q,
x_scale,
w,
w_scale,
out,
M,
N,
K,
x_q.stride(0),
x_q.stride(1),
x_scale.stride(0),
x_scale.stride(1),
w.stride(0),
w.stride(1),
w_scale.stride(0),
w_scale.stride(1),
out.stride(0),
out.stride(1),
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
num_warps=num_warps,
num_stages=num_stages,
)
return out
# gfx950's async-copy LDS (default in the triton 3.8 this branch pins) leaves no
# room for num_stages=3 at BLOCK_K=256; use 2 there.
_BK256_STAGES = 3
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
if on_gfx950():
_BK256_STAGES = 2
def _select_cfg(M, N, K):
"""(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) — graph-tuned on gfx950.
The M-bucketed, shape-adaptive tile selection here is the speedup over the
upstream 2-bucket launcher. Tiles are pipelined (num_stages>=2, larger BLOCK_K)
and occupancy- and shape-aware: keyed on the LOCAL (M, N, K), so it adapts to the
TP-sharded shapes (e.g. MiniMax-M3 TP=4 vs TP=8, where local N and K differ) —
large-K prefill uses BLOCK_K=256; short-K (K=768) widens N. BLOCK_K must divide K
(the K-loop is unmasked), so every BLOCK_K below is guarded to be K-divisible
(served K: 384/768/1024/2048/6144).
"""
if M <= 64:
# decode (M in {1,32,64}): tiny-M GEMV is weight-BW + GPU-OCCUPANCY bound. The
# lever is NARROW BLOCK_N=16 (maximize N-tiles so more CUs stream the weight in
# parallel) + LARGE BLOCK_K (fewer K-iters, bigger coalesced weight loads).
# Tuned by CUDA-graph replay latency. Optimal at both TP=4 and TP=8.
if K % 1024 == 0: # K=2048, 6144 -> graph-best 16x16x1024 (all M)
return 16, 16, 1024, 2, 2
if K % 512 == 0:
return 16, 16, 512, 2, 3
if K % 256 == 0: # K=768 (shared_down) -> graph-best 16x32x256
return 16, 32, 256, 4, 3
return 16, 32, 128, 4, 3
# mid-M (65..256) on SMALL local-N: still occupancy-bound (a 64x64 tile makes too
# few N-tiles), so the narrow-BLOCK_N decode-style tile fills the CUs better.
# N<=1536 covers the real fused-qkv local N at TP=8: q heads shard but the GQA KV
# (4) + sparse-indexer (4) heads are < TP=8, so vLLM replicates them to 1/rank ->
# N = 1024 + 4*128 = 1536 (not 2560/2=1280). For the wider 1280<N<=1536 band the
# narrow tile only wins up to M=128 (at M=256 the 64x64 tile is better), so cap it
# there; N<=1280 keeps the narrow tile through M=256. TP=4 qkv N=2560 is unchanged.
if (M <= 256 and N <= 1280) or (M <= 128 and N <= 1536):
if K % 1024 == 0:
return 16, 16, 1024, 2, 2
if K % 512 == 0:
return 16, 16, 512, 2, 3
if K % 256 == 0:
return 16, 16, 256, 2, 3
return 16, 16, 128, 2, 3
# right-sized launch grid (host-side), used to gate the tall 256-BLOCK_M tile.
occ = triton.cdiv(M, 256) * triton.cdiv(N, 128)
if K <= 1024: # short-K (shared_down K=384/768; TP=8 o_proj K=1024)
if M <= 256:
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
# large prefill. (The former 256x128x256 tall tile was faster only on triton
# 3.6; on triton 3.7 its large BLOCK_M register/LDS footprint spills or hits
# "out of resources", so use 128x128x256 -- within the known-good footprint.)
if M >= 4096 and K >= 1024 and K % 256 == 0 and occ >= 256:
return 128, 128, 256, 8, _BK256_STAGES
return 128, 128, 128, 8, 3
# large-K (K >= 2048). BLOCK_K is K-divisibility-guarded (the K-loop is unmasked):
# served large-K is 2048/6144 (%256==0), but fall back to 128 (always divides, since
# the entry requires K%128==0) for any other K to stay correct.
if M <= 256: # conc~128 decode + small prefill chunk: occupancy tile
if K % 512 == 0:
return 64, 64, 512, 8, 2
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
if M <= 1024: # medium prefill chunk: BN=64 keeps small-N occupied
return (128, 64, 256, 8, 3) if K % 256 == 0 else (128, 64, 128, 8, 3)
# large prefill (M > 1024). The previously graph-tuned tall 256x128x256 and deep
# 128x128x512 tiles won only on triton 3.6; on triton 3.7 their larger BLOCK_M /
# BLOCK_K register+LDS footprint spills (or hits "out of resources" on stricter
# ROCm/triton builds). The 128x128x256 tile is equal-or-faster on triton 3.7 (the
# M=4096,N=2048,K=6144 shape: ~104us vs the 256x128x256 tile's ~184us), within ~5%
# on 3.6, and inside the footprint of the tiles used elsewhere in this selector.
# Covers the qkv-class local N=1536 (TP=8 qkv / TP=4 shared_gate_up) and the deep-K
# / very-large-M shapes.
if K % 256 == 0 and (1280 < N <= 1536 or (occ >= 128 and (K >= 4096 or M >= 4096))):
return 128, 128, 256, 8, _BK256_STAGES
# small local-N (e.g. TP=8 shared_gate_up N=768): a 64-wide BLOCK_N doubles the
# N-tile count -> better CU fill than 128x128 at this mid-large M (~1.4x there).
if N <= 1024 and K % 256 == 0:
return 128, 64, 256, 8, 3
return (128, 128, 256, 8, 2) if K % 256 == 0 else (128, 256, 128, 8, 3)
_DOT_SCALED_K_ALIGN = 128
def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None:
"""The [N / 32, K / 32] block scales, if every 32 rows share their scales."""
N = weight_scale.shape[0]
if N % BLOCK_ROWS != 0:
return None
blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1)
if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)):
return None
return blocks[:, 0].contiguous()
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
supports_pre_processed_weights = True
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "not ROCm"
# supports_mx() == gfx95x (CDNA4 native microscaling hardware). On other
# archs dot_scaled would upcast to BF16, so the kernel selector falls
# through to the BF16 emulation (hipBLASLt) path instead.
if not current_platform.supports_mx():
return False, "native MX requires CDNA4 (gfx95x)"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.data # [N, K] fp8
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
# rocm_mxfp8_block32_gemm's split-K varies with M.
block_scale = (
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
)
if block_scale is not None:
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
# back from the loader expanded per row; keep the blocks and use
# the block-scaled GEMM, which also handles any K % 32 == 0.
weight_scale = block_scale
elif K % _DOT_SCALED_K_ALIGN != 0:
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if layer.weight_scale.dtype != MXFP8_SCALE_DTYPE:
raise ValueError(
f"Expected {MXFP8_SCALE_DTYPE} weight_scale, got "
f"{layer.weight_scale.dtype}."
)
out_shape = (*x.shape[:-1], layer.weight.shape[0])
x2d = x.reshape(-1, x.shape[-1])
if layer.weight_scale.shape[0] != layer.weight.shape[0]:
# One scale row per 32 weight rows (see process_weights_after_loading).
x_q, x_scale = mxfp8_e4m3_quantize(x2d)
out = rocm_mxfp8_block32_gemm(
x_q, x_scale, layer.weight, layer.weight_scale, x.dtype
)
elif layer.weight.element_size() >= 2:
out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype))
elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0:
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
else:
# process_weights_after_loading dequantizes these weights once, so
# this runs only if it did not; dot_scaled would be invalid here.
w_bf16 = dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale)
out = torch.nn.functional.linear(x2d, w_bf16).to(x.dtype)
out = out.reshape(out_shape)
if bias is not None:
out = out + bias
return out
@@ -28,9 +28,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
PerTensorTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
ROCmFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
@@ -62,7 +59,6 @@ __all__ = [
"ChannelWiseTorchFP8ScaledMMLinearKernel",
"PerTensorTorchFP8ScaledMMLinearKernel",
"RowWiseTorchFP8ScaledMMLinearKernel",
"ROCmFP8ScaledMMLinearKernel",
"TritonFP8ScaledMMLinearKernel",
"TritonInt8ScaledMMLinearKernel",
"ZentorchInt8ScaledMMLinearKernel",
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import direct_register_custom_op
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
)
def rocm_per_tensor_float_w8a8_scaled_mm_impl(
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor,
) -> torch.Tensor:
if (
A.shape[0] <= 4
and B.shape[0] % 16 == 0 # M TODO: needed?
and B.shape[1] % 16 == 0 # K
and ((bias is None) or (bias.dtype == out_dtype))
):
output = ops.wvSplitKQ(
B.t(),
A,
out_dtype,
As,
Bs,
num_compute_units(),
bias,
)
# Fallback
else:
output = torch._scaled_mm(
A,
B,
out_dtype=out_dtype,
scale_a=As,
scale_b=Bs,
bias=bias,
)
return output
def rocm_per_tensor_float_w8a8_scaled_mm_fake(
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor,
) -> torch.Tensor:
return A.new_empty((*A.shape[:-1], B.shape[1]), dtype=out_dtype)
if current_platform.is_rocm():
direct_register_custom_op(
op_name="rocm_per_tensor_float_w8a8_scaled_mm_impl",
op_func=rocm_per_tensor_float_w8a8_scaled_mm_impl,
fake_impl=rocm_per_tensor_float_w8a8_scaled_mm_fake,
)
class ROCmFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "requires ROCm."
from vllm.platforms.rocm import get_cdna_version, on_gfx12x, on_gfx1250
# wvSplitKQ (skinny GEMM) is excluded from the gfx1250 build.
if on_gfx1250():
return False, "wvSplitKQ (skinny GEMM) is not built on gfx1250"
# Restore RDNA4 (gfx12x) dropped by the get_cdna_version()>2 refactor.
if get_cdna_version() <= 2 and not on_gfx12x():
return False, "requires CDNA3+ (gfx942/gfx950) or RDNA4 (gfx12x)"
if not envs.VLLM_ROCM_USE_SKINNY_GEMM:
return False, "requires VLLM_ROCM_USE_SKINNY_GEMM to be enabled."
return True, None
@classmethod
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
per_tensor_activation_scales = (
c.activation_quant_key.scale.group_shape.is_per_tensor()
)
per_tensor_weight_scales = c.weight_quant_key.scale.group_shape.is_per_tensor()
if not (per_tensor_activation_scales and per_tensor_weight_scales):
return False, "requires per tensor activation and weight scales."
return True, None
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
output = torch.ops.vllm.rocm_per_tensor_float_w8a8_scaled_mm_impl(
A, B, out_dtype, As, Bs, bias
)
return torch.narrow(output, 0, 0, A.shape[0]).view(*output_shape)