[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests

- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm
  target path; drop the rocm requirement files
- Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules
  and retarget the linear kernel registries away from them
- Reduce platforms/rocm.py to the GCN architecture predicates that live
  code still imports, and delete the RocmPlatform class and plugin
- Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc,
  server_info and sleep routes plus the log_error_stack flag
- Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins;
  the live server process maps none of them
- Delete tests that only build architectures this fork no longer
  registers, and test directories left without any test file
- Restore the tests/v1/kv_connector mock helpers that scheduler tests
  import at module level

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 19:31:07 +07:00
co-authored by Hinata
parent 2066d3143f
commit 5f3c7bd8ab
162 changed files with 599 additions and 27490 deletions
@@ -1,737 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Portions adapted from ROCm/aiter (https://github.com/ROCm/aiter/pull/5750):
# SPDX-License-Identifier: MIT
# Copyright (C) 2026, Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``).
``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per
row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in
the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being
expanded to every row. Each weight scale byte is then read once per 32 output
rows, and small-M shapes can use the packed kernel below.
Two kernels, picked per shape from a table tuned on MI355X:
* a tiled kernel for larger M;
* a packed kernel for small M, which fills the MFMA's rows with K panels
instead of tokens and keeps only the matching-panel (block-diagonal)
products, so a handful of tokens still streams the weight at full width.
Either can split K. The partials are normally summed in the same launch by
the last program of each output tile to finish, in split order, so the result
does not depend on scheduling. Otherwise a second launch reduces them.
The packed kernel and the in-launch split-K reduction on one XCD
(``_split_tile``, ``_sum_splits``, ``_split_counters``, ``_k_partition``) are
adapted from the group32 GEMM in ROCm/aiter#5750.
"""
from typing import NamedTuple
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
BLOCK_ROWS = 32
# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is
# a multiple of the XCD count in every partition mode. _split_tile hardcodes
# the same stride.
_XCD_STRIDE = 8
# One arrival counter per output tile of an in-launch split-K GEMM.
_SPLIT_COUNTERS = 1 << 16
@triton.jit
def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr):
"""(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on
one XCD, so they meet in that XCD's L2."""
pid = tl.program_id(0)
tile = pid // (8 * SPLITS) * 8 + pid % 8
grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N
return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS
@triton.jit
def _sum_splits(
acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr
):
"""Publish this split's partial; the tile's last arrival sums all of them
in split order and re-arms the counter."""
BM: tl.constexpr = acc.shape[0]
BN: tl.constexpr = acc.shape[1]
local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :]
tl.store(slot_ptr + split * BM * BN + local, acc)
# The partial must be in L2 before the arrival is counted. The splits
# share an XCD, so no device-scope release (an L2 writeback) is needed.
tl.inline_asm_elementwise(
"s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1
)
tl.debug_barrier()
if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1:
total = tl.zeros((BM, BN), dtype=tl.float32)
for s in tl.static_range(SPLITS):
total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv")
tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask)
tl.store(count_ptr, 0)
@triton.jit(do_not_specialize=["M"])
def _block32_tiled_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
slot_ptr,
count_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
stride_om,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
K_PER_SPLIT: tl.constexpr,
N_FIRST: tl.constexpr,
EVEN_K: tl.constexpr,
FUSED_SPLITS: tl.constexpr,
):
if FUSED_SPLITS > 1:
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
if pid_m * BLOCK_M >= M:
return
else:
# N_FIRST launches output-column tiles fastest, so consecutive
# programs share an activation tile.
if N_FIRST:
pid_n, pid_m = tl.program_id(0), tl.program_id(1)
else:
pid_m, pid_n = tl.program_id(0), tl.program_id(1)
pid_k = tl.program_id(2)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
m_mask = offs_m < M
n_mask = offs_n < N
k0 = pid_k * K_PER_SPLIT
offs_k = k0 + tl.arange(0, BLOCK_K)
offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32)
x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :]
xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :]
w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :]
ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for kk in range(0, K_PER_SPLIT, BLOCK_K):
if EVEN_K:
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
else:
k_ok = (offs_k + kk) < K
s_ok = (offs_sk + kk // 32) < K // 32
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=0)
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=0)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
x_ptrs += BLOCK_K
w_ptrs += BLOCK_K
xs_ptrs += BLOCK_K // 32
ws_ptrs += BLOCK_K // 32
out_mask = m_mask[:, None] & n_mask[None, :]
if FUSED_SPLITS > 1:
_sum_splits(
acc,
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
out_mask,
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
count_ptr + tile,
pid_k,
FUSED_SPLITS,
)
else:
out_ptrs = (
out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :]
)
tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
@triton.jit(do_not_specialize=["M"])
def _block32_packed_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
slot_ptr,
count_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
stride_om,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
K_PACK: tl.constexpr,
K_PER_SPLIT: tl.constexpr,
EVEN_K: tl.constexpr,
FUSED_SPLITS: tl.constexpr,
):
if FUSED_SPLITS > 1:
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
if pid_m * BLOCK_M >= M:
return
k_start = pid_k * K_PER_SPLIT
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
else:
pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
if K_PER_SPLIT >= K:
k_start = 0
k_stop = K
else:
k_start = pid_k * K_PER_SPLIT
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
# Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise
# for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices.
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK
x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK
w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK
ks = tl.arange(0, BLOCK_K)
gs = tl.arange(0, BLOCK_K // 32)
row_ok = rows < M
col_ok = cols < N
acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32)
for base in range(k_start, k_stop, BLOCK_K * K_PACK):
xk = base + x_panel[:, None] * BLOCK_K + ks[None, :]
wk = base + w_panel[:, None] * BLOCK_K + ks[None, :]
xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
x_mask = row_ok[:, None]
w_mask = col_ok[:, None]
xs_mask = row_ok[:, None]
ws_mask = col_ok[:, None]
if not EVEN_K:
x_mask = x_mask & (xk < K)
w_mask = w_mask & (wk < K)
xs_mask = xs_mask & (xg < K // 32)
ws_mask = ws_mask & (wg < K // 32)
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=0)
ws = tl.load(
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=0
)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
# Keep only products of matching K panels: the block diagonal.
blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3)
pair = tl.arange(0, K_PACK)
diagonal = tl.where(
pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0
)
y = tl.sum(tl.sum(diagonal, 3), 2)
out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :]
out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N)
if FUSED_SPLITS > 1:
_sum_splits(
y,
out_ptrs,
out_mask,
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
count_ptr + tile,
pid_k,
FUSED_SPLITS,
)
else:
tl.store(
out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask
)
@triton.jit(do_not_specialize=["M"])
def _block32_splitk_reduce_kernel(
partial_ptr,
out_ptr,
M,
N: tl.constexpr,
stride_om,
SPLIT_K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for split in tl.static_range(SPLIT_K):
acc += tl.load(
partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :],
mask=mask,
other=0.0,
)
tl.store(
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
acc.to(out_ptr.dtype.element_ty),
mask=mask,
)
class _Config(NamedTuple):
packed: bool
block_m: int
block_n: int
block_k: int
# K panels per MFMA tile (packed) or K partitions reduced by a second
# launch (tiled).
k_split: int
n_first: bool
num_warps: int
num_stages: int
matrix_instr_nonkdim: int
waves_per_eu: int
# K partitions summed within the launch.
fused_splits: int = 1
def _tiled(
bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1
) -> _Config:
return _Config(
False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused
)
def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config:
return _Config(
True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused
)
# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at
# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A
# tier serves every M up to its bound; larger M reuse the last tier.
_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = {
# shared expert gate_up, TP4
(1152, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)),
(2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)),
(4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)),
(8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)),
(16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)),
(24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)),
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)),
(48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)),
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)),
(96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)),
(128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)),
(160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
(192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)),
(256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)),
(384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
(512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)),
(1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)),
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
],
# fused wq_a|wkv (replicated)
(1792, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
(16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)),
(32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)),
(48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)),
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)),
(96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)),
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)),
(384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)),
],
# shared expert gate_up, TP2
(2304, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
(16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)),
(24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)),
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)),
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)),
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)),
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)),
(256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
(4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)),
],
# indexer wq_b
(4096, 1280): [
(1, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
(2, _packed(4, 16, 256, 4, 4, 2, 16, 0)),
(4, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
(8, _packed(8, 16, 256, 4, 4, 3, 16, 1)),
(16, _packed(16, 16, 512, 2, 2, 3, 16, 1)),
(32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)),
(64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)),
(128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)),
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)),
(256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)),
],
# shared expert down, TP4
(5120, 576): [
(8, _packed(8, 32, 256, 2, 2, 2, 16, 1)),
(16, _packed(16, 32, 512, 1, 2, 2, 16, 0)),
(32, _packed(32, 32, 256, 1, 8, 3, 16, 1)),
(64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
(96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)),
(128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)),
(192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)),
(256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)),
(768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
],
# shared expert down, TP2
(5120, 1152): [
(1, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
(32, _packed(32, 32, 512, 1, 2, 3, 16, 1)),
(128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)),
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)),
],
# wo_b, TP4
(5120, 2048): [
(1, _packed(4, 32, 256, 4, 2, 2, 16, 0)),
(4, _packed(4, 32, 256, 4, 2, 2, 16, 1)),
(8, _packed(8, 32, 256, 4, 2, 3, 16, 0)),
(16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)),
(32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)),
(64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
(96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)),
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)),
],
# wo_b, TP2
(5120, 4096): [
(4, _packed(4, 32, 512, 4, 2, 2, 16, 0)),
(8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)),
(16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)),
(24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)),
(32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)),
(48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)),
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)),
(128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
(3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
],
# wq_b, TP4
(8192, 1280): [
(1, _packed(8, 32, 256, 2, 4, 3, 16, 1)),
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
(16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)),
(32, _packed(32, 32, 512, 1, 2, 3, 16, 0)),
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)),
(64, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
],
# wq_b, TP2
(16384, 1280): [
(2, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
(4, _packed(4, 32, 128, 4, 4, 2, 16, 0)),
(8, _packed(8, 32, 256, 2, 4, 2, 16, 0)),
(32, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
(64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)),
(128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
],
}
def _default_config(M: int, N: int, K: int) -> _Config:
"""Untuned shapes: the tiers most tuned shapes settle on."""
if M <= 8:
return _packed(8, 16, 256, 2, 2, 3, 16, 0)
if M <= 32:
return _packed(32, 32, 256, 1, 2, 3, 16, 0)
if M <= 256:
return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0)
return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)
def _config(M: int, N: int, K: int) -> _Config:
tiers = _TUNED.get((N, K))
if tiers is None:
return _default_config(M, N, K)
for max_m, cfg in tiers:
if max_m >= M:
return cfg
return tiers[-1][1]
_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {}
def _split_counters(device: torch.device) -> torch.Tensor | None:
"""Zeroed arrival counters, one set per stream so concurrent launches never
share a tile's counter. Each launch leaves the counters it used at zero.
None when a stream first needs them inside a CUDA graph capture: that
allocation would come from the graph's pool, where it can take the address
of an intermediate that every replay overwrites. vLLM warms up on its
capture streams first, so this only affects captures without a warmup.
"""
key = (device.index, torch.accelerator.current_stream(device).stream_id)
counters = _split_counters_by_stream.get(key)
if counters is None:
if torch.cuda.is_current_stream_capturing():
return None
counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device)
_split_counters_by_stream[key] = counters
return counters
def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]:
"""K per split, rounded up to whole steps, and the number of non-empty
splits."""
size = triton.cdiv(triton.cdiv(K, splits), step) * step
return size, triton.cdiv(K, size)
def _launch(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out: torch.Tensor,
cfg: _Config,
) -> None:
M, K = x.shape
N = weight.shape[0]
opts = dict(
num_warps=cfg.num_warps,
num_stages=cfg.num_stages,
matrix_instr_nonkdim=cfg.matrix_instr_nonkdim,
waves_per_eu=cfg.waves_per_eu,
)
grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n)
tiles = grid_m * grid_n
step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k
splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split
k_per_split, splits = _k_partition(K, splits, step)
counters = None
if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS:
counters = _split_counters(out.device)
if counters is not None:
fused = splits
target = out
slots = torch.empty(
tiles * fused * cfg.block_m * cfg.block_n,
dtype=torch.float32,
device=out.device,
)
grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,)
else:
# Without counters, split K through a second launch.
fused = 1
target = (
out
if splits == 1
else torch.empty((splits, M, N), dtype=torch.float32, device=x.device)
)
slots = counters = target # Unused by the kernels.
if cfg.packed or not cfg.n_first:
grid = (grid_m, grid_n, splits)
else:
grid = (grid_n, grid_m, splits)
args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K)
stride = out.stride(0) if splits == 1 or fused > 1 else N
if cfg.packed:
_block32_packed_kernel[grid](
*args,
stride,
BLOCK_M=cfg.block_m,
BLOCK_N=cfg.block_n,
BLOCK_K=cfg.block_k,
K_PACK=cfg.k_split,
K_PER_SPLIT=k_per_split,
EVEN_K=K % step == 0,
FUSED_SPLITS=fused,
**opts,
)
else:
_block32_tiled_kernel[grid](
*args,
stride,
BLOCK_M=cfg.block_m,
BLOCK_N=cfg.block_n,
BLOCK_K=cfg.block_k,
K_PER_SPLIT=k_per_split,
N_FIRST=cfg.n_first,
EVEN_K=K % k_per_split == 0,
FUSED_SPLITS=fused,
**opts,
)
if fused == 1 and splits > 1:
_block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))](
target,
out,
M,
N,
out.stride(0),
SPLIT_K=splits,
BLOCK_M=32,
BLOCK_N=64,
)
def _rocm_mxfp8_block32_gemm_impl(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
M, K = x.shape
out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device)
if M > 0:
_launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K))
return out
def _rocm_mxfp8_block32_gemm_fake(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype)
# An opaque op: the kernel and tile choice branch on the token count, which a
# compiled caller must not freeze at its trace-time value.
direct_register_custom_op(
op_name="rocm_mxfp8_block32_gemm",
op_func=_rocm_mxfp8_block32_gemm_impl,
fake_impl=_rocm_mxfp8_block32_gemm_fake,
)
def rocm_mxfp8_block32_gemm(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight.
Args:
x: [M, K] e4m3 activation, contiguous.
x_scale: [M, K / 32] E8M0 (uint8) activation scales.
weight: [N, K] e4m3 weight, contiguous.
weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales.
out_dtype: Output dtype.
Returns:
The [M, N] product.
"""
return torch.ops.vllm.rocm_mxfp8_block32_gemm(
x, x_scale, weight, weight_scale, out_dtype
)
@@ -1,295 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Native MXFP8 linear GEMM for AMD CDNA4 (gfx950) via Triton ``tl.dot_scaled``.
Consumes the FP8 E4M3 weights + E8M0 block scales directly (no dequant-to-BF16);
activations are MXFP8-quantized per token. Uses the CDNA4 hardware microscaling
matrix cores. ``dot_scaled`` tiles K by 128; a weight whose K is not a multiple
of that is dequantized to BF16 once at load and served by a plain linear
instead, since ``can_implement`` does not filter on K.
"""
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm
@triton.jit
def _mxfp8_linear_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
M,
N,
K,
stride_xm,
stride_xk,
stride_xsm,
stride_xsk,
stride_wn,
stride_wk,
stride_wsn,
stride_wsk,
stride_om,
stride_on,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
):
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
offs_k = tl.arange(0, BLOCK_K)
offs_sk = tl.arange(0, BLOCK_K // 32)
m_mask = offs_m < M
n_mask = offs_n < N
x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk
xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(0, tl.cdiv(K, BLOCK_K)):
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0)
acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3")
x_ptrs += BLOCK_K * stride_xk
w_ptrs += BLOCK_K * stride_wk
xs_ptrs += (BLOCK_K // 32) * stride_xsk
ws_ptrs += (BLOCK_K // 32) * stride_wsk
o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on
tl.store(
o_ptrs, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None] & n_mask[None, :]
)
def _mxfp8_dot_scaled_linear(
x: torch.Tensor, # [M, K] bf16/fp16
w: torch.Tensor, # [N, K] fp8 e4m3
w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0)
) -> torch.Tensor:
M, K = x.shape
N = w.shape[0]
x_q, x_scale = mxfp8_e4m3_quantize(x)
out = torch.empty((M, N), dtype=x.dtype, device=x.device)
BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages = _select_cfg(M, N, K)
grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N))
_mxfp8_linear_kernel[grid](
x_q,
x_scale,
w,
w_scale,
out,
M,
N,
K,
x_q.stride(0),
x_q.stride(1),
x_scale.stride(0),
x_scale.stride(1),
w.stride(0),
w.stride(1),
w_scale.stride(0),
w_scale.stride(1),
out.stride(0),
out.stride(1),
BLOCK_M=BLOCK_M,
BLOCK_N=BLOCK_N,
BLOCK_K=BLOCK_K,
num_warps=num_warps,
num_stages=num_stages,
)
return out
# gfx950's async-copy LDS (default in the triton 3.8 this branch pins) leaves no
# room for num_stages=3 at BLOCK_K=256; use 2 there.
_BK256_STAGES = 3
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
if on_gfx950():
_BK256_STAGES = 2
def _select_cfg(M, N, K):
"""(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) — graph-tuned on gfx950.
The M-bucketed, shape-adaptive tile selection here is the speedup over the
upstream 2-bucket launcher. Tiles are pipelined (num_stages>=2, larger BLOCK_K)
and occupancy- and shape-aware: keyed on the LOCAL (M, N, K), so it adapts to the
TP-sharded shapes (e.g. MiniMax-M3 TP=4 vs TP=8, where local N and K differ) —
large-K prefill uses BLOCK_K=256; short-K (K=768) widens N. BLOCK_K must divide K
(the K-loop is unmasked), so every BLOCK_K below is guarded to be K-divisible
(served K: 384/768/1024/2048/6144).
"""
if M <= 64:
# decode (M in {1,32,64}): tiny-M GEMV is weight-BW + GPU-OCCUPANCY bound. The
# lever is NARROW BLOCK_N=16 (maximize N-tiles so more CUs stream the weight in
# parallel) + LARGE BLOCK_K (fewer K-iters, bigger coalesced weight loads).
# Tuned by CUDA-graph replay latency. Optimal at both TP=4 and TP=8.
if K % 1024 == 0: # K=2048, 6144 -> graph-best 16x16x1024 (all M)
return 16, 16, 1024, 2, 2
if K % 512 == 0:
return 16, 16, 512, 2, 3
if K % 256 == 0: # K=768 (shared_down) -> graph-best 16x32x256
return 16, 32, 256, 4, 3
return 16, 32, 128, 4, 3
# mid-M (65..256) on SMALL local-N: still occupancy-bound (a 64x64 tile makes too
# few N-tiles), so the narrow-BLOCK_N decode-style tile fills the CUs better.
# N<=1536 covers the real fused-qkv local N at TP=8: q heads shard but the GQA KV
# (4) + sparse-indexer (4) heads are < TP=8, so vLLM replicates them to 1/rank ->
# N = 1024 + 4*128 = 1536 (not 2560/2=1280). For the wider 1280<N<=1536 band the
# narrow tile only wins up to M=128 (at M=256 the 64x64 tile is better), so cap it
# there; N<=1280 keeps the narrow tile through M=256. TP=4 qkv N=2560 is unchanged.
if (M <= 256 and N <= 1280) or (M <= 128 and N <= 1536):
if K % 1024 == 0:
return 16, 16, 1024, 2, 2
if K % 512 == 0:
return 16, 16, 512, 2, 3
if K % 256 == 0:
return 16, 16, 256, 2, 3
return 16, 16, 128, 2, 3
# right-sized launch grid (host-side), used to gate the tall 256-BLOCK_M tile.
occ = triton.cdiv(M, 256) * triton.cdiv(N, 128)
if K <= 1024: # short-K (shared_down K=384/768; TP=8 o_proj K=1024)
if M <= 256:
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
# large prefill. (The former 256x128x256 tall tile was faster only on triton
# 3.6; on triton 3.7 its large BLOCK_M register/LDS footprint spills or hits
# "out of resources", so use 128x128x256 -- within the known-good footprint.)
if M >= 4096 and K >= 1024 and K % 256 == 0 and occ >= 256:
return 128, 128, 256, 8, _BK256_STAGES
return 128, 128, 128, 8, 3
# large-K (K >= 2048). BLOCK_K is K-divisibility-guarded (the K-loop is unmasked):
# served large-K is 2048/6144 (%256==0), but fall back to 128 (always divides, since
# the entry requires K%128==0) for any other K to stay correct.
if M <= 256: # conc~128 decode + small prefill chunk: occupancy tile
if K % 512 == 0:
return 64, 64, 512, 8, 2
return (64, 64, 256, 8, 2) if K % 256 == 0 else (64, 64, 128, 8, 2)
if M <= 1024: # medium prefill chunk: BN=64 keeps small-N occupied
return (128, 64, 256, 8, 3) if K % 256 == 0 else (128, 64, 128, 8, 3)
# large prefill (M > 1024). The previously graph-tuned tall 256x128x256 and deep
# 128x128x512 tiles won only on triton 3.6; on triton 3.7 their larger BLOCK_M /
# BLOCK_K register+LDS footprint spills (or hits "out of resources" on stricter
# ROCm/triton builds). The 128x128x256 tile is equal-or-faster on triton 3.7 (the
# M=4096,N=2048,K=6144 shape: ~104us vs the 256x128x256 tile's ~184us), within ~5%
# on 3.6, and inside the footprint of the tiles used elsewhere in this selector.
# Covers the qkv-class local N=1536 (TP=8 qkv / TP=4 shared_gate_up) and the deep-K
# / very-large-M shapes.
if K % 256 == 0 and (1280 < N <= 1536 or (occ >= 128 and (K >= 4096 or M >= 4096))):
return 128, 128, 256, 8, _BK256_STAGES
# small local-N (e.g. TP=8 shared_gate_up N=768): a 64-wide BLOCK_N doubles the
# N-tile count -> better CU fill than 128x128 at this mid-large M (~1.4x there).
if N <= 1024 and K % 256 == 0:
return 128, 64, 256, 8, 3
return (128, 128, 256, 8, 2) if K % 256 == 0 else (128, 256, 128, 8, 3)
_DOT_SCALED_K_ALIGN = 128
def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None:
"""The [N / 32, K / 32] block scales, if every 32 rows share their scales."""
N = weight_scale.shape[0]
if N % BLOCK_ROWS != 0:
return None
blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1)
if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)):
return None
return blocks[:, 0].contiguous()
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
supports_pre_processed_weights = True
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_rocm():
return False, "not ROCm"
# supports_mx() == gfx95x (CDNA4 native microscaling hardware). On other
# archs dot_scaled would upcast to BF16, so the kernel selector falls
# through to the BF16 emulation (hipBLASLt) path instead.
if not current_platform.supports_mx():
return False, "native MX requires CDNA4 (gfx95x)"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight = layer.weight.data # [N, K] fp8
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
# rocm_mxfp8_block32_gemm's split-K varies with M.
block_scale = (
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
)
if block_scale is not None:
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
# back from the loader expanded per row; keep the blocks and use
# the block-scaled GEMM, which also handles any K % 32 == 0.
weight_scale = block_scale
elif K % _DOT_SCALED_K_ALIGN != 0:
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if layer.weight_scale.dtype != MXFP8_SCALE_DTYPE:
raise ValueError(
f"Expected {MXFP8_SCALE_DTYPE} weight_scale, got "
f"{layer.weight_scale.dtype}."
)
out_shape = (*x.shape[:-1], layer.weight.shape[0])
x2d = x.reshape(-1, x.shape[-1])
if layer.weight_scale.shape[0] != layer.weight.shape[0]:
# One scale row per 32 weight rows (see process_weights_after_loading).
x_q, x_scale = mxfp8_e4m3_quantize(x2d)
out = rocm_mxfp8_block32_gemm(
x_q, x_scale, layer.weight, layer.weight_scale, x.dtype
)
elif layer.weight.element_size() >= 2:
out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype))
elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0:
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
else:
# process_weights_after_loading dequantizes these weights once, so
# this runs only if it did not; dot_scaled would be invalid here.
w_bf16 = dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale)
out = torch.nn.functional.linear(x2d, w_bf16).to(x.dtype)
out = out.reshape(out_shape)
if bias is not None:
out = out + bias
return out