[ROCm][Perf] MXFP8 GEMM on native 32x32 block scales for gfx950 (#58510)

Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Fangzhou Ai
2026-09-24 21:26:30 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 055f150df5
commit 3059155b47
5 changed files with 899 additions and 7 deletions
@@ -1204,8 +1204,14 @@ def test_sparse_attn_decode_mxfp8_output() -> None:
(1100, 8),
],
)
@pytest.mark.parametrize("block_scales", [False, True])
@torch.inference_mode()
def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
def test_rocm_mxfp8_wo_a_bmm(
num_tokens: int, n_groups: int, block_scales: bool
) -> None:
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
_as_block32_scale,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
_mxfp8_e4m3_quantize_torch,
)
@@ -1220,6 +1226,11 @@ def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
w, w_scale = _mxfp8_e4m3_quantize_torch(
torch.randn(n_groups * o_lora_rank, group_dim, device=device)
)
if block_scales:
# A 32x32 checkpoint loads as per-row scales that repeat every 32 rows;
# RocmDotScaledMxfp8LinearKernel compacts them before wo_a sees them.
w_scale = _as_block32_scale(w_scale[::32].repeat_interleave(32, dim=0))
assert w_scale is not None
wo_a = SimpleNamespace(weight=w, weight_scale=w_scale)
out = rocm_mxfp8_wo_a_bmm(a, a_scale, wo_a, n_groups, o_lora_rank)
@@ -1227,7 +1238,9 @@ def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
expected = torch.einsum(
"tgd,grd->tgr",
_mxfp8_dequant(a, a_scale).view(num_tokens, n_groups, group_dim),
_mxfp8_dequant(w, w_scale).view(n_groups, o_lora_rank, group_dim),
_mxfp8_dequant(
w, w_scale.repeat_interleave(w.shape[0] // w_scale.shape[0], 0)
).view(n_groups, o_lora_rank, group_dim),
)
torch.testing.assert_close(out.float(), expected.flatten(1), atol=5e-2, rtol=1e-2)
@@ -117,6 +117,147 @@ def test_rocm_mxfp8_linear_handles_extreme_magnitudes():
assert (out[0] == 0).all(), "an all-zero activation row must give a zero row"
# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The
# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has
# K % 128 != 0, which only the block-scaled GEMM runs natively.
V41_BLOCK32_SHAPES = [
("fused_wqa_wkv", 1280 + 512, 5120),
("wq_b_tp2", 64 * 512 // 2, 1280),
("indexer_wq_b", 32 * 128, 1280),
("wo_b_tp2", 5120, 8192 // 2),
("shared_gate_up_tp2", 2 * 2304 // 2, 5120),
("shared_down_tp2", 5120, 2304 // 2),
("shared_down_tp4", 5120, 2304 // 4),
]
def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module:
"""An MXFP8 layer with 32x32 block scales, expanded per row as loaded."""
blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32)
amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny)
bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254)
weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
weight.to(torch.float8_e4m3fn), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(
bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False
)
return layer
@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v))
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024])
@torch.inference_mode()
def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens):
"""32x32 block scales stay compact and the block-scaled GEMM is exact.
The token counts cover the packed, tiled and split-K configurations.
"""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
torch.manual_seed(num_tokens)
device = "cuda"
layer = _make_block32_layer(n, k, device)
row_scale = layer.weight_scale.data.clone()
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None))
kernel.process_weights_after_loading(layer)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.shape == (n // 32, k // 32)
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
out = kernel.apply_weights(layer, x)
x_q, x_scale = mxfp8_e4m3_quantize(x)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T
)
assert out.shape == (num_tokens, n)
rel = (out.float() - expected).norm() / expected.norm()
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
@torch.inference_mode()
def test_rocm_mxfp8_linear_keeps_per_row_scales():
"""Scales that differ within a 32-row block are not compacted."""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
layer = _make_block32_layer(64, 256, "cuda")
layer.weight_scale.data[1, 0] += 1
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None))
kernel.process_weights_after_loading(layer)
assert layer.weight_scale.shape == (64, 256 // 32)
def _split_k_case(packed: bool):
"""A 3-way in-launch split-K on shared_down TP4, with K and M tails."""
from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g
n, k, m = 5120, 576, 19
layer = _make_block32_layer(n, k, "cuda")
x_q, x_scale = mxfp8_e4m3_quantize(
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
)
if packed:
cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3)
else:
cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3)
args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous())
return g, cfg, args, expected
def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float:
return ((out.float() - expected).norm() / expected.norm()).item()
@pytest.mark.parametrize("packed", [True, False])
@torch.inference_mode()
def test_rocm_mxfp8_block32_split_k_in_launch(packed):
"""Splits summed by the tile's last program are exact and repeatable."""
torch.manual_seed(0)
g, cfg, args, expected = _split_k_case(packed)
out = torch.empty_like(expected, dtype=torch.bfloat16)
g._launch(*args, out, cfg)
assert _rel_err(out, expected) < 5e-3
first = out.clone()
for _ in range(20):
g._launch(*args, out, cfg)
assert torch.equal(out, first)
@pytest.mark.parametrize("packed", [True, False])
@torch.inference_mode()
def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed):
"""A stream whose first split-K launch is being captured has no counters
yet; it must fall back to an unfused split instead of failing."""
torch.manual_seed(0)
g, cfg, args, expected = _split_k_case(packed)
out = torch.zeros_like(expected, dtype=torch.bfloat16)
stream = torch.cuda.Stream()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
g._launch(*args, out, cfg)
graph.replay()
torch.accelerator.synchronize()
assert _rel_err(out, expected) < 5e-3
@pytest.mark.parametrize("num_rows", [1, 64, 65])
@torch.inference_mode()
def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows):
@@ -0,0 +1,711 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``).
``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per
row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in
the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being
expanded to every row. Each weight scale byte is then read once per 32 output
rows, and small-M shapes can use the packed kernel below.
Two kernels, picked per shape from a table tuned on MI355X:
* a tiled kernel for larger M;
* a packed kernel for small M, which fills the MFMA's rows with K panels
instead of tokens and keeps only the matching-panel (block-diagonal)
products, so a handful of tokens still streams the weight at full width.
Either can split K. The partials are normally summed in the same launch by
the last program of each output tile to finish, in split order, so the result
does not depend on scheduling. Otherwise a second launch reduces them.
"""
from typing import NamedTuple
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
BLOCK_ROWS = 32
# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is
# a multiple of the XCD count in every partition mode. _split_tile hardcodes
# the same stride.
_XCD_STRIDE = 8
# One arrival counter per output tile of an in-launch split-K GEMM.
_SPLIT_COUNTERS = 1 << 16
@triton.jit
def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr):
"""(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on
one XCD, so they meet in that XCD's L2."""
pid = tl.program_id(0)
tile = pid // (8 * SPLITS) * 8 + pid % 8
grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N
return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS
@triton.jit
def _sum_splits(
acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr
):
"""Publish this split's partial; the tile's last arrival sums all of them
in split order and re-arms the counter."""
BM: tl.constexpr = acc.shape[0]
BN: tl.constexpr = acc.shape[1]
local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :]
tl.store(slot_ptr + split * BM * BN + local, acc)
# The partial must be in L2 before the arrival is counted. The splits
# share an XCD, so no device-scope release (an L2 writeback) is needed.
tl.inline_asm_elementwise(
"s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1
)
tl.debug_barrier()
if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1:
total = tl.zeros((BM, BN), dtype=tl.float32)
for s in tl.static_range(SPLITS):
total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv")
tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask)
tl.store(count_ptr, 0)
@triton.jit(do_not_specialize=["M"])
def _block32_tiled_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
slot_ptr,
count_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
stride_om,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
K_PER_SPLIT: tl.constexpr,
N_FIRST: tl.constexpr,
EVEN_K: tl.constexpr,
FUSED_SPLITS: tl.constexpr,
):
if FUSED_SPLITS > 1:
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
if pid_m * BLOCK_M >= M:
return
else:
# N_FIRST launches output-column tiles fastest, so consecutive
# programs share an activation tile.
if N_FIRST:
pid_n, pid_m = tl.program_id(0), tl.program_id(1)
else:
pid_m, pid_n = tl.program_id(0), tl.program_id(1)
pid_k = tl.program_id(2)
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
m_mask = offs_m < M
n_mask = offs_n < N
k0 = pid_k * K_PER_SPLIT
offs_k = k0 + tl.arange(0, BLOCK_K)
offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32)
x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :]
xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :]
w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :]
ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for kk in range(0, K_PER_SPLIT, BLOCK_K):
if EVEN_K:
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=127)
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=127)
else:
k_ok = (offs_k + kk) < K
s_ok = (offs_sk + kk // 32) < K // 32
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=127)
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=127)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
x_ptrs += BLOCK_K
w_ptrs += BLOCK_K
xs_ptrs += BLOCK_K // 32
ws_ptrs += BLOCK_K // 32
out_mask = m_mask[:, None] & n_mask[None, :]
if FUSED_SPLITS > 1:
_sum_splits(
acc,
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
out_mask,
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
count_ptr + tile,
pid_k,
FUSED_SPLITS,
)
else:
out_ptrs = (
out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :]
)
tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
@triton.jit(do_not_specialize=["M"])
def _block32_packed_kernel(
x_ptr,
xs_ptr,
w_ptr,
ws_ptr,
out_ptr,
slot_ptr,
count_ptr,
M,
N: tl.constexpr,
K: tl.constexpr,
stride_om,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
K_PACK: tl.constexpr,
K_PER_SPLIT: tl.constexpr,
EVEN_K: tl.constexpr,
FUSED_SPLITS: tl.constexpr,
):
if FUSED_SPLITS > 1:
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
if pid_m * BLOCK_M >= M:
return
k_start = pid_k * K_PER_SPLIT
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
else:
pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
if K_PER_SPLIT >= K:
k_start = 0
k_stop = K
else:
k_start = pid_k * K_PER_SPLIT
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
# Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise
# for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices.
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK
x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK
w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK
ks = tl.arange(0, BLOCK_K)
gs = tl.arange(0, BLOCK_K // 32)
row_ok = rows < M
col_ok = cols < N
acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32)
for base in range(k_start, k_stop, BLOCK_K * K_PACK):
xk = base + x_panel[:, None] * BLOCK_K + ks[None, :]
wk = base + w_panel[:, None] * BLOCK_K + ks[None, :]
xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
x_mask = row_ok[:, None]
w_mask = col_ok[:, None]
xs_mask = row_ok[:, None]
ws_mask = col_ok[:, None]
if not EVEN_K:
x_mask = x_mask & (xk < K)
w_mask = w_mask & (wk < K)
xs_mask = xs_mask & (xg < K // 32)
ws_mask = ws_mask & (wg < K // 32)
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=127)
ws = tl.load(
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=127
)
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
# Keep only products of matching K panels: the block diagonal.
blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3)
pair = tl.arange(0, K_PACK)
diagonal = tl.where(
pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0
)
y = tl.sum(tl.sum(diagonal, 3), 2)
out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :]
out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N)
if FUSED_SPLITS > 1:
_sum_splits(
y,
out_ptrs,
out_mask,
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
count_ptr + tile,
pid_k,
FUSED_SPLITS,
)
else:
tl.store(
out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask
)
@triton.jit(do_not_specialize=["M"])
def _block32_splitk_reduce_kernel(
partial_ptr,
out_ptr,
M,
N: tl.constexpr,
stride_om,
SPLIT_K: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
):
offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for split in tl.static_range(SPLIT_K):
acc += tl.load(
partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :],
mask=mask,
other=0.0,
)
tl.store(
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
acc.to(out_ptr.dtype.element_ty),
mask=mask,
)
class _Config(NamedTuple):
packed: bool
block_m: int
block_n: int
block_k: int
# K panels per MFMA tile (packed) or K partitions reduced by a second
# launch (tiled).
k_split: int
n_first: bool
num_warps: int
num_stages: int
matrix_instr_nonkdim: int
waves_per_eu: int
# K partitions summed within the launch.
fused_splits: int = 1
def _tiled(
bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1
) -> _Config:
return _Config(
False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused
)
def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config:
return _Config(
True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused
)
# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at
# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A
# tier serves every M up to its bound; larger M reuse the last tier.
_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = {
# shared expert gate_up, TP4
(1152, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)),
(2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)),
(4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)),
(8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)),
(16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)),
(24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)),
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)),
(48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)),
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)),
(96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)),
(128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)),
(160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
(192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)),
(256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)),
(384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
(512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)),
(1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)),
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
],
# fused wq_a|wkv (replicated)
(1792, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
(16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)),
(32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)),
(48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)),
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)),
(96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)),
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)),
(384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)),
],
# shared expert gate_up, TP2
(2304, 5120): [
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
(16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)),
(24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)),
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)),
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)),
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)),
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
(160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)),
(256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
(4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)),
],
# indexer wq_b
(4096, 1280): [
(1, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
(2, _packed(4, 16, 256, 4, 4, 2, 16, 0)),
(4, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
(8, _packed(8, 16, 256, 4, 4, 3, 16, 1)),
(16, _packed(16, 16, 512, 2, 2, 3, 16, 1)),
(32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)),
(64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)),
(128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)),
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)),
(256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)),
],
# shared expert down, TP4
(5120, 576): [
(8, _packed(8, 32, 256, 2, 2, 2, 16, 1)),
(16, _packed(16, 32, 512, 1, 2, 2, 16, 0)),
(32, _packed(32, 32, 256, 1, 8, 3, 16, 1)),
(64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
(96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)),
(128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)),
(192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)),
(256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)),
(768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)),
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
],
# shared expert down, TP2
(5120, 1152): [
(1, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
(32, _packed(32, 32, 512, 1, 2, 3, 16, 1)),
(128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)),
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)),
],
# wo_b, TP4
(5120, 2048): [
(1, _packed(4, 32, 256, 4, 2, 2, 16, 0)),
(4, _packed(4, 32, 256, 4, 2, 2, 16, 1)),
(8, _packed(8, 32, 256, 4, 2, 3, 16, 0)),
(16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)),
(32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)),
(64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
(96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)),
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)),
],
# wo_b, TP2
(5120, 4096): [
(4, _packed(4, 32, 512, 4, 2, 2, 16, 0)),
(8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)),
(16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)),
(24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)),
(32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)),
(48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)),
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)),
(128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
(256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)),
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
(3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
],
# wq_b, TP4
(8192, 1280): [
(1, _packed(8, 32, 256, 2, 4, 3, 16, 1)),
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
(16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)),
(32, _packed(32, 32, 512, 1, 2, 3, 16, 0)),
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)),
(64, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
(192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
(256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
(384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
(1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
],
# wq_b, TP2
(16384, 1280): [
(2, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
(4, _packed(4, 32, 128, 4, 4, 2, 16, 0)),
(8, _packed(8, 32, 256, 2, 4, 2, 16, 0)),
(32, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
(64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)),
(128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
(256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
(384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
(768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
],
}
def _default_config(M: int, N: int, K: int) -> _Config:
"""Untuned shapes: the tiers most tuned shapes settle on."""
if M <= 8:
return _packed(8, 16, 256, 2, 2, 3, 16, 0)
if M <= 32:
return _packed(32, 32, 256, 1, 2, 3, 16, 0)
if M <= 256:
return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0)
return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)
def _config(M: int, N: int, K: int) -> _Config:
tiers = _TUNED.get((N, K))
if tiers is None:
return _default_config(M, N, K)
for max_m, cfg in tiers:
if max_m >= M:
return cfg
return tiers[-1][1]
_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {}
def _split_counters(device: torch.device) -> torch.Tensor | None:
"""Zeroed arrival counters, one set per stream so concurrent launches never
share a tile's counter. Each launch leaves the counters it used at zero.
None when a stream first needs them inside a CUDA graph capture: that
allocation would come from the graph's pool, where it can take the address
of an intermediate that every replay overwrites. vLLM warms up on its
capture streams first, so this only affects captures without a warmup.
"""
key = (device.index, torch.accelerator.current_stream(device).stream_id)
counters = _split_counters_by_stream.get(key)
if counters is None:
if torch.cuda.is_current_stream_capturing():
return None
counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device)
_split_counters_by_stream[key] = counters
return counters
def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]:
"""K per split, rounded up to whole steps, and the number of non-empty
splits."""
size = triton.cdiv(triton.cdiv(K, splits), step) * step
return size, triton.cdiv(K, size)
def _launch(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out: torch.Tensor,
cfg: _Config,
) -> None:
M, K = x.shape
N = weight.shape[0]
opts = dict(
num_warps=cfg.num_warps,
num_stages=cfg.num_stages,
matrix_instr_nonkdim=cfg.matrix_instr_nonkdim,
waves_per_eu=cfg.waves_per_eu,
)
grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n)
tiles = grid_m * grid_n
step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k
splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split
k_per_split, splits = _k_partition(K, splits, step)
counters = None
if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS:
counters = _split_counters(out.device)
if counters is not None:
fused = splits
target = out
slots = torch.empty(
tiles * fused * cfg.block_m * cfg.block_n,
dtype=torch.float32,
device=out.device,
)
grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,)
else:
# Without counters, split K through a second launch.
fused = 1
target = (
out
if splits == 1
else torch.empty((splits, M, N), dtype=torch.float32, device=x.device)
)
slots = counters = target # Unused by the kernels.
if cfg.packed or not cfg.n_first:
grid = (grid_m, grid_n, splits)
else:
grid = (grid_n, grid_m, splits)
args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K)
stride = out.stride(0) if splits == 1 or fused > 1 else N
if cfg.packed:
_block32_packed_kernel[grid](
*args,
stride,
BLOCK_M=cfg.block_m,
BLOCK_N=cfg.block_n,
BLOCK_K=cfg.block_k,
K_PACK=cfg.k_split,
K_PER_SPLIT=k_per_split,
EVEN_K=K % step == 0,
FUSED_SPLITS=fused,
**opts,
)
else:
_block32_tiled_kernel[grid](
*args,
stride,
BLOCK_M=cfg.block_m,
BLOCK_N=cfg.block_n,
BLOCK_K=cfg.block_k,
K_PER_SPLIT=k_per_split,
N_FIRST=cfg.n_first,
EVEN_K=K % k_per_split == 0,
FUSED_SPLITS=fused,
**opts,
)
if fused == 1 and splits > 1:
_block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))](
target,
out,
M,
N,
out.stride(0),
SPLIT_K=splits,
BLOCK_M=32,
BLOCK_N=64,
)
def _rocm_mxfp8_block32_gemm_impl(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
M, K = x.shape
out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device)
if M > 0:
_launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K))
return out
def _rocm_mxfp8_block32_gemm_fake(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype)
# An opaque op: the kernel and tile choice branch on the token count, which a
# compiled caller must not freeze at its trace-time value.
direct_register_custom_op(
op_name="rocm_mxfp8_block32_gemm",
op_func=_rocm_mxfp8_block32_gemm_impl,
fake_impl=_rocm_mxfp8_block32_gemm_fake,
)
def rocm_mxfp8_block32_gemm(
x: torch.Tensor,
x_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight.
Args:
x: [M, K] e4m3 activation, contiguous.
x_scale: [M, K / 32] E8M0 (uint8) activation scales.
weight: [N, K] e4m3 weight, contiguous.
weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales.
out_dtype: Output dtype.
Returns:
The [M, N] product.
"""
return torch.ops.vllm.rocm_mxfp8_block32_gemm(
x, x_scale, weight, weight_scale, out_dtype
)
@@ -22,6 +22,7 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm
@triton.jit
@@ -209,6 +210,17 @@ def _select_cfg(M, N, K):
_DOT_SCALED_K_ALIGN = 128
def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None:
"""The [N / 32, K / 32] block scales, if every 32 rows share their scales."""
N = weight_scale.shape[0]
if N % BLOCK_ROWS != 0:
return None
blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1)
if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)):
return None
return blocks[:, 0].contiguous()
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
@@ -236,7 +248,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
N, K = weight.shape
scale_k = K // MXFP8_BLOCK_SIZE
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
if K % _DOT_SCALED_K_ALIGN != 0:
block_scale = _as_block32_scale(weight_scale)
if block_scale is not None:
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
# back from the loader expanded per row; keep the blocks and use
# the block-scaled GEMM, which also handles any K % 32 == 0.
weight_scale = block_scale
elif K % _DOT_SCALED_K_ALIGN != 0:
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
@@ -254,7 +272,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
)
out_shape = (*x.shape[:-1], layer.weight.shape[0])
x2d = x.reshape(-1, x.shape[-1])
if layer.weight.element_size() >= 2:
if layer.weight_scale.shape[0] != layer.weight.shape[0]:
# One scale row per 32 weight rows (see process_weights_after_loading).
x_q, x_scale = mxfp8_e4m3_quantize(x2d)
out = rocm_mxfp8_block32_gemm(
x_q, x_scale, layer.weight, layer.weight_scale, x.dtype
)
elif layer.weight.element_size() >= 2:
out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype))
elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0:
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
@@ -1811,7 +1811,7 @@ def _mxfp8_wo_a_bmm_kernel(
a_ptr, # [T, G * K] e4m3
as_ptr, # [T, G * K // 32] E8M0
w_ptr, # [G * R, K] e4m3
ws_ptr, # [G * R, K // 32] E8M0
ws_ptr, # [G * R // SCALE_ROWS, K // 32] E8M0
out_ptr, # [T, G * R]
num_tokens,
stride_at,
@@ -1821,6 +1821,7 @@ def _mxfp8_wo_a_bmm_kernel(
stride_out,
R: tl.constexpr,
K: tl.constexpr,
SCALE_ROWS: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr,
@@ -1841,7 +1842,7 @@ def _mxfp8_wo_a_bmm_kernel(
as_ptr + offs_m[:, None] * stride_ast + (group * (K // 32) + offs_sk)[None, :]
)
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :]
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :]
ws_ptrs = ws_ptr + (offs_n // SCALE_ROWS)[:, None] * stride_wsn + offs_sk[None, :]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(K // BLOCK_K):
@@ -1926,6 +1927,7 @@ def _rocm_mxfp8_wo_a_bmm_impl(
out.stride(0),
R=o_lora_rank,
K=group_dim,
SCALE_ROWS=weight.shape[0] // weight_scale.shape[0],
BLOCK_M=block_m,
BLOCK_N=block_n,
BLOCK_K=block_k,
@@ -1967,7 +1969,8 @@ def rocm_mxfp8_wo_a_bmm(
``a`` is the [T, G * K] e4m3 attention output and ``a_scale`` its
[T, G * K // 32] E8M0 scales, as the sparse decode reduce writes them.
The weight is the checkpoint's MXFP8 ``wo_a`` as loaded, [G * R, K] with
[G * R, K // 32] scales, so there is no dequantized copy to keep.
either [G * R // 32, K // 32] block scales or [G * R, K // 32] per-row
scales, so there is no dequantized copy to keep.
Returns [T, G * R].
"""
return torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(