From 3059155b47cee5269eaf446e24c9d0f8f908c7c9 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:26:30 -0400 Subject: [PATCH] [ROCm][Perf] MXFP8 GEMM on native 32x32 block scales for gfx950 (#58510) Signed-off-by: fai Co-authored-by: Claude Opus 5.5 --- .../attention/test_rocm_triton_attn_dsv4.py | 17 +- .../quantization/test_rocm_mxfp8_linear.py | 141 ++++ .../kernels/linear/mxfp8/rocm_block32_gemm.py | 711 ++++++++++++++++++ .../kernels/linear/mxfp8/rocm_native.py | 28 +- .../v1/attention/ops/rocm_aiter_mla_sparse.py | 9 +- 5 files changed, 899 insertions(+), 7 deletions(-) create mode 100644 vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index 6c10f169be..87e4884b7e 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -1204,8 +1204,14 @@ def test_sparse_attn_decode_mxfp8_output() -> None: (1100, 8), ], ) +@pytest.mark.parametrize("block_scales", [False, True]) @torch.inference_mode() -def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None: +def test_rocm_mxfp8_wo_a_bmm( + num_tokens: int, n_groups: int, block_scales: bool +) -> None: + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + _as_block32_scale, + ) from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( _mxfp8_e4m3_quantize_torch, ) @@ -1220,6 +1226,11 @@ def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None: w, w_scale = _mxfp8_e4m3_quantize_torch( torch.randn(n_groups * o_lora_rank, group_dim, device=device) ) + if block_scales: + # A 32x32 checkpoint loads as per-row scales that repeat every 32 rows; + # RocmDotScaledMxfp8LinearKernel compacts them before wo_a sees them. + w_scale = _as_block32_scale(w_scale[::32].repeat_interleave(32, dim=0)) + assert w_scale is not None wo_a = SimpleNamespace(weight=w, weight_scale=w_scale) out = rocm_mxfp8_wo_a_bmm(a, a_scale, wo_a, n_groups, o_lora_rank) @@ -1227,7 +1238,9 @@ def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None: expected = torch.einsum( "tgd,grd->tgr", _mxfp8_dequant(a, a_scale).view(num_tokens, n_groups, group_dim), - _mxfp8_dequant(w, w_scale).view(n_groups, o_lora_rank, group_dim), + _mxfp8_dequant( + w, w_scale.repeat_interleave(w.shape[0] // w_scale.shape[0], 0) + ).view(n_groups, o_lora_rank, group_dim), ) torch.testing.assert_close(out.float(), expected.flatten(1), atol=5e-2, rtol=1e-2) diff --git a/tests/kernels/quantization/test_rocm_mxfp8_linear.py b/tests/kernels/quantization/test_rocm_mxfp8_linear.py index 1d5e62c6c4..81d74a229e 100644 --- a/tests/kernels/quantization/test_rocm_mxfp8_linear.py +++ b/tests/kernels/quantization/test_rocm_mxfp8_linear.py @@ -117,6 +117,147 @@ def test_rocm_mxfp8_linear_handles_extreme_magnitudes(): assert (out[0] == 0).all(), "an all-zero activation row must give a zero row" +# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The +# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has +# K % 128 != 0, which only the block-scaled GEMM runs natively. +V41_BLOCK32_SHAPES = [ + ("fused_wqa_wkv", 1280 + 512, 5120), + ("wq_b_tp2", 64 * 512 // 2, 1280), + ("indexer_wq_b", 32 * 128, 1280), + ("wo_b_tp2", 5120, 8192 // 2), + ("shared_gate_up_tp2", 2 * 2304 // 2, 5120), + ("shared_down_tp2", 5120, 2304 // 2), + ("shared_down_tp4", 5120, 2304 // 4), +] + + +def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module: + """An MXFP8 layer with 32x32 block scales, expanded per row as loaded.""" + blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32) + amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny) + bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254) + weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k) + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter( + weight.to(torch.float8_e4m3fn), requires_grad=False + ) + layer.weight_scale = torch.nn.Parameter( + bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False + ) + return layer + + +@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v)) +@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024]) +@torch.inference_mode() +def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens): + """32x32 block scales stay compact and the block-scaled GEMM is exact. + + The token counts cover the packed, tiled and split-K configurations. + """ + from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import ( + Mxfp8LinearLayerConfig, + ) + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + RocmDotScaledMxfp8LinearKernel, + ) + + torch.manual_seed(num_tokens) + device = "cuda" + layer = _make_block32_layer(n, k, device) + row_scale = layer.weight_scale.data.clone() + kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None)) + kernel.process_weights_after_loading(layer) + assert layer.weight.dtype == torch.float8_e4m3fn + assert layer.weight_scale.shape == (n // 32, k // 32) + + x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5 + out = kernel.apply_weights(layer, x) + + x_q, x_scale = mxfp8_e4m3_quantize(x) + expected = ( + dequant_mxfp8_to_bf16(x_q, x_scale).float() + @ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T + ) + assert out.shape == (num_tokens, n) + rel = (out.float() - expected).norm() / expected.norm() + assert rel < 5e-3, f"{name}: relative error {rel:.4f}" + + +@torch.inference_mode() +def test_rocm_mxfp8_linear_keeps_per_row_scales(): + """Scales that differ within a 32-row block are not compacted.""" + from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import ( + Mxfp8LinearLayerConfig, + ) + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + RocmDotScaledMxfp8LinearKernel, + ) + + layer = _make_block32_layer(64, 256, "cuda") + layer.weight_scale.data[1, 0] += 1 + kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None)) + kernel.process_weights_after_loading(layer) + assert layer.weight_scale.shape == (64, 256 // 32) + + +def _split_k_case(packed: bool): + """A 3-way in-launch split-K on shared_down TP4, with K and M tails.""" + from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g + + n, k, m = 5120, 576, 19 + layer = _make_block32_layer(n, k, "cuda") + x_q, x_scale = mxfp8_e4m3_quantize( + torch.randn(m, k, device="cuda", dtype=torch.bfloat16) + ) + expected = ( + dequant_mxfp8_to_bf16(x_q, x_scale).float() + @ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T + ) + if packed: + cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3) + else: + cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3) + args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous()) + return g, cfg, args, expected + + +def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float: + return ((out.float() - expected).norm() / expected.norm()).item() + + +@pytest.mark.parametrize("packed", [True, False]) +@torch.inference_mode() +def test_rocm_mxfp8_block32_split_k_in_launch(packed): + """Splits summed by the tile's last program are exact and repeatable.""" + torch.manual_seed(0) + g, cfg, args, expected = _split_k_case(packed) + out = torch.empty_like(expected, dtype=torch.bfloat16) + g._launch(*args, out, cfg) + assert _rel_err(out, expected) < 5e-3 + first = out.clone() + for _ in range(20): + g._launch(*args, out, cfg) + assert torch.equal(out, first) + + +@pytest.mark.parametrize("packed", [True, False]) +@torch.inference_mode() +def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed): + """A stream whose first split-K launch is being captured has no counters + yet; it must fall back to an unfused split instead of failing.""" + torch.manual_seed(0) + g, cfg, args, expected = _split_k_case(packed) + out = torch.zeros_like(expected, dtype=torch.bfloat16) + stream = torch.cuda.Stream() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + g._launch(*args, out, cfg) + graph.replay() + torch.accelerator.synchronize() + assert _rel_err(out, expected) < 5e-3 + + @pytest.mark.parametrize("num_rows", [1, 64, 65]) @torch.inference_mode() def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows): diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py new file mode 100644 index 0000000000..f4d8441c6a --- /dev/null +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py @@ -0,0 +1,711 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``). + +``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per +row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in +the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being +expanded to every row. Each weight scale byte is then read once per 32 output +rows, and small-M shapes can use the packed kernel below. + +Two kernels, picked per shape from a table tuned on MI355X: + +* a tiled kernel for larger M; +* a packed kernel for small M, which fills the MFMA's rows with K panels + instead of tokens and keeps only the matching-panel (block-diagonal) + products, so a handful of tokens still streams the weight at full width. + +Either can split K. The partials are normally summed in the same launch by +the last program of each output tile to finish, in split order, so the result +does not depend on scheduling. Otherwise a second launch reduces them. +""" + +from typing import NamedTuple + +import torch + +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op + +BLOCK_ROWS = 32 + +# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is +# a multiple of the XCD count in every partition mode. _split_tile hardcodes +# the same stride. +_XCD_STRIDE = 8 +# One arrival counter per output tile of an in-launch split-K GEMM. +_SPLIT_COUNTERS = 1 << 16 + + +@triton.jit +def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr): + """(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on + one XCD, so they meet in that XCD's L2.""" + pid = tl.program_id(0) + tile = pid // (8 * SPLITS) * 8 + pid % 8 + grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N + return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS + + +@triton.jit +def _sum_splits( + acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr +): + """Publish this split's partial; the tile's last arrival sums all of them + in split order and re-arms the counter.""" + BM: tl.constexpr = acc.shape[0] + BN: tl.constexpr = acc.shape[1] + local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :] + tl.store(slot_ptr + split * BM * BN + local, acc) + # The partial must be in L2 before the arrival is counted. The splits + # share an XCD, so no device-scope release (an L2 writeback) is needed. + tl.inline_asm_elementwise( + "s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1 + ) + tl.debug_barrier() + if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1: + total = tl.zeros((BM, BN), dtype=tl.float32) + for s in tl.static_range(SPLITS): + total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv") + tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask) + tl.store(count_ptr, 0) + + +@triton.jit(do_not_specialize=["M"]) +def _block32_tiled_kernel( + x_ptr, + xs_ptr, + w_ptr, + ws_ptr, + out_ptr, + slot_ptr, + count_ptr, + M, + N: tl.constexpr, + K: tl.constexpr, + stride_om, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + K_PER_SPLIT: tl.constexpr, + N_FIRST: tl.constexpr, + EVEN_K: tl.constexpr, + FUSED_SPLITS: tl.constexpr, +): + if FUSED_SPLITS > 1: + pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS) + if pid_m * BLOCK_M >= M: + return + else: + # N_FIRST launches output-column tiles fastest, so consecutive + # programs share an activation tile. + if N_FIRST: + pid_n, pid_m = tl.program_id(0), tl.program_id(1) + else: + pid_m, pid_n = tl.program_id(0), tl.program_id(1) + pid_k = tl.program_id(2) + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + m_mask = offs_m < M + n_mask = offs_n < N + k0 = pid_k * K_PER_SPLIT + offs_k = k0 + tl.arange(0, BLOCK_K) + offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32) + x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :] + xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :] + w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :] + ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :] + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for kk in range(0, K_PER_SPLIT, BLOCK_K): + if EVEN_K: + x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0) + xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=127) + w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0) + ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=127) + else: + k_ok = (offs_k + kk) < K + s_ok = (offs_sk + kk // 32) < K // 32 + x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0) + xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=127) + w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0) + ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=127) + acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc) + x_ptrs += BLOCK_K + w_ptrs += BLOCK_K + xs_ptrs += BLOCK_K // 32 + ws_ptrs += BLOCK_K // 32 + out_mask = m_mask[:, None] & n_mask[None, :] + if FUSED_SPLITS > 1: + _sum_splits( + acc, + out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], + out_mask, + slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N), + count_ptr + tile, + pid_k, + FUSED_SPLITS, + ) + else: + out_ptrs = ( + out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :] + ) + tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask) + + +@triton.jit(do_not_specialize=["M"]) +def _block32_packed_kernel( + x_ptr, + xs_ptr, + w_ptr, + ws_ptr, + out_ptr, + slot_ptr, + count_ptr, + M, + N: tl.constexpr, + K: tl.constexpr, + stride_om, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + K_PACK: tl.constexpr, + K_PER_SPLIT: tl.constexpr, + EVEN_K: tl.constexpr, + FUSED_SPLITS: tl.constexpr, +): + if FUSED_SPLITS > 1: + pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS) + if pid_m * BLOCK_M >= M: + return + k_start = pid_k * K_PER_SPLIT + k_stop = tl.minimum(k_start + K_PER_SPLIT, K) + else: + pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2) + if K_PER_SPLIT >= K: + k_start = 0 + k_stop = K + else: + k_start = pid_k * K_PER_SPLIT + k_stop = tl.minimum(k_start + K_PER_SPLIT, K) + # Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise + # for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices. + rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK + x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK + cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK + w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK + ks = tl.arange(0, BLOCK_K) + gs = tl.arange(0, BLOCK_K // 32) + row_ok = rows < M + col_ok = cols < N + acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32) + for base in range(k_start, k_stop, BLOCK_K * K_PACK): + xk = base + x_panel[:, None] * BLOCK_K + ks[None, :] + wk = base + w_panel[:, None] * BLOCK_K + ks[None, :] + xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :] + wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :] + x_mask = row_ok[:, None] + w_mask = col_ok[:, None] + xs_mask = row_ok[:, None] + ws_mask = col_ok[:, None] + if not EVEN_K: + x_mask = x_mask & (xk < K) + w_mask = w_mask & (wk < K) + xs_mask = xs_mask & (xg < K // 32) + ws_mask = ws_mask & (wg < K // 32) + x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0) + w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0) + xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=127) + ws = tl.load( + ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=127 + ) + acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc) + # Keep only products of matching K panels: the block diagonal. + blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3) + pair = tl.arange(0, K_PACK) + diagonal = tl.where( + pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0 + ) + y = tl.sum(tl.sum(diagonal, 3), 2) + out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :] + out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N) + if FUSED_SPLITS > 1: + _sum_splits( + y, + out_ptrs, + out_mask, + slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N), + count_ptr + tile, + pid_k, + FUSED_SPLITS, + ) + else: + tl.store( + out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask + ) + + +@triton.jit(do_not_specialize=["M"]) +def _block32_splitk_reduce_kernel( + partial_ptr, + out_ptr, + M, + N: tl.constexpr, + stride_om, + SPLIT_K: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + mask = (offs_m[:, None] < M) & (offs_n[None, :] < N) + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for split in tl.static_range(SPLIT_K): + acc += tl.load( + partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :], + mask=mask, + other=0.0, + ) + tl.store( + out_ptr + offs_m[:, None] * stride_om + offs_n[None, :], + acc.to(out_ptr.dtype.element_ty), + mask=mask, + ) + + +class _Config(NamedTuple): + packed: bool + block_m: int + block_n: int + block_k: int + # K panels per MFMA tile (packed) or K partitions reduced by a second + # launch (tiled). + k_split: int + n_first: bool + num_warps: int + num_stages: int + matrix_instr_nonkdim: int + waves_per_eu: int + # K partitions summed within the launch. + fused_splits: int = 1 + + +def _tiled( + bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1 +) -> _Config: + return _Config( + False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused + ) + + +def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config: + return _Config( + True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused + ) + + +# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at +# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A +# tier serves every M up to its bound; larger M reuse the last tier. +_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = { + # shared expert gate_up, TP4 + (1152, 5120): [ + (1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)), + (2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)), + (4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)), + (8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)), + (16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)), + (24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)), + (32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)), + (48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)), + (64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)), + (96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)), + (128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)), + (160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)), + (192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)), + (256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)), + (320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)), + (384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)), + (512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)), + (1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)), + (2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + ], + # fused wq_a|wkv (replicated) + (1792, 5120): [ + (1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)), + (8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)), + (16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)), + (32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)), + (48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)), + (64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)), + (96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)), + (128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)), + (192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)), + (256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)), + (384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)), + (1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + (3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)), + ], + # shared expert gate_up, TP2 + (2304, 5120): [ + (1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)), + (8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)), + (16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)), + (24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)), + (32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)), + (48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)), + (64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)), + (128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)), + (160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)), + (256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)), + (1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)), + (4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)), + (8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)), + ], + # indexer wq_b + (4096, 1280): [ + (1, _packed(4, 16, 256, 4, 4, 2, 16, 1)), + (2, _packed(4, 16, 256, 4, 4, 2, 16, 0)), + (4, _packed(4, 16, 256, 4, 4, 2, 16, 1)), + (8, _packed(8, 16, 256, 4, 4, 3, 16, 1)), + (16, _packed(16, 16, 512, 2, 2, 3, 16, 1)), + (32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)), + (64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)), + (128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)), + (192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)), + (256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)), + (1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)), + ], + # shared expert down, TP4 + (5120, 576): [ + (8, _packed(8, 32, 256, 2, 2, 2, 16, 1)), + (16, _packed(16, 32, 512, 1, 2, 2, 16, 0)), + (32, _packed(32, 32, 256, 1, 8, 3, 16, 1)), + (64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)), + (96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)), + (128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)), + (192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)), + (256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)), + (512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)), + (768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)), + (3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + ], + # shared expert down, TP2 + (5120, 1152): [ + (1, _packed(4, 32, 128, 4, 2, 3, 16, 1)), + (8, _packed(8, 32, 256, 2, 4, 3, 16, 0)), + (32, _packed(32, 32, 512, 1, 2, 3, 16, 1)), + (128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)), + (512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)), + (1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)), + ], + # wo_b, TP4 + (5120, 2048): [ + (1, _packed(4, 32, 256, 4, 2, 2, 16, 0)), + (4, _packed(4, 32, 256, 4, 2, 2, 16, 1)), + (8, _packed(8, 32, 256, 4, 2, 3, 16, 0)), + (16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)), + (32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)), + (64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)), + (96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)), + (128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)), + ], + # wo_b, TP2 + (5120, 4096): [ + (4, _packed(4, 32, 512, 4, 2, 2, 16, 0)), + (8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)), + (16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)), + (24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)), + (32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)), + (48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)), + (64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)), + (128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)), + (192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)), + (256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)), + (320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)), + (768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)), + (3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)), + (8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + ], + # wq_b, TP4 + (8192, 1280): [ + (1, _packed(8, 32, 256, 2, 4, 3, 16, 1)), + (8, _packed(8, 32, 256, 2, 4, 3, 16, 0)), + (16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)), + (32, _packed(32, 32, 512, 1, 2, 3, 16, 0)), + (48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)), + (64, _packed(32, 64, 256, 1, 4, 3, 16, 1)), + (128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)), + (192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)), + (256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)), + (384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)), + (1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)), + (4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)), + ], + # wq_b, TP2 + (16384, 1280): [ + (2, _packed(4, 32, 128, 4, 2, 3, 16, 1)), + (4, _packed(4, 32, 128, 4, 4, 2, 16, 0)), + (8, _packed(8, 32, 256, 2, 4, 2, 16, 0)), + (32, _packed(32, 64, 256, 1, 4, 3, 16, 1)), + (64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)), + (128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)), + (256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)), + (384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)), + (768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)), + (1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)), + (2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)), + (8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)), + ], +} + + +def _default_config(M: int, N: int, K: int) -> _Config: + """Untuned shapes: the tiers most tuned shapes settle on.""" + if M <= 8: + return _packed(8, 16, 256, 2, 2, 3, 16, 0) + if M <= 32: + return _packed(32, 32, 256, 1, 2, 3, 16, 0) + if M <= 256: + return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0) + return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0) + + +def _config(M: int, N: int, K: int) -> _Config: + tiers = _TUNED.get((N, K)) + if tiers is None: + return _default_config(M, N, K) + for max_m, cfg in tiers: + if max_m >= M: + return cfg + return tiers[-1][1] + + +_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {} + + +def _split_counters(device: torch.device) -> torch.Tensor | None: + """Zeroed arrival counters, one set per stream so concurrent launches never + share a tile's counter. Each launch leaves the counters it used at zero. + + None when a stream first needs them inside a CUDA graph capture: that + allocation would come from the graph's pool, where it can take the address + of an intermediate that every replay overwrites. vLLM warms up on its + capture streams first, so this only affects captures without a warmup. + """ + key = (device.index, torch.accelerator.current_stream(device).stream_id) + counters = _split_counters_by_stream.get(key) + if counters is None: + if torch.cuda.is_current_stream_capturing(): + return None + counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device) + _split_counters_by_stream[key] = counters + return counters + + +def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]: + """K per split, rounded up to whole steps, and the number of non-empty + splits.""" + size = triton.cdiv(triton.cdiv(K, splits), step) * step + return size, triton.cdiv(K, size) + + +def _launch( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out: torch.Tensor, + cfg: _Config, +) -> None: + M, K = x.shape + N = weight.shape[0] + opts = dict( + num_warps=cfg.num_warps, + num_stages=cfg.num_stages, + matrix_instr_nonkdim=cfg.matrix_instr_nonkdim, + waves_per_eu=cfg.waves_per_eu, + ) + grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n) + tiles = grid_m * grid_n + step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k + splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split + k_per_split, splits = _k_partition(K, splits, step) + counters = None + if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS: + counters = _split_counters(out.device) + if counters is not None: + fused = splits + target = out + slots = torch.empty( + tiles * fused * cfg.block_m * cfg.block_n, + dtype=torch.float32, + device=out.device, + ) + grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,) + else: + # Without counters, split K through a second launch. + fused = 1 + target = ( + out + if splits == 1 + else torch.empty((splits, M, N), dtype=torch.float32, device=x.device) + ) + slots = counters = target # Unused by the kernels. + if cfg.packed or not cfg.n_first: + grid = (grid_m, grid_n, splits) + else: + grid = (grid_n, grid_m, splits) + args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K) + stride = out.stride(0) if splits == 1 or fused > 1 else N + if cfg.packed: + _block32_packed_kernel[grid]( + *args, + stride, + BLOCK_M=cfg.block_m, + BLOCK_N=cfg.block_n, + BLOCK_K=cfg.block_k, + K_PACK=cfg.k_split, + K_PER_SPLIT=k_per_split, + EVEN_K=K % step == 0, + FUSED_SPLITS=fused, + **opts, + ) + else: + _block32_tiled_kernel[grid]( + *args, + stride, + BLOCK_M=cfg.block_m, + BLOCK_N=cfg.block_n, + BLOCK_K=cfg.block_k, + K_PER_SPLIT=k_per_split, + N_FIRST=cfg.n_first, + EVEN_K=K % k_per_split == 0, + FUSED_SPLITS=fused, + **opts, + ) + if fused == 1 and splits > 1: + _block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))]( + target, + out, + M, + N, + out.stride(0), + SPLIT_K=splits, + BLOCK_M=32, + BLOCK_N=64, + ) + + +def _rocm_mxfp8_block32_gemm_impl( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + M, K = x.shape + out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device) + if M > 0: + _launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K)) + return out + + +def _rocm_mxfp8_block32_gemm_fake( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype) + + +# An opaque op: the kernel and tile choice branch on the token count, which a +# compiled caller must not freeze at its trace-time value. +direct_register_custom_op( + op_name="rocm_mxfp8_block32_gemm", + op_func=_rocm_mxfp8_block32_gemm_impl, + fake_impl=_rocm_mxfp8_block32_gemm_fake, +) + + +def rocm_mxfp8_block32_gemm( + x: torch.Tensor, + x_scale: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + """``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight. + + Args: + x: [M, K] e4m3 activation, contiguous. + x_scale: [M, K / 32] E8M0 (uint8) activation scales. + weight: [N, K] e4m3 weight, contiguous. + weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales. + out_dtype: Output dtype. + + Returns: + The [M, N] product. + + """ + return torch.ops.vllm.rocm_mxfp8_block32_gemm( + x, x_scale, weight, weight_scale, out_dtype + ) diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py index c7e6537f49..1f2e8fdf48 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_native.py @@ -22,6 +22,7 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig +from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm @triton.jit @@ -209,6 +210,17 @@ def _select_cfg(M, N, K): _DOT_SCALED_K_ALIGN = 128 +def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None: + """The [N / 32, K / 32] block scales, if every 32 rows share their scales.""" + N = weight_scale.shape[0] + if N % BLOCK_ROWS != 0: + return None + blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1) + if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)): + return None + return blocks[:, 0].contiguous() + + class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): """Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``.""" @@ -236,7 +248,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): N, K = weight.shape scale_k = K // MXFP8_BLOCK_SIZE weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous() - if K % _DOT_SCALED_K_ALIGN != 0: + block_scale = _as_block32_scale(weight_scale) + if block_scale is not None: + # Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them + # back from the loader expanded per row; keep the blocks and use + # the block-scaled GEMM, which also handles any K % 32 == 0. + weight_scale = block_scale + elif K % _DOT_SCALED_K_ALIGN != 0: weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale) layer.weight = Parameter(weight.contiguous(), requires_grad=False) layer.weight_scale = Parameter(weight_scale, requires_grad=False) @@ -254,7 +272,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel): ) out_shape = (*x.shape[:-1], layer.weight.shape[0]) x2d = x.reshape(-1, x.shape[-1]) - if layer.weight.element_size() >= 2: + if layer.weight_scale.shape[0] != layer.weight.shape[0]: + # One scale row per 32 weight rows (see process_weights_after_loading). + x_q, x_scale = mxfp8_e4m3_quantize(x2d) + out = rocm_mxfp8_block32_gemm( + x_q, x_scale, layer.weight, layer.weight_scale, x.dtype + ) + elif layer.weight.element_size() >= 2: out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype)) elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0: out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale) diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index c7cef64ddf..a3608a9c7e 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -1811,7 +1811,7 @@ def _mxfp8_wo_a_bmm_kernel( a_ptr, # [T, G * K] e4m3 as_ptr, # [T, G * K // 32] E8M0 w_ptr, # [G * R, K] e4m3 - ws_ptr, # [G * R, K // 32] E8M0 + ws_ptr, # [G * R // SCALE_ROWS, K // 32] E8M0 out_ptr, # [T, G * R] num_tokens, stride_at, @@ -1821,6 +1821,7 @@ def _mxfp8_wo_a_bmm_kernel( stride_out, R: tl.constexpr, K: tl.constexpr, + SCALE_ROWS: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, @@ -1841,7 +1842,7 @@ def _mxfp8_wo_a_bmm_kernel( as_ptr + offs_m[:, None] * stride_ast + (group * (K // 32) + offs_sk)[None, :] ) w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] - ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] + ws_ptrs = ws_ptr + (offs_n // SCALE_ROWS)[:, None] * stride_wsn + offs_sk[None, :] acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) for _ in range(K // BLOCK_K): @@ -1926,6 +1927,7 @@ def _rocm_mxfp8_wo_a_bmm_impl( out.stride(0), R=o_lora_rank, K=group_dim, + SCALE_ROWS=weight.shape[0] // weight_scale.shape[0], BLOCK_M=block_m, BLOCK_N=block_n, BLOCK_K=block_k, @@ -1967,7 +1969,8 @@ def rocm_mxfp8_wo_a_bmm( ``a`` is the [T, G * K] e4m3 attention output and ``a_scale`` its [T, G * K // 32] E8M0 scales, as the sparse decode reduce writes them. The weight is the checkpoint's MXFP8 ``wo_a`` as loaded, [G * R, K] with - [G * R, K // 32] scales, so there is no dequantized copy to keep. + either [G * R // 32, K // 32] block scales or [G * R, K // 32] per-row + scales, so there is no dequantized copy to keep. Returns [T, G * R]. """ return torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(