[ROCm][Perf] MXFP8 GEMM on native 32x32 block scales for gfx950 (#58510)
Signed-off-by: fai <fangzhouai@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
055f150df5
commit
3059155b47
@@ -1204,8 +1204,14 @@ def test_sparse_attn_decode_mxfp8_output() -> None:
|
||||
(1100, 8),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("block_scales", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
|
||||
def test_rocm_mxfp8_wo_a_bmm(
|
||||
num_tokens: int, n_groups: int, block_scales: bool
|
||||
) -> None:
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
_as_block32_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
_mxfp8_e4m3_quantize_torch,
|
||||
)
|
||||
@@ -1220,6 +1226,11 @@ def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
|
||||
w, w_scale = _mxfp8_e4m3_quantize_torch(
|
||||
torch.randn(n_groups * o_lora_rank, group_dim, device=device)
|
||||
)
|
||||
if block_scales:
|
||||
# A 32x32 checkpoint loads as per-row scales that repeat every 32 rows;
|
||||
# RocmDotScaledMxfp8LinearKernel compacts them before wo_a sees them.
|
||||
w_scale = _as_block32_scale(w_scale[::32].repeat_interleave(32, dim=0))
|
||||
assert w_scale is not None
|
||||
wo_a = SimpleNamespace(weight=w, weight_scale=w_scale)
|
||||
|
||||
out = rocm_mxfp8_wo_a_bmm(a, a_scale, wo_a, n_groups, o_lora_rank)
|
||||
@@ -1227,7 +1238,9 @@ def test_rocm_mxfp8_wo_a_bmm(num_tokens: int, n_groups: int) -> None:
|
||||
expected = torch.einsum(
|
||||
"tgd,grd->tgr",
|
||||
_mxfp8_dequant(a, a_scale).view(num_tokens, n_groups, group_dim),
|
||||
_mxfp8_dequant(w, w_scale).view(n_groups, o_lora_rank, group_dim),
|
||||
_mxfp8_dequant(
|
||||
w, w_scale.repeat_interleave(w.shape[0] // w_scale.shape[0], 0)
|
||||
).view(n_groups, o_lora_rank, group_dim),
|
||||
)
|
||||
torch.testing.assert_close(out.float(), expected.flatten(1), atol=5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@@ -117,6 +117,147 @@ def test_rocm_mxfp8_linear_handles_extreme_magnitudes():
|
||||
assert (out[0] == 0).all(), "an all-zero activation row must give a zero row"
|
||||
|
||||
|
||||
# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The
|
||||
# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has
|
||||
# K % 128 != 0, which only the block-scaled GEMM runs natively.
|
||||
V41_BLOCK32_SHAPES = [
|
||||
("fused_wqa_wkv", 1280 + 512, 5120),
|
||||
("wq_b_tp2", 64 * 512 // 2, 1280),
|
||||
("indexer_wq_b", 32 * 128, 1280),
|
||||
("wo_b_tp2", 5120, 8192 // 2),
|
||||
("shared_gate_up_tp2", 2 * 2304 // 2, 5120),
|
||||
("shared_down_tp2", 5120, 2304 // 2),
|
||||
("shared_down_tp4", 5120, 2304 // 4),
|
||||
]
|
||||
|
||||
|
||||
def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module:
|
||||
"""An MXFP8 layer with 32x32 block scales, expanded per row as loaded."""
|
||||
blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32)
|
||||
amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny)
|
||||
bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254)
|
||||
weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
weight.to(torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v))
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens):
|
||||
"""32x32 block scales stay compact and the block-scaled GEMM is exact.
|
||||
|
||||
The token counts cover the packed, tiled and split-K configurations.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
torch.manual_seed(num_tokens)
|
||||
device = "cuda"
|
||||
layer = _make_block32_layer(n, k, device)
|
||||
row_scale = layer.weight_scale.data.clone()
|
||||
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight_scale.shape == (n // 32, k // 32)
|
||||
|
||||
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
|
||||
out = kernel.apply_weights(layer, x)
|
||||
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x)
|
||||
expected = (
|
||||
dequant_mxfp8_to_bf16(x_q, x_scale).float()
|
||||
@ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T
|
||||
)
|
||||
assert out.shape == (num_tokens, n)
|
||||
rel = (out.float() - expected).norm() / expected.norm()
|
||||
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_linear_keeps_per_row_scales():
|
||||
"""Scales that differ within a 32-row block are not compacted."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
layer = _make_block32_layer(64, 256, "cuda")
|
||||
layer.weight_scale.data[1, 0] += 1
|
||||
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(bmm_batch_size=None))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.weight_scale.shape == (64, 256 // 32)
|
||||
|
||||
|
||||
def _split_k_case(packed: bool):
|
||||
"""A 3-way in-launch split-K on shared_down TP4, with K and M tails."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g
|
||||
|
||||
n, k, m = 5120, 576, 19
|
||||
layer = _make_block32_layer(n, k, "cuda")
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(
|
||||
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
expected = (
|
||||
dequant_mxfp8_to_bf16(x_q, x_scale).float()
|
||||
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
|
||||
)
|
||||
if packed:
|
||||
cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3)
|
||||
else:
|
||||
cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3)
|
||||
args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous())
|
||||
return g, cfg, args, expected
|
||||
|
||||
|
||||
def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float:
|
||||
return ((out.float() - expected).norm() / expected.norm()).item()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_block32_split_k_in_launch(packed):
|
||||
"""Splits summed by the tile's last program are exact and repeatable."""
|
||||
torch.manual_seed(0)
|
||||
g, cfg, args, expected = _split_k_case(packed)
|
||||
out = torch.empty_like(expected, dtype=torch.bfloat16)
|
||||
g._launch(*args, out, cfg)
|
||||
assert _rel_err(out, expected) < 5e-3
|
||||
first = out.clone()
|
||||
for _ in range(20):
|
||||
g._launch(*args, out, cfg)
|
||||
assert torch.equal(out, first)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed):
|
||||
"""A stream whose first split-K launch is being captured has no counters
|
||||
yet; it must fall back to an unfused split instead of failing."""
|
||||
torch.manual_seed(0)
|
||||
g, cfg, args, expected = _split_k_case(packed)
|
||||
out = torch.zeros_like(expected, dtype=torch.bfloat16)
|
||||
stream = torch.cuda.Stream()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
g._launch(*args, out, cfg)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert _rel_err(out, expected) < 5e-3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_rows", [1, 64, 65])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows):
|
||||
|
||||
@@ -0,0 +1,711 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MXFP8 GEMM on 32x32 block-scaled weights for gfx950 (``tl.dot_scaled``).
|
||||
|
||||
``y = x @ w.T`` with an MXFP8 activation (e4m3 values, one E8M0 scale per
|
||||
row and 32 K, ``[M, K / 32]``) and an e4m3 weight whose E8M0 scales stay in
|
||||
the checkpoint's 32x32 blocks, ``[N / 32, K / 32]``, instead of being
|
||||
expanded to every row. Each weight scale byte is then read once per 32 output
|
||||
rows, and small-M shapes can use the packed kernel below.
|
||||
|
||||
Two kernels, picked per shape from a table tuned on MI355X:
|
||||
|
||||
* a tiled kernel for larger M;
|
||||
* a packed kernel for small M, which fills the MFMA's rows with K panels
|
||||
instead of tokens and keeps only the matching-panel (block-diagonal)
|
||||
products, so a handful of tokens still streams the weight at full width.
|
||||
|
||||
Either can split K. The partials are normally summed in the same launch by
|
||||
the last program of each output tile to finish, in split order, so the result
|
||||
does not depend on scheduling. Otherwise a second launch reduces them.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
BLOCK_ROWS = 32
|
||||
|
||||
# gfx950 hands out workgroups to its XCDs round-robin by program id, and 8 is
|
||||
# a multiple of the XCD count in every partition mode. _split_tile hardcodes
|
||||
# the same stride.
|
||||
_XCD_STRIDE = 8
|
||||
# One arrival counter per output tile of an in-launch split-K GEMM.
|
||||
_SPLIT_COUNTERS = 1 << 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _split_tile(N: tl.constexpr, BLOCK_N: tl.constexpr, SPLITS: tl.constexpr):
|
||||
"""(pid_m, pid_n, tile, split) of a 1D grid that keeps a tile's splits on
|
||||
one XCD, so they meet in that XCD's L2."""
|
||||
pid = tl.program_id(0)
|
||||
tile = pid // (8 * SPLITS) * 8 + pid % 8
|
||||
grid_n: tl.constexpr = (N + BLOCK_N - 1) // BLOCK_N
|
||||
return tile // grid_n, tile % grid_n, tile, pid // 8 % SPLITS
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sum_splits(
|
||||
acc, out_ptrs, out_mask, slot_ptr, count_ptr, split, SPLITS: tl.constexpr
|
||||
):
|
||||
"""Publish this split's partial; the tile's last arrival sums all of them
|
||||
in split order and re-arms the counter."""
|
||||
BM: tl.constexpr = acc.shape[0]
|
||||
BN: tl.constexpr = acc.shape[1]
|
||||
local = tl.arange(0, BM)[:, None] * BN + tl.arange(0, BN)[None, :]
|
||||
tl.store(slot_ptr + split * BM * BN + local, acc)
|
||||
# The partial must be in L2 before the arrival is counted. The splits
|
||||
# share an XCD, so no device-scope release (an L2 writeback) is needed.
|
||||
tl.inline_asm_elementwise(
|
||||
"s_waitcnt vmcnt(0)", "=v,v", [split], dtype=tl.int32, is_pure=False, pack=1
|
||||
)
|
||||
tl.debug_barrier()
|
||||
if tl.atomic_add(count_ptr, 1, sem="acq_rel", scope="cta") == SPLITS - 1:
|
||||
total = tl.zeros((BM, BN), dtype=tl.float32)
|
||||
for s in tl.static_range(SPLITS):
|
||||
total += tl.load(slot_ptr + s * BM * BN + local, cache_modifier=".cv")
|
||||
tl.store(out_ptrs, total.to(out_ptrs.dtype.element_ty), mask=out_mask)
|
||||
tl.store(count_ptr, 0)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_tiled_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
slot_ptr,
|
||||
count_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
stride_om,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
K_PER_SPLIT: tl.constexpr,
|
||||
N_FIRST: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FUSED_SPLITS: tl.constexpr,
|
||||
):
|
||||
if FUSED_SPLITS > 1:
|
||||
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
|
||||
if pid_m * BLOCK_M >= M:
|
||||
return
|
||||
else:
|
||||
# N_FIRST launches output-column tiles fastest, so consecutive
|
||||
# programs share an activation tile.
|
||||
if N_FIRST:
|
||||
pid_n, pid_m = tl.program_id(0), tl.program_id(1)
|
||||
else:
|
||||
pid_m, pid_n = tl.program_id(0), tl.program_id(1)
|
||||
pid_k = tl.program_id(2)
|
||||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
m_mask = offs_m < M
|
||||
n_mask = offs_n < N
|
||||
k0 = pid_k * K_PER_SPLIT
|
||||
offs_k = k0 + tl.arange(0, BLOCK_K)
|
||||
offs_sk = k0 // 32 + tl.arange(0, BLOCK_K // 32)
|
||||
x_ptrs = x_ptr + offs_m[:, None] * K + offs_k[None, :]
|
||||
xs_ptrs = xs_ptr + offs_m[:, None] * (K // 32) + offs_sk[None, :]
|
||||
w_ptrs = w_ptr + offs_n[:, None] * K + offs_k[None, :]
|
||||
ws_ptrs = ws_ptr + (offs_n[:, None] // 32) * (K // 32) + offs_sk[None, :]
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for kk in range(0, K_PER_SPLIT, BLOCK_K):
|
||||
if EVEN_K:
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=127)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=127)
|
||||
else:
|
||||
k_ok = (offs_k + kk) < K
|
||||
s_ok = (offs_sk + kk // 32) < K // 32
|
||||
x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0)
|
||||
xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=127)
|
||||
w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0)
|
||||
ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=127)
|
||||
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
|
||||
x_ptrs += BLOCK_K
|
||||
w_ptrs += BLOCK_K
|
||||
xs_ptrs += BLOCK_K // 32
|
||||
ws_ptrs += BLOCK_K // 32
|
||||
out_mask = m_mask[:, None] & n_mask[None, :]
|
||||
if FUSED_SPLITS > 1:
|
||||
_sum_splits(
|
||||
acc,
|
||||
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
|
||||
out_mask,
|
||||
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
|
||||
count_ptr + tile,
|
||||
pid_k,
|
||||
FUSED_SPLITS,
|
||||
)
|
||||
else:
|
||||
out_ptrs = (
|
||||
out_ptr + pid_k * M * N + offs_m[:, None] * stride_om + offs_n[None, :]
|
||||
)
|
||||
tl.store(out_ptrs, acc.to(out_ptr.dtype.element_ty), mask=out_mask)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_packed_kernel(
|
||||
x_ptr,
|
||||
xs_ptr,
|
||||
w_ptr,
|
||||
ws_ptr,
|
||||
out_ptr,
|
||||
slot_ptr,
|
||||
count_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
stride_om,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
K_PACK: tl.constexpr,
|
||||
K_PER_SPLIT: tl.constexpr,
|
||||
EVEN_K: tl.constexpr,
|
||||
FUSED_SPLITS: tl.constexpr,
|
||||
):
|
||||
if FUSED_SPLITS > 1:
|
||||
pid_m, pid_n, tile, pid_k = _split_tile(N, BLOCK_N, FUSED_SPLITS)
|
||||
if pid_m * BLOCK_M >= M:
|
||||
return
|
||||
k_start = pid_k * K_PER_SPLIT
|
||||
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
|
||||
else:
|
||||
pid_m, pid_n, pid_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
if K_PER_SPLIT >= K:
|
||||
k_start = 0
|
||||
k_stop = K
|
||||
else:
|
||||
k_start = pid_k * K_PER_SPLIT
|
||||
k_stop = tl.minimum(k_start + K_PER_SPLIT, K)
|
||||
# Tile row r is token r // K_PACK reading K panel r % K_PACK (and likewise
|
||||
# for weight rows), so one dot covers K_PACK consecutive BLOCK_K slices.
|
||||
rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M * K_PACK) // K_PACK
|
||||
x_panel = tl.arange(0, BLOCK_M * K_PACK) % K_PACK
|
||||
cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N * K_PACK) // K_PACK
|
||||
w_panel = tl.arange(0, BLOCK_N * K_PACK) % K_PACK
|
||||
ks = tl.arange(0, BLOCK_K)
|
||||
gs = tl.arange(0, BLOCK_K // 32)
|
||||
row_ok = rows < M
|
||||
col_ok = cols < N
|
||||
acc = tl.zeros((BLOCK_M * K_PACK, BLOCK_N * K_PACK), dtype=tl.float32)
|
||||
for base in range(k_start, k_stop, BLOCK_K * K_PACK):
|
||||
xk = base + x_panel[:, None] * BLOCK_K + ks[None, :]
|
||||
wk = base + w_panel[:, None] * BLOCK_K + ks[None, :]
|
||||
xg = base // 32 + x_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
|
||||
wg = base // 32 + w_panel[:, None] * (BLOCK_K // 32) + gs[None, :]
|
||||
x_mask = row_ok[:, None]
|
||||
w_mask = col_ok[:, None]
|
||||
xs_mask = row_ok[:, None]
|
||||
ws_mask = col_ok[:, None]
|
||||
if not EVEN_K:
|
||||
x_mask = x_mask & (xk < K)
|
||||
w_mask = w_mask & (wk < K)
|
||||
xs_mask = xs_mask & (xg < K // 32)
|
||||
ws_mask = ws_mask & (wg < K // 32)
|
||||
x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0)
|
||||
w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0)
|
||||
xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=127)
|
||||
ws = tl.load(
|
||||
ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=127
|
||||
)
|
||||
acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc)
|
||||
# Keep only products of matching K panels: the block diagonal.
|
||||
blocks = acc.reshape(BLOCK_M, K_PACK, BLOCK_N, K_PACK).trans(0, 2, 1, 3)
|
||||
pair = tl.arange(0, K_PACK)
|
||||
diagonal = tl.where(
|
||||
pair[None, None, :, None] == pair[None, None, None, :], blocks, 0.0
|
||||
)
|
||||
y = tl.sum(tl.sum(diagonal, 3), 2)
|
||||
out_rows = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
out_cols = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
out_ptrs = out_ptr + out_rows[:, None] * stride_om + out_cols[None, :]
|
||||
out_mask = (out_rows[:, None] < M) & (out_cols[None, :] < N)
|
||||
if FUSED_SPLITS > 1:
|
||||
_sum_splits(
|
||||
y,
|
||||
out_ptrs,
|
||||
out_mask,
|
||||
slot_ptr + tile * (FUSED_SPLITS * BLOCK_M * BLOCK_N),
|
||||
count_ptr + tile,
|
||||
pid_k,
|
||||
FUSED_SPLITS,
|
||||
)
|
||||
else:
|
||||
tl.store(
|
||||
out_ptrs + pid_k * M * N, y.to(out_ptr.dtype.element_ty), mask=out_mask
|
||||
)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["M"])
|
||||
def _block32_splitk_reduce_kernel(
|
||||
partial_ptr,
|
||||
out_ptr,
|
||||
M,
|
||||
N: tl.constexpr,
|
||||
stride_om,
|
||||
SPLIT_K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
offs_m = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
offs_n = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
||||
mask = (offs_m[:, None] < M) & (offs_n[None, :] < N)
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for split in tl.static_range(SPLIT_K):
|
||||
acc += tl.load(
|
||||
partial_ptr + split * M * N + offs_m[:, None] * N + offs_n[None, :],
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
tl.store(
|
||||
out_ptr + offs_m[:, None] * stride_om + offs_n[None, :],
|
||||
acc.to(out_ptr.dtype.element_ty),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
class _Config(NamedTuple):
|
||||
packed: bool
|
||||
block_m: int
|
||||
block_n: int
|
||||
block_k: int
|
||||
# K panels per MFMA tile (packed) or K partitions reduced by a second
|
||||
# launch (tiled).
|
||||
k_split: int
|
||||
n_first: bool
|
||||
num_warps: int
|
||||
num_stages: int
|
||||
matrix_instr_nonkdim: int
|
||||
waves_per_eu: int
|
||||
# K partitions summed within the launch.
|
||||
fused_splits: int = 1
|
||||
|
||||
|
||||
def _tiled(
|
||||
bm, bn, bk, split_k, n_first, warps, stages, nonkdim, waves, fused=1
|
||||
) -> _Config:
|
||||
return _Config(
|
||||
False, bm, bn, bk, split_k, bool(n_first), warps, stages, nonkdim, waves, fused
|
||||
)
|
||||
|
||||
|
||||
def _packed(bm, bn, bk, k_pack, warps, stages, nonkdim, waves, fused=1) -> _Config:
|
||||
return _Config(
|
||||
True, bm, bn, bk, k_pack, False, warps, stages, nonkdim, waves, fused
|
||||
)
|
||||
|
||||
|
||||
# (N, K) -> [(max M, config), ...] for the DeepSeek-V4.1-Flash MXFP8 linears at
|
||||
# TP2/TP4, tuned on MI355X under HIP graphs with weights cold in the LLC. A
|
||||
# tier serves every M up to its bound; larger M reuse the last tier.
|
||||
_TUNED: dict[tuple[int, int], list[tuple[int, _Config]]] = {
|
||||
# shared expert gate_up, TP4
|
||||
(1152, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 2, 16, 1, 3)),
|
||||
(2, _packed(4, 32, 256, 4, 2, 2, 16, 1, 5)),
|
||||
(4, _packed(4, 32, 256, 4, 2, 2, 16, 0, 5)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 2, 16, 0, 3)),
|
||||
(16, _tiled(16, 32, 512, 1, True, 2, 3, 16, 0, 5)),
|
||||
(24, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1, 5)),
|
||||
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 0, 5)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 4, 3, 16, 1, 5)),
|
||||
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 5)),
|
||||
(96, _tiled(32, 16, 1024, 1, True, 2, 3, 16, 1)),
|
||||
(128, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 3)),
|
||||
(160, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
|
||||
(192, _tiled(32, 32, 1024, 1, True, 4, 2, 16, 0)),
|
||||
(256, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(320, _tiled(64, 32, 256, 1, True, 8, 3, 16, 0, 2)),
|
||||
(384, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
|
||||
(512, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(768, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 256, 2, False, 4, 2, 16, 1)),
|
||||
(1536, _tiled(128, 128, 256, 2, False, 4, 2, 16, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
],
|
||||
# fused wq_a|wkv (replicated)
|
||||
(1792, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
|
||||
(16, _tiled(16, 64, 1024, 1, True, 4, 2, 16, 0, 5)),
|
||||
(32, _packed(16, 16, 1024, 1, 2, 3, 16, 1)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 2, 16, 0, 4)),
|
||||
(64, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1, 4)),
|
||||
(96, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 2)),
|
||||
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 4)),
|
||||
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(256, _tiled(64, 64, 512, 1, True, 4, 2, 16, 1, 2)),
|
||||
(384, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(512, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(1024, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(3072, _tiled(128, 128, 256, 1, False, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 16, 1)),
|
||||
],
|
||||
# shared expert gate_up, TP2
|
||||
(2304, 5120): [
|
||||
(1, _packed(8, 16, 1024, 2, 2, 3, 16, 1)),
|
||||
(8, _packed(8, 16, 1024, 2, 2, 3, 16, 0)),
|
||||
(16, _tiled(16, 64, 1024, 1, True, 4, 3, 16, 1, 5)),
|
||||
(24, _packed(32, 32, 512, 1, 4, 2, 16, 0, 5)),
|
||||
(32, _packed(32, 32, 1024, 1, 4, 2, 16, 1, 3)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0, 3)),
|
||||
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 5)),
|
||||
(128, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0, 3)),
|
||||
(160, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 2)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 16, 0, 2)),
|
||||
(256, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(320, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 8, 2, 16, 0, 3)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(1536, _tiled(128, 128, 256, 1, False, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
|
||||
(4096, _tiled(128, 128, 256, 1, False, 4, 2, 32, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, False, 8, 2, 32, 0)),
|
||||
],
|
||||
# indexer wq_b
|
||||
(4096, 1280): [
|
||||
(1, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
|
||||
(2, _packed(4, 16, 256, 4, 4, 2, 16, 0)),
|
||||
(4, _packed(4, 16, 256, 4, 4, 2, 16, 1)),
|
||||
(8, _packed(8, 16, 256, 4, 4, 3, 16, 1)),
|
||||
(16, _packed(16, 16, 512, 2, 2, 3, 16, 1)),
|
||||
(32, _packed(32, 16, 1024, 1, 2, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 512, 1, True, 4, 3, 16, 1)),
|
||||
(128, _tiled(64, 32, 512, 1, True, 8, 3, 16, 0)),
|
||||
(192, _tiled(64, 32, 256, 1, True, 4, 3, 16, 0)),
|
||||
(256, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(384, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(512, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(1024, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)),
|
||||
],
|
||||
# shared expert down, TP4
|
||||
(5120, 576): [
|
||||
(8, _packed(8, 32, 256, 2, 2, 2, 16, 1)),
|
||||
(16, _packed(16, 32, 512, 1, 2, 2, 16, 0)),
|
||||
(32, _packed(32, 32, 256, 1, 8, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
|
||||
(96, _tiled(32, 64, 256, 1, True, 4, 3, 16, 0)),
|
||||
(128, _tiled(32, 32, 256, 1, True, 2, 2, 16, 0)),
|
||||
(192, _tiled(64, 64, 256, 1, True, 4, 3, 32, 0)),
|
||||
(256, _tiled(64, 64, 256, 1, True, 2, 3, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(512, _tiled(64, 64, 128, 1, True, 4, 2, 32, 0)),
|
||||
(768, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1024, _tiled(128, 64, 128, 1, True, 4, 2, 32, 0)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
],
|
||||
# shared expert down, TP2
|
||||
(5120, 1152): [
|
||||
(1, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
|
||||
(32, _packed(32, 32, 512, 1, 2, 3, 16, 1)),
|
||||
(128, _tiled(32, 32, 128, 1, True, 4, 3, 16, 0)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 64, 128, 1, True, 8, 2, 32, 0)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(3072, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(8192, _tiled(128, 64, 128, 1, True, 8, 2, 32, 1)),
|
||||
],
|
||||
# wo_b, TP4
|
||||
(5120, 2048): [
|
||||
(1, _packed(4, 32, 256, 4, 2, 2, 16, 0)),
|
||||
(4, _packed(4, 32, 256, 4, 2, 2, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 4, 2, 3, 16, 0)),
|
||||
(16, _tiled(16, 32, 1024, 1, True, 2, 3, 16, 0)),
|
||||
(32, _tiled(32, 32, 512, 1, False, 4, 3, 16, 1)),
|
||||
(64, _tiled(32, 32, 512, 1, True, 4, 2, 16, 0)),
|
||||
(96, _tiled(32, 64, 512, 1, True, 8, 2, 16, 0)),
|
||||
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 2, 2, 32, 1)),
|
||||
],
|
||||
# wo_b, TP2
|
||||
(5120, 4096): [
|
||||
(4, _packed(4, 32, 512, 4, 2, 2, 16, 0)),
|
||||
(8, _packed(8, 16, 128, 4, 2, 2, 16, 0, 2)),
|
||||
(16, _tiled(16, 32, 256, 1, True, 2, 2, 16, 0, 4)),
|
||||
(24, _tiled(32, 32, 256, 1, True, 4, 3, 16, 0, 4)),
|
||||
(32, _tiled(32, 64, 512, 1, True, 8, 3, 16, 0, 3)),
|
||||
(48, _tiled(64, 64, 512, 1, True, 8, 2, 16, 1, 3)),
|
||||
(64, _tiled(64, 64, 512, 1, True, 8, 2, 16, 0, 3)),
|
||||
(128, _tiled(64, 64, 256, 1, True, 4, 2, 16, 0, 3)),
|
||||
(192, _tiled(64, 64, 512, 1, True, 4, 2, 32, 0)),
|
||||
(256, _tiled(128, 128, 256, 1, True, 4, 2, 16, 1, 2)),
|
||||
(320, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(384, _tiled(128, 128, 256, 1, True, 4, 2, 16, 0, 2)),
|
||||
(768, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(2048, _tiled(128, 128, 256, 1, True, 8, 2, 32, 1)),
|
||||
(3072, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
|
||||
(8192, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
],
|
||||
# wq_b, TP4
|
||||
(8192, 1280): [
|
||||
(1, _packed(8, 32, 256, 2, 4, 3, 16, 1)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 3, 16, 0)),
|
||||
(16, _packed(16, 32, 1024, 1, 2, 3, 16, 1)),
|
||||
(32, _packed(32, 32, 512, 1, 2, 3, 16, 0)),
|
||||
(48, _tiled(64, 32, 512, 1, True, 8, 3, 16, 1)),
|
||||
(64, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
|
||||
(128, _tiled(64, 64, 512, 1, True, 4, 2, 32, 1)),
|
||||
(192, _tiled(64, 64, 256, 1, True, 4, 2, 32, 0)),
|
||||
(256, _tiled(128, 64, 256, 1, True, 8, 3, 32, 1)),
|
||||
(384, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(512, _tiled(128, 128, 256, 1, True, 4, 2, 32, 0)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(1536, _tiled(128, 128, 128, 1, True, 8, 2, 16, 0)),
|
||||
(4096, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
],
|
||||
# wq_b, TP2
|
||||
(16384, 1280): [
|
||||
(2, _packed(4, 32, 128, 4, 2, 3, 16, 1)),
|
||||
(4, _packed(4, 32, 128, 4, 4, 2, 16, 0)),
|
||||
(8, _packed(8, 32, 256, 2, 4, 2, 16, 0)),
|
||||
(32, _packed(32, 64, 256, 1, 4, 3, 16, 1)),
|
||||
(64, _tiled(64, 64, 256, 1, False, 4, 3, 32, 1)),
|
||||
(128, _tiled(128, 64, 256, 1, True, 8, 3, 32, 0)),
|
||||
(256, _tiled(128, 128, 256, 1, True, 4, 2, 32, 1)),
|
||||
(384, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(512, _tiled(128, 128, 128, 1, True, 8, 2, 32, 0)),
|
||||
(768, _tiled(128, 128, 128, 1, True, 8, 2, 16, 1)),
|
||||
(1024, _tiled(128, 128, 128, 1, True, 8, 2, 32, 1)),
|
||||
(2048, _tiled(128, 128, 128, 1, True, 4, 2, 32, 0)),
|
||||
(8192, _tiled(128, 256, 128, 1, True, 2, 2, 32, 0)),
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _default_config(M: int, N: int, K: int) -> _Config:
|
||||
"""Untuned shapes: the tiers most tuned shapes settle on."""
|
||||
if M <= 8:
|
||||
return _packed(8, 16, 256, 2, 2, 3, 16, 0)
|
||||
if M <= 32:
|
||||
return _packed(32, 32, 256, 1, 2, 3, 16, 0)
|
||||
if M <= 256:
|
||||
return _tiled(64, 64, 256, 1, True, 4, 2, 16, 0)
|
||||
return _tiled(128, 128, 128, 1, True, 4, 2, 16, 0)
|
||||
|
||||
|
||||
def _config(M: int, N: int, K: int) -> _Config:
|
||||
tiers = _TUNED.get((N, K))
|
||||
if tiers is None:
|
||||
return _default_config(M, N, K)
|
||||
for max_m, cfg in tiers:
|
||||
if max_m >= M:
|
||||
return cfg
|
||||
return tiers[-1][1]
|
||||
|
||||
|
||||
_split_counters_by_stream: dict[tuple[int, int], torch.Tensor] = {}
|
||||
|
||||
|
||||
def _split_counters(device: torch.device) -> torch.Tensor | None:
|
||||
"""Zeroed arrival counters, one set per stream so concurrent launches never
|
||||
share a tile's counter. Each launch leaves the counters it used at zero.
|
||||
|
||||
None when a stream first needs them inside a CUDA graph capture: that
|
||||
allocation would come from the graph's pool, where it can take the address
|
||||
of an intermediate that every replay overwrites. vLLM warms up on its
|
||||
capture streams first, so this only affects captures without a warmup.
|
||||
"""
|
||||
key = (device.index, torch.accelerator.current_stream(device).stream_id)
|
||||
counters = _split_counters_by_stream.get(key)
|
||||
if counters is None:
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return None
|
||||
counters = torch.zeros(_SPLIT_COUNTERS, dtype=torch.int32, device=device)
|
||||
_split_counters_by_stream[key] = counters
|
||||
return counters
|
||||
|
||||
|
||||
def _k_partition(K: int, splits: int, step: int) -> tuple[int, int]:
|
||||
"""K per split, rounded up to whole steps, and the number of non-empty
|
||||
splits."""
|
||||
size = triton.cdiv(triton.cdiv(K, splits), step) * step
|
||||
return size, triton.cdiv(K, size)
|
||||
|
||||
|
||||
def _launch(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
cfg: _Config,
|
||||
) -> None:
|
||||
M, K = x.shape
|
||||
N = weight.shape[0]
|
||||
opts = dict(
|
||||
num_warps=cfg.num_warps,
|
||||
num_stages=cfg.num_stages,
|
||||
matrix_instr_nonkdim=cfg.matrix_instr_nonkdim,
|
||||
waves_per_eu=cfg.waves_per_eu,
|
||||
)
|
||||
grid_m, grid_n = triton.cdiv(M, cfg.block_m), triton.cdiv(N, cfg.block_n)
|
||||
tiles = grid_m * grid_n
|
||||
step = cfg.block_k * cfg.k_split if cfg.packed else cfg.block_k
|
||||
splits = cfg.fused_splits if cfg.fused_splits > 1 or cfg.packed else cfg.k_split
|
||||
k_per_split, splits = _k_partition(K, splits, step)
|
||||
counters = None
|
||||
if cfg.fused_splits > 1 and splits > 1 and tiles <= _SPLIT_COUNTERS:
|
||||
counters = _split_counters(out.device)
|
||||
if counters is not None:
|
||||
fused = splits
|
||||
target = out
|
||||
slots = torch.empty(
|
||||
tiles * fused * cfg.block_m * cfg.block_n,
|
||||
dtype=torch.float32,
|
||||
device=out.device,
|
||||
)
|
||||
grid: tuple[int, ...] = (triton.cdiv(tiles, _XCD_STRIDE) * _XCD_STRIDE * fused,)
|
||||
else:
|
||||
# Without counters, split K through a second launch.
|
||||
fused = 1
|
||||
target = (
|
||||
out
|
||||
if splits == 1
|
||||
else torch.empty((splits, M, N), dtype=torch.float32, device=x.device)
|
||||
)
|
||||
slots = counters = target # Unused by the kernels.
|
||||
if cfg.packed or not cfg.n_first:
|
||||
grid = (grid_m, grid_n, splits)
|
||||
else:
|
||||
grid = (grid_n, grid_m, splits)
|
||||
args = (x, x_scale, weight, weight_scale, target, slots, counters, M, N, K)
|
||||
stride = out.stride(0) if splits == 1 or fused > 1 else N
|
||||
if cfg.packed:
|
||||
_block32_packed_kernel[grid](
|
||||
*args,
|
||||
stride,
|
||||
BLOCK_M=cfg.block_m,
|
||||
BLOCK_N=cfg.block_n,
|
||||
BLOCK_K=cfg.block_k,
|
||||
K_PACK=cfg.k_split,
|
||||
K_PER_SPLIT=k_per_split,
|
||||
EVEN_K=K % step == 0,
|
||||
FUSED_SPLITS=fused,
|
||||
**opts,
|
||||
)
|
||||
else:
|
||||
_block32_tiled_kernel[grid](
|
||||
*args,
|
||||
stride,
|
||||
BLOCK_M=cfg.block_m,
|
||||
BLOCK_N=cfg.block_n,
|
||||
BLOCK_K=cfg.block_k,
|
||||
K_PER_SPLIT=k_per_split,
|
||||
N_FIRST=cfg.n_first,
|
||||
EVEN_K=K % k_per_split == 0,
|
||||
FUSED_SPLITS=fused,
|
||||
**opts,
|
||||
)
|
||||
if fused == 1 and splits > 1:
|
||||
_block32_splitk_reduce_kernel[(triton.cdiv(M, 32), triton.cdiv(N, 64))](
|
||||
target,
|
||||
out,
|
||||
M,
|
||||
N,
|
||||
out.stride(0),
|
||||
SPLIT_K=splits,
|
||||
BLOCK_M=32,
|
||||
BLOCK_N=64,
|
||||
)
|
||||
|
||||
|
||||
def _rocm_mxfp8_block32_gemm_impl(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
M, K = x.shape
|
||||
out = torch.empty((M, weight.shape[0]), dtype=out_dtype, device=x.device)
|
||||
if M > 0:
|
||||
_launch(x, x_scale, weight, weight_scale, out, _config(M, weight.shape[0], K))
|
||||
return out
|
||||
|
||||
|
||||
def _rocm_mxfp8_block32_gemm_fake(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((x.shape[0], weight.shape[0]), dtype=out_dtype)
|
||||
|
||||
|
||||
# An opaque op: the kernel and tile choice branch on the token count, which a
|
||||
# compiled caller must not freeze at its trace-time value.
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_mxfp8_block32_gemm",
|
||||
op_func=_rocm_mxfp8_block32_gemm_impl,
|
||||
fake_impl=_rocm_mxfp8_block32_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def rocm_mxfp8_block32_gemm(
|
||||
x: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""``x @ weight.T`` for MXFP8 ``x`` and a 32x32 block-scaled MXFP8 weight.
|
||||
|
||||
Args:
|
||||
x: [M, K] e4m3 activation, contiguous.
|
||||
x_scale: [M, K / 32] E8M0 (uint8) activation scales.
|
||||
weight: [N, K] e4m3 weight, contiguous.
|
||||
weight_scale: [ceil(N / 32), K / 32] E8M0 (uint8) weight block scales.
|
||||
out_dtype: Output dtype.
|
||||
|
||||
Returns:
|
||||
The [M, N] product.
|
||||
|
||||
"""
|
||||
return torch.ops.vllm.rocm_mxfp8_block32_gemm(
|
||||
x, x_scale, weight, weight_scale, out_dtype
|
||||
)
|
||||
@@ -22,6 +22,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
from .rocm_block32_gemm import BLOCK_ROWS, rocm_mxfp8_block32_gemm
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -209,6 +210,17 @@ def _select_cfg(M, N, K):
|
||||
_DOT_SCALED_K_ALIGN = 128
|
||||
|
||||
|
||||
def _as_block32_scale(weight_scale: torch.Tensor) -> torch.Tensor | None:
|
||||
"""The [N / 32, K / 32] block scales, if every 32 rows share their scales."""
|
||||
N = weight_scale.shape[0]
|
||||
if N % BLOCK_ROWS != 0:
|
||||
return None
|
||||
blocks = weight_scale.view(N // BLOCK_ROWS, BLOCK_ROWS, -1)
|
||||
if not torch.equal(blocks, blocks[:, :1].expand_as(blocks)):
|
||||
return None
|
||||
return blocks[:, 0].contiguous()
|
||||
|
||||
|
||||
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
|
||||
|
||||
@@ -236,7 +248,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
if K % _DOT_SCALED_K_ALIGN != 0:
|
||||
block_scale = _as_block32_scale(weight_scale)
|
||||
if block_scale is not None:
|
||||
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
|
||||
# back from the loader expanded per row; keep the blocks and use
|
||||
# the block-scaled GEMM, which also handles any K % 32 == 0.
|
||||
weight_scale = block_scale
|
||||
elif K % _DOT_SCALED_K_ALIGN != 0:
|
||||
weight = dequant_mxfp8_to_bf16(weight.contiguous(), weight_scale)
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
|
||||
@@ -254,7 +272,13 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
)
|
||||
out_shape = (*x.shape[:-1], layer.weight.shape[0])
|
||||
x2d = x.reshape(-1, x.shape[-1])
|
||||
if layer.weight.element_size() >= 2:
|
||||
if layer.weight_scale.shape[0] != layer.weight.shape[0]:
|
||||
# One scale row per 32 weight rows (see process_weights_after_loading).
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x2d)
|
||||
out = rocm_mxfp8_block32_gemm(
|
||||
x_q, x_scale, layer.weight, layer.weight_scale, x.dtype
|
||||
)
|
||||
elif layer.weight.element_size() >= 2:
|
||||
out = torch.nn.functional.linear(x2d, layer.weight.to(x.dtype))
|
||||
elif x2d.shape[-1] % _DOT_SCALED_K_ALIGN == 0:
|
||||
out = _mxfp8_dot_scaled_linear(x2d, layer.weight, layer.weight_scale)
|
||||
|
||||
@@ -1811,7 +1811,7 @@ def _mxfp8_wo_a_bmm_kernel(
|
||||
a_ptr, # [T, G * K] e4m3
|
||||
as_ptr, # [T, G * K // 32] E8M0
|
||||
w_ptr, # [G * R, K] e4m3
|
||||
ws_ptr, # [G * R, K // 32] E8M0
|
||||
ws_ptr, # [G * R // SCALE_ROWS, K // 32] E8M0
|
||||
out_ptr, # [T, G * R]
|
||||
num_tokens,
|
||||
stride_at,
|
||||
@@ -1821,6 +1821,7 @@ def _mxfp8_wo_a_bmm_kernel(
|
||||
stride_out,
|
||||
R: tl.constexpr,
|
||||
K: tl.constexpr,
|
||||
SCALE_ROWS: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
@@ -1841,7 +1842,7 @@ def _mxfp8_wo_a_bmm_kernel(
|
||||
as_ptr + offs_m[:, None] * stride_ast + (group * (K // 32) + offs_sk)[None, :]
|
||||
)
|
||||
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :]
|
||||
ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :]
|
||||
ws_ptrs = ws_ptr + (offs_n // SCALE_ROWS)[:, None] * stride_wsn + offs_sk[None, :]
|
||||
|
||||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
for _ in range(K // BLOCK_K):
|
||||
@@ -1926,6 +1927,7 @@ def _rocm_mxfp8_wo_a_bmm_impl(
|
||||
out.stride(0),
|
||||
R=o_lora_rank,
|
||||
K=group_dim,
|
||||
SCALE_ROWS=weight.shape[0] // weight_scale.shape[0],
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
BLOCK_K=block_k,
|
||||
@@ -1967,7 +1969,8 @@ def rocm_mxfp8_wo_a_bmm(
|
||||
``a`` is the [T, G * K] e4m3 attention output and ``a_scale`` its
|
||||
[T, G * K // 32] E8M0 scales, as the sparse decode reduce writes them.
|
||||
The weight is the checkpoint's MXFP8 ``wo_a`` as loaded, [G * R, K] with
|
||||
[G * R, K // 32] scales, so there is no dequantized copy to keep.
|
||||
either [G * R // 32, K // 32] block scales or [G * R, K // 32] per-row
|
||||
scales, so there is no dequantized copy to keep.
|
||||
Returns [T, G * R].
|
||||
"""
|
||||
return torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(
|
||||
|
||||
Reference in New Issue
Block a user