[Kernel][DSV4.1] Fuse MXFP8 wo_b GEMM with sequence-parallel reduce-scatter (#57428)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Signed-off-by: Canlin <canlinguosdu@gmail.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Canlin Guo
2026-09-22 01:07:17 -07:00
committed by GitHub
co-authored by Thien Tran OpenAI Codex Claude Fable 5.1 Jee Jee Li
parent a6c47fbbf4
commit f92b78f6ef
18 changed files with 1037 additions and 624 deletions
@@ -6,7 +6,7 @@ import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from tests.utils import ensure_current_vllm_config
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.parallel_state import (
init_distributed_environment,
@@ -17,6 +17,7 @@ from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
_SHAPES = (
(129, 512),
(129, 768),
(257, 768),
(1023, 4224),
@@ -29,6 +30,7 @@ _N = 512
def _reference(
x: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor | None,
world_size: int,
group: dist.ProcessGroup,
all_reduce: bool,
@@ -40,7 +42,21 @@ def _reference(
dtype=x.dtype,
device=x.device,
)
torch.mm(x, weight.T, out=partial[:M])
if weight_scale is None:
torch.mm(x, weight.T, out=partial[:M])
else:
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
x_q, x_sf = mxfp8_e4m3_quantize(x, is_sf_swizzled_layout=True)
partial[:M] = torch._scaled_mm(
x_q,
weight.T,
x_sf.view(torch.float8_e8m0fnu),
weight_scale.view(torch.float8_e8m0fnu),
out_dtype=x.dtype,
)
if padded_M > M:
partial[M:].zero_()
@@ -75,6 +91,10 @@ def _assert_valid_rows_close(
rtol=5e-2,
atol=4.0,
)
# Sequence-parallel callers may pass an unpadded M and read the padding
# rows back (they flow through per-token kernels before being dropped),
# so RS must leave them zero like ``sp_reduce_scatter`` does.
assert not actual[valid_rows:].any()
def _run_mode(
@@ -85,19 +105,52 @@ def _run_mode(
rank: int,
world_size: int,
weights: dict[int, torch.Tensor],
backend: str,
) -> None:
# cute_dsl is unavailable off CUDA, so import it only inside the GPU worker.
from vllm.models.kimi_k3.nvidia.ops.cute_dsl.gemm_rs_ar import GemmRsAr
from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import GemmRsAr
gemm_rs_ar = GemmRsAr(
max_M=max(M for M, _ in _SHAPES),
N=_N,
all_reduce=all_reduce,
)
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
quant_config = (
None
if backend == "bf16"
else OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
)
projections = {}
operands = {}
for K, w in weights.items():
with torch.device(device):
linear = RowParallelLinear(
K * world_size,
_N,
bias=False,
params_dtype=torch.bfloat16,
quant_config=quant_config,
)
# Select the fused path before online quantization replaces meta weights.
assert gemm_rs_ar.can_run(linear)
linear.weight = torch.nn.Parameter(w.clone(), requires_grad=False)
linear.quant_method.process_weights_after_loading(linear)
projections[K] = linear
weight = linear.weight
if backend == "flashinfer_cutedsl":
weight = weight.t()
operands[K] = (weight, getattr(linear, "weight_scale", None))
input_generator = torch.Generator(device=device)
# Alternating shapes exercise producer-flag reuse across different grids,
# CTA-group choices, and both BN=128 and BN=256 dispatches.
# Alternating shapes exercise producer-flag reuse across different grids
# and CTA-group choices.
for M, K in (*_SHAPES, *_SHAPES[::-1]):
input_generator.manual_seed(2000 + M + K)
x = torch.randn(
@@ -107,8 +160,8 @@ def _run_mode(
device=device,
generator=input_generator,
)
expected = _reference(x, weights[K], world_size, group, all_reduce)
actual = gemm_rs_ar(x, weights[K])
expected = _reference(x, *operands[K], world_size, group, all_reduce)
actual = gemm_rs_ar.apply(x, projections[K])
torch.accelerator.synchronize(device)
_assert_valid_rows_close(actual, expected, M, rank, all_reduce)
@@ -123,9 +176,9 @@ def _run_mode(
device=device,
generator=input_generator,
)
first_output = gemm_rs_ar(lifetime_x, weights[lifetime_K])
first_output = gemm_rs_ar.apply(lifetime_x, projections[lifetime_K])
first_snapshot = first_output.clone()
second_output = gemm_rs_ar(-lifetime_x, weights[lifetime_K])
second_output = gemm_rs_ar.apply(-lifetime_x, projections[lifetime_K])
torch.accelerator.synchronize(device)
assert first_output.data_ptr() != second_output.data_ptr()
torch.testing.assert_close(first_output, first_snapshot, rtol=0, atol=0)
@@ -141,7 +194,7 @@ def _run_mode(
)
graph_expected = _reference(
graph_x,
weights[graph_K],
*operands[graph_K],
world_size,
group,
all_reduce,
@@ -152,13 +205,13 @@ def _run_mode(
capture_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(capture_stream):
for _ in range(3):
graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K]))
graph_output.copy_(gemm_rs_ar.apply(graph_x, projections[graph_K]))
capture_stream.synchronize()
dist.barrier(group=group)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=capture_stream):
graph_output.copy_(gemm_rs_ar(graph_x, weights[graph_K]))
graph_output.copy_(gemm_rs_ar.apply(graph_x, projections[graph_K]))
torch.cuda.current_stream().wait_stream(capture_stream)
dist.barrier(group=group)
@@ -177,7 +230,7 @@ def _run_mode(
del gemm_rs_ar, capture_stream, graph, graph_output
def _worker(local_rank: int, world_size: int, master_port: int) -> None:
def _worker(local_rank: int, world_size: int, master_port: int, backend: str) -> None:
# This module pulls in cute_dsl, which is unavailable off CUDA, so importing
# it at module level would fail collection. Import it where it is used, as
# `kda.py` does.
@@ -195,7 +248,10 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None:
)
init_distributed_environment()
with ensure_current_vllm_config():
config = VllmConfig()
config.model_config = ModelConfig(dtype="bfloat16")
config.kernel_config.linear_backend = "auto" if backend == "bf16" else backend
with set_current_vllm_config(config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
group = dist.group.WORLD
@@ -215,24 +271,29 @@ def _worker(local_rank: int, world_size: int, master_port: int) -> None:
# Production binds one mode per worker; exercise both mode-bound instances
# sequentially in this test without implying that both are initialized.
for all_reduce in (False, True):
_run_mode(
all_reduce=all_reduce,
device=device,
group=group,
rank=rank,
world_size=world_size,
weights=weights,
)
with set_current_vllm_config(config):
for all_reduce in (False, True):
_run_mode(
all_reduce=all_reduce,
device=device,
group=group,
rank=rank,
world_size=world_size,
weights=weights,
backend=backend,
)
cleanup_dist_env_and_memory()
@pytest.mark.parametrize(
"backend", ["bf16", "flashinfer_cutlass", "flashinfer_cutedsl"]
)
@pytest.mark.distributed(num_gpus=2)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Kimi-K3 GEMM-RS/AR requires SM100",
reason="GEMM-RS/AR requires SM100",
)
def test_kimi_k3_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch) -> None:
def test_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch, backend: str) -> None:
world_size = 2
if torch.accelerator.device_count() < world_size:
pytest.skip("GEMM-RS/AR requires two GPUs")
@@ -242,7 +303,7 @@ def test_kimi_k3_gemm_rs_ar(monkeypatch: pytest.MonkeyPatch) -> None:
try:
mp.spawn(
_worker,
args=(world_size, get_open_port()),
args=(world_size, get_open_port(), backend),
nprocs=world_size,
)
finally: