[ROCm][CI] Add MI355 dense NVFP4 and MoRI kernel mirrors (#58281)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-23 03:06:15 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent a9f07d0bc5
commit 153242a314
4 changed files with 405 additions and 2 deletions
+54 -2
View File
@@ -1886,7 +1886,7 @@ steps:
- vllm/config/
- tests/kernels/quantization/test_rocm_skinny_gemms.py
commands:
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD
timeout_in_minutes: 55
@@ -4307,6 +4307,35 @@ steps:
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
- label: ":amd: (MI355) MoRI EP Numerics"
timeout_in_minutes: 20
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_2
num_gpus: 2
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- tests/kernels/moe/test_mori_moe.py
- tests/kernels/moe/utils.py
- tests/kernels/moe/modular_kernel_tools/parallel_utils.py
- tests/kernels/utils.py
- tests/utils.py
- tests/conftest.py
- vllm/utils/torch_utils.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/platforms/
- vllm/_aiter_ops.py
- requirements/rocm-test.txt
optional: true
commands:
- amd-smi
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py
- label: ":amd: (MI355) Quantization Kernels Shard %N"
timeout_in_minutes: 120
parallelism: 6
@@ -4332,7 +4361,30 @@ steps:
- tests/kernels/quantization/test_rocm_skinny_gemms.py
optional: true
commands:
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N"
timeout_in_minutes: 10
parallelism: 3
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- tests/kernels/quantization/test_nvfp4_emulation.py
- tests/kernels/quantization/nvfp4_utils.py
- tests/conftest.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py
- vllm/config/kernel.py
- vllm/platforms/
- vllm/envs.py
- vllm/utils/torch_utils.py
- requirements/rocm-test.txt
optional: true
commands:
- amd-smi
- pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":amd: (MI355) FP8 MoE Kernels" # TBD
timeout_in_minutes: 40
+60
View File
@@ -330,6 +330,8 @@ steps:
- vllm/envs.py
depends_on:
- image-build-amd
commands:
- pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (L4) MoE Kernels Shard %N"
key: kernels-moe-test
@@ -522,6 +524,32 @@ steps:
exit 1
;;
esac
mirror:
amd:
label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N"
dind: false
device: mi355_1
num_devices: 1
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/kernels/quantization/test_nvfp4_emulation.py
- tests/kernels/quantization/nvfp4_utils.py
- tests/conftest.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py
- vllm/config/kernel.py
- vllm/platforms/
- vllm/envs.py
- vllm/utils/torch_utils.py
- requirements/rocm-test.txt
# W4A4 format numerics and replay via emulation. TODO: native packed
# activation/KV-cache layouts and CUDA-only kernels need AMD backends.
commands:
- amd-smi
- pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- label: ":nvidia: (DGX) Spark b12x Linear Kernels Nightly"
key: b12x-linear-kernels-dgx-spark-nightly
@@ -701,6 +729,38 @@ steps:
commands:
- pytest -v -s kernels/moe/test_moe_layer.py
- pytest -v -s kernels/moe/test_deepep_v2_moe.py
mirror:
amd:
label: ":amd: (MI355) MoRI EP Numerics"
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 20
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- tests/kernels/moe/test_mori_moe.py
- tests/kernels/moe/utils.py
- tests/kernels/moe/modular_kernel_tools/parallel_utils.py
- tests/kernels/utils.py
- tests/utils.py
- tests/conftest.py
- vllm/utils/torch_utils.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/platforms/
- vllm/_aiter_ops.py
- requirements/rocm-test.txt
# Same-node MoRI/AITER BF16 and per-token/channel FP8. TODO: the
# DeepEP v2 FlashInfer backend matrix has no direct AMD implementation.
commands:
- amd-smi
- VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py
- label: ":amd: (MI355) FP8 MoE Kernels"
key: amd-fp8-moe-kernels-mi355
+196
View File
@@ -0,0 +1,196 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Two-rank MoRI/AITER numerical counterpart to the DeepEP v2 MoE matrix."""
from dataclasses import replace
import pytest
import torch
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
parallel_launch_with_config,
)
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights
from tests.kernels.utils import torch_experts
from tests.utils import multi_gpu_test
from vllm.config import ParallelConfig, SchedulerConfig, VllmConfig
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_mori
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
MNKS = [
(1, 256, 256),
(2, 256, 512),
(3, 1024, 2048),
(32, 256, 1024),
(45, 512, 2048),
(64, 1024, 1024),
(222, 1024, 2048),
]
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
)
rocm_aiter_ops.refresh_env_variables()
init_workspace_manager(pgi.device)
set_random_seed(47)
dtype = current_platform.fp8_dtype() if fp8 else None
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
)
# Share global expert weights, but give each rank different token inputs.
set_random_seed(101 + pgi.rank)
x = torch.randn(m, k, device=pgi.device, dtype=torch.bfloat16) / 10
routing = torch.randn(m, 32, device=pgi.device)
# Guarantee a route to both ranks, including the one-token case.
routing[:, 0] = routing[:, 16] = float("inf")
ids = routing.topk(6, dim=-1).indices.to(torch.int32)
weights = torch.randn(m, 6, device=pgi.device, dtype=torch.float32)
parallel = replace(
FusedMoEParallelConfig.make_no_parallel(),
dp_size=2,
dp_rank=pgi.rank,
ep_size=2,
ep_rank=pgi.rank,
use_ep=True,
all2all_backend="mori_high_throughput",
)
config = replace(
make_dummy_moe_config(
num_experts=32,
num_local_experts=16,
experts_per_token=6,
hidden_dim=k,
intermediate_size=n,
max_num_tokens=256,
),
moe_parallel_config=parallel,
)
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
quant = FusedMoEQuantConfig.make(
dtype,
per_act_token_quant=fp8,
per_out_ch_quant=fp8,
w1_scale=None if s1 is None else s1[start:end].contiguous(),
w2_scale=None if s2 is None else s2[start:end].contiguous(),
)
prepare = maybe_make_prepare_finalize(config, quant)
assert isinstance(prepare, MoriPrepareAndFinalize)
assert prepare.use_fp8_dispatch == fp8
assert prepare.num_dispatchers() == 2
experts = AiterExperts(config, quant)
assert not experts.expects_unquantized_inputs
kernel = FusedMoEKernel(prepare, experts)
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
w1[start:end].contiguous(), w2[start:end].contiguous()
)
local_w1.is_shuffled = local_w2.is_shuffled = True
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
expert_mask[start:end] = 1
def reference(route_weights):
return torch_experts(
x,
w1,
w2,
route_weights,
ids,
w1_scale=s1,
w2_scale=s2,
quant_dtype=dtype,
per_act_token_quant=fp8,
)
def run():
with set_forward_context(None, vllm_config):
return kernel.apply(
x,
local_w1,
local_w2,
weights,
ids,
activation=MoEActivation.SILU,
global_num_experts=32,
expert_map=expert_mask,
apply_router_weight_on_input=False,
)
def check(actual):
expected = reference(weights)
assert actual.shape == expected.shape
assert actual.dtype == expected.dtype
local_weights = weights * ((ids >= start) & (ids < end))
missing_remote = reference(local_weights)
norm = expected.float().norm()
assert norm > 0
norm_budget = 0.2 if fp8 else 0.06
assert (actual.float() - expected.float()).norm() / norm < norm_budget
# The norm check also rejects low-signal zero or rank-local results.
for incorrect in (torch.zeros_like(expected), missing_remote):
assert (incorrect.float() - expected.float()).norm() / norm > norm_budget
if fp8:
assert (
torch.isclose(actual, expected, atol=0.2, rtol=0.2)
.float()
.mean()
.item()
> 0.99
)
else:
torch.testing.assert_close(actual, expected, atol=0.06, rtol=0.06)
assert torch.isfinite(actual).all()
actual = run()
check(actual)
if graph:
saved = actual.clone()
run()
capture = torch.cuda.CUDAGraph()
with torch.cuda.graph(capture):
captured = run()
for factor in (-0.5, 1.5):
x.mul_(factor)
weights.mul_(-0.75)
capture.replay()
torch.accelerator.synchronize()
check(captured)
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
)
@pytest.mark.parametrize("m,n,k", MNKS)
@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"])
@multi_gpu_test(num_gpus=2)
def test_mori_moe(m, n, k, fp8):
config = VllmConfig(
parallel_config=ParallelConfig(
data_parallel_size=2,
enable_expert_parallel=True,
all2all_backend="mori_high_throughput",
),
scheduler_config=SchedulerConfig.default_factory(max_num_batched_tokens=256),
)
env = {
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_MOE": "1",
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
}
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32)
@@ -4,9 +4,21 @@ from typing import cast
import pytest
import torch
from nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
break_fp4_bytes,
kE2M1ToFloat,
)
from safetensors import safe_open
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import KernelConfig
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
EmulationNvFp4LinearKernel,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -34,6 +46,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.triton_utils import triton
from vllm.utils.torch_utils import set_random_seed
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
@@ -769,3 +782,85 @@ def test_nvfp4_moe_correctness(
atol=0.0 if on_gfx950() else 0.02,
rtol=0,
)
def _dense_nvfp4_reference(
layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
"""Decode E2M1 nibbles and both scale levels without production helpers."""
levels = kE2M1ToFloat.cpu()
weight = break_fp4_bytes(layer.weight.cpu(), torch.float32)
weight *= (
layer.weight_scale.cpu().float() * layer.weight_global_scale.cpu()
).repeat_interleave(16, dim=-1)
blocks = x.cpu().float().reshape(x.shape[0], -1, 16)
global_scale = layer.input_global_scale_inv.cpu()
scales = (
(blocks.abs().amax(-1, keepdim=True) * (1.0 / FLOAT4_E2M1_MAX) * global_scale)
.clamp(max=FLOAT8_E4M3_MAX)
.to(torch.float8_e4m3fn)
.float()
)
inverse = torch.where(scales == 0, 0, global_scale / scales)
scaled = (blocks * inverse).abs().clamp(max=FLOAT4_E2M1_MAX)
distances = (scaled.unsqueeze(-1) - levels).abs()
# E2M1 ties round to the even significand (even encoding).
nearest_mask = distances == distances.amin(-1, keepdim=True)
even_mask = nearest_mask & (torch.arange(8, device="cpu") % 2 == 0)
candidates = torch.where(even_mask.any(-1, keepdim=True), even_mask, nearest_mask)
nearest = candidates.to(torch.int8).argmax(-1)
activation = levels[nearest] * blocks.sign() * (scales / global_scale)
expected = activation.reshape_as(x).to(x.dtype) @ weight.to(x.dtype).T
if bias is not None:
expected += bias.cpu()
return expected
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm NVFP4 fallback")
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("m,n,k", [(1, 32, 64), (17, 128, 256)])
@pytest.mark.parametrize("with_bias", [False, True])
def test_nvfp4_dense_emulation(
dtype: torch.dtype, m: int, n: int, k: int, with_bias: bool
) -> None:
"""W4A4 fallback preserves packed scales, bias and outputs across replay."""
set_random_seed(47)
layer = torch.nn.Module()
# Every signed E2M1 code appears, including both zero encodings.
codes = (torch.randperm(n * k, device="cuda") % 16).reshape(n, k)
layer.weight = (codes[:, ::2] | (codes[:, 1::2] << 4)).to(torch.uint8)
layer.weight_scale = (
torch.arange(n * k // 16, device="cuda").reshape(n, k // 16) % 7 / 8
).to(torch.float8_e4m3fn)
layer.weight_global_scale = torch.tensor(0.25, device="cuda")
layer.input_global_scale_inv = torch.tensor(3.0, device="cuda")
x = torch.randn(m, k, device="cuda", dtype=dtype)
x[:, :16] = 0
bias = torch.randn(n, device="cuda", dtype=dtype) if with_bias else None
with set_current_vllm_config(
VllmConfig(kernel_config=KernelConfig(linear_backend="emulation"))
):
kernel = init_nvfp4_linear_kernel()
assert isinstance(kernel, EmulationNvFp4LinearKernel)
kernel.process_weights_after_loading(layer)
original_weight = layer.weight.clone()
original_scale = layer.weight_scale.view(torch.uint8).clone()
actual = kernel.apply_weights(layer, x, bias)
torch.testing.assert_close(
actual.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
)
assert actual.abs().max() > 0.1
saved = actual.clone()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = kernel.apply_weights(layer, x, bias)
for sign in (-1, 1):
x.mul_(sign * 0.75)
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(
captured.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
)
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
assert torch.equal(layer.weight, original_weight)
assert torch.equal(layer.weight_scale.view(torch.uint8), original_scale)