diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index fe4e4fd9f2..55dde1ec1c 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1886,7 +1886,7 @@ steps: - vllm/config/ - tests/kernels/quantization/test_rocm_skinny_gemms.py commands: - - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI300) DeepEP FP8 MoE Kernels" # TBD timeout_in_minutes: 55 @@ -4307,6 +4307,35 @@ steps: commands: - pytest -v -s kernels/moe/test_moe_layer.py +- label: ":amd: (MI355) MoRI EP Numerics" + timeout_in_minutes: 20 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/kernels/moe/test_mori_moe.py + - tests/kernels/moe/utils.py + - tests/kernels/moe/modular_kernel_tools/parallel_utils.py + - tests/kernels/utils.py + - tests/utils.py + - tests/conftest.py + - vllm/utils/torch_utils.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/platforms/ + - vllm/_aiter_ops.py + - requirements/rocm-test.txt + optional: true + commands: + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py + - label: ":amd: (MI355) Quantization Kernels Shard %N" timeout_in_minutes: 120 parallelism: 6 @@ -4332,7 +4361,30 @@ steps: - tests/kernels/quantization/test_rocm_skinny_gemms.py optional: true commands: - - pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + +- label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N" + timeout_in_minutes: 10 + parallelism: 3 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/kernels/quantization/test_nvfp4_emulation.py + - tests/kernels/quantization/nvfp4_utils.py + - tests/conftest.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py + - vllm/config/kernel.py + - vllm/platforms/ + - vllm/envs.py + - vllm/utils/torch_utils.py + - requirements/rocm-test.txt + optional: true + commands: + - amd-smi + - pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":amd: (MI355) FP8 MoE Kernels" # TBD timeout_in_minutes: 40 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 78d57ca455..37af3e9643 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -330,6 +330,8 @@ steps: - vllm/envs.py depends_on: - image-build-amd + commands: + - pytest -v -s kernels/quantization -k "not test_nvfp4_dense_emulation" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (L4) MoE Kernels Shard %N" key: kernels-moe-test @@ -522,6 +524,32 @@ steps: exit 1 ;; esac + mirror: + amd: + label: ":amd: (MI355) NVFP4 Dense Emulation Shard %N" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 10 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/kernels/quantization/test_nvfp4_emulation.py + - tests/kernels/quantization/nvfp4_utils.py + - tests/conftest.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py + - vllm/config/kernel.py + - vllm/platforms/ + - vllm/envs.py + - vllm/utils/torch_utils.py + - requirements/rocm-test.txt + # W4A4 format numerics and replay via emulation. TODO: native packed + # activation/KV-cache layouts and CUDA-only kernels need AMD backends. + commands: + - amd-smi + - pytest -v -s kernels/quantization/test_nvfp4_emulation.py -k test_nvfp4_dense_emulation --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: ":nvidia: (DGX) Spark b12x Linear Kernels Nightly" key: b12x-linear-kernels-dgx-spark-nightly @@ -701,6 +729,38 @@ steps: commands: - pytest -v -s kernels/moe/test_moe_layer.py - pytest -v -s kernels/moe/test_deepep_v2_moe.py + mirror: + amd: + label: ":amd: (MI355) MoRI EP Numerics" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 20 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/kernels/moe/test_mori_moe.py + - tests/kernels/moe/utils.py + - tests/kernels/moe/modular_kernel_tools/parallel_utils.py + - tests/kernels/utils.py + - tests/utils.py + - tests/conftest.py + - vllm/utils/torch_utils.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/distributed/ + - vllm/config/ + - vllm/forward_context.py + - vllm/v1/worker/workspace.py + - vllm/platforms/ + - vllm/_aiter_ops.py + - requirements/rocm-test.txt + # Same-node MoRI/AITER BF16 and per-token/channel FP8. TODO: the + # DeepEP v2 FlashInfer backend matrix has no direct AMD implementation. + commands: + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 pytest -v -s kernels/moe/test_mori_moe.py - label: ":amd: (MI355) FP8 MoE Kernels" key: amd-fp8-moe-kernels-mi355 diff --git a/tests/kernels/moe/test_mori_moe.py b/tests/kernels/moe/test_mori_moe.py new file mode 100644 index 0000000000..9869b6bc54 --- /dev/null +++ b/tests/kernels/moe/test_mori_moe.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Two-rank MoRI/AITER numerical counterpart to the DeepEP v2 MoE matrix.""" + +from dataclasses import replace + +import pytest +import torch + +from tests.kernels.moe.modular_kernel_tools.parallel_utils import ( + parallel_launch_with_config, +) +from tests.kernels.moe.utils import make_dummy_moe_config, make_test_weights +from tests.kernels.utils import torch_experts +from tests.utils import multi_gpu_test +from vllm.config import ParallelConfig, SchedulerConfig, VllmConfig +from vllm.forward_context import set_forward_context +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_mori +from vllm.utils.torch_utils import set_random_seed +from vllm.v1.worker.workspace import init_workspace_manager + +MNKS = [ + (1, 256, 256), + (2, 256, 512), + (3, 1024, 2048), + (32, 256, 1024), + (45, 512, 2048), + (64, 1024, 1024), + (222, 1024, 2048), +] + + +def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph): + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts + from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import ( + MoriPrepareAndFinalize, + ) + + rocm_aiter_ops.refresh_env_variables() + init_workspace_manager(pgi.device) + set_random_seed(47) + dtype = current_platform.fp8_dtype() if fp8 else None + (_, w1, s1, _), (_, w2, s2, _) = make_test_weights( + 32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8 + ) + # Share global expert weights, but give each rank different token inputs. + set_random_seed(101 + pgi.rank) + x = torch.randn(m, k, device=pgi.device, dtype=torch.bfloat16) / 10 + routing = torch.randn(m, 32, device=pgi.device) + # Guarantee a route to both ranks, including the one-token case. + routing[:, 0] = routing[:, 16] = float("inf") + ids = routing.topk(6, dim=-1).indices.to(torch.int32) + weights = torch.randn(m, 6, device=pgi.device, dtype=torch.float32) + parallel = replace( + FusedMoEParallelConfig.make_no_parallel(), + dp_size=2, + dp_rank=pgi.rank, + ep_size=2, + ep_rank=pgi.rank, + use_ep=True, + all2all_backend="mori_high_throughput", + ) + config = replace( + make_dummy_moe_config( + num_experts=32, + num_local_experts=16, + experts_per_token=6, + hidden_dim=k, + intermediate_size=n, + max_num_tokens=256, + ), + moe_parallel_config=parallel, + ) + start, end = pgi.rank * 16, (pgi.rank + 1) * 16 + quant = FusedMoEQuantConfig.make( + dtype, + per_act_token_quant=fp8, + per_out_ch_quant=fp8, + w1_scale=None if s1 is None else s1[start:end].contiguous(), + w2_scale=None if s2 is None else s2[start:end].contiguous(), + ) + prepare = maybe_make_prepare_finalize(config, quant) + assert isinstance(prepare, MoriPrepareAndFinalize) + assert prepare.use_fp8_dispatch == fp8 + assert prepare.num_dispatchers() == 2 + experts = AiterExperts(config, quant) + assert not experts.expects_unquantized_inputs + kernel = FusedMoEKernel(prepare, experts) + local_w1, local_w2 = rocm_aiter_ops.shuffle_weights( + w1[start:end].contiguous(), w2[start:end].contiguous() + ) + local_w1.is_shuffled = local_w2.is_shuffled = True + expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device) + expert_mask[start:end] = 1 + + def reference(route_weights): + return torch_experts( + x, + w1, + w2, + route_weights, + ids, + w1_scale=s1, + w2_scale=s2, + quant_dtype=dtype, + per_act_token_quant=fp8, + ) + + def run(): + with set_forward_context(None, vllm_config): + return kernel.apply( + x, + local_w1, + local_w2, + weights, + ids, + activation=MoEActivation.SILU, + global_num_experts=32, + expert_map=expert_mask, + apply_router_weight_on_input=False, + ) + + def check(actual): + expected = reference(weights) + assert actual.shape == expected.shape + assert actual.dtype == expected.dtype + local_weights = weights * ((ids >= start) & (ids < end)) + missing_remote = reference(local_weights) + norm = expected.float().norm() + assert norm > 0 + norm_budget = 0.2 if fp8 else 0.06 + assert (actual.float() - expected.float()).norm() / norm < norm_budget + # The norm check also rejects low-signal zero or rank-local results. + for incorrect in (torch.zeros_like(expected), missing_remote): + assert (incorrect.float() - expected.float()).norm() / norm > norm_budget + if fp8: + assert ( + torch.isclose(actual, expected, atol=0.2, rtol=0.2) + .float() + .mean() + .item() + > 0.99 + ) + else: + torch.testing.assert_close(actual, expected, atol=0.06, rtol=0.06) + assert torch.isfinite(actual).all() + + actual = run() + check(actual) + if graph: + saved = actual.clone() + run() + capture = torch.cuda.CUDAGraph() + with torch.cuda.graph(capture): + captured = run() + for factor in (-0.5, 1.5): + x.mul_(factor) + weights.mul_(-0.75) + capture.replay() + torch.accelerator.synchronize() + check(captured) + torch.testing.assert_close(actual, saved, atol=0, rtol=0) + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI" +) +@pytest.mark.parametrize("m,n,k", MNKS) +@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"]) +@multi_gpu_test(num_gpus=2) +def test_mori_moe(m, n, k, fp8): + config = VllmConfig( + parallel_config=ParallelConfig( + data_parallel_size=2, + enable_expert_parallel=True, + all2all_backend="mori_high_throughput", + ), + scheduler_config=SchedulerConfig.default_factory(max_num_batched_tokens=256), + ) + env = { + "VLLM_ROCM_USE_AITER": "1", + "VLLM_ROCM_USE_AITER_MOE": "1", + "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", + } + parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32) diff --git a/tests/kernels/quantization/test_nvfp4_emulation.py b/tests/kernels/quantization/test_nvfp4_emulation.py index 33295b8784..3a136484ac 100644 --- a/tests/kernels/quantization/test_nvfp4_emulation.py +++ b/tests/kernels/quantization/test_nvfp4_emulation.py @@ -4,9 +4,21 @@ from typing import cast import pytest import torch +from nvfp4_utils import ( + FLOAT4_E2M1_MAX, + FLOAT8_E4M3_MAX, + break_fp4_bytes, + kE2M1ToFloat, +) from safetensors import safe_open import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.config.kernel import KernelConfig +from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel +from vllm.model_executor.kernels.linear.nvfp4.emulation import ( + EmulationNvFp4LinearKernel, +) from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -34,6 +46,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api from vllm.triton_utils import triton +from vllm.utils.torch_utils import set_random_seed if current_platform.is_rocm(): from vllm.platforms.rocm import on_gfx950 @@ -769,3 +782,85 @@ def test_nvfp4_moe_correctness( atol=0.0 if on_gfx950() else 0.02, rtol=0, ) + + +def _dense_nvfp4_reference( + layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None +) -> torch.Tensor: + """Decode E2M1 nibbles and both scale levels without production helpers.""" + levels = kE2M1ToFloat.cpu() + weight = break_fp4_bytes(layer.weight.cpu(), torch.float32) + weight *= ( + layer.weight_scale.cpu().float() * layer.weight_global_scale.cpu() + ).repeat_interleave(16, dim=-1) + blocks = x.cpu().float().reshape(x.shape[0], -1, 16) + global_scale = layer.input_global_scale_inv.cpu() + scales = ( + (blocks.abs().amax(-1, keepdim=True) * (1.0 / FLOAT4_E2M1_MAX) * global_scale) + .clamp(max=FLOAT8_E4M3_MAX) + .to(torch.float8_e4m3fn) + .float() + ) + inverse = torch.where(scales == 0, 0, global_scale / scales) + scaled = (blocks * inverse).abs().clamp(max=FLOAT4_E2M1_MAX) + distances = (scaled.unsqueeze(-1) - levels).abs() + # E2M1 ties round to the even significand (even encoding). + nearest_mask = distances == distances.amin(-1, keepdim=True) + even_mask = nearest_mask & (torch.arange(8, device="cpu") % 2 == 0) + candidates = torch.where(even_mask.any(-1, keepdim=True), even_mask, nearest_mask) + nearest = candidates.to(torch.int8).argmax(-1) + activation = levels[nearest] * blocks.sign() * (scales / global_scale) + expected = activation.reshape_as(x).to(x.dtype) @ weight.to(x.dtype).T + if bias is not None: + expected += bias.cpu() + return expected + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm NVFP4 fallback") +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("m,n,k", [(1, 32, 64), (17, 128, 256)]) +@pytest.mark.parametrize("with_bias", [False, True]) +def test_nvfp4_dense_emulation( + dtype: torch.dtype, m: int, n: int, k: int, with_bias: bool +) -> None: + """W4A4 fallback preserves packed scales, bias and outputs across replay.""" + set_random_seed(47) + layer = torch.nn.Module() + # Every signed E2M1 code appears, including both zero encodings. + codes = (torch.randperm(n * k, device="cuda") % 16).reshape(n, k) + layer.weight = (codes[:, ::2] | (codes[:, 1::2] << 4)).to(torch.uint8) + layer.weight_scale = ( + torch.arange(n * k // 16, device="cuda").reshape(n, k // 16) % 7 / 8 + ).to(torch.float8_e4m3fn) + layer.weight_global_scale = torch.tensor(0.25, device="cuda") + layer.input_global_scale_inv = torch.tensor(3.0, device="cuda") + x = torch.randn(m, k, device="cuda", dtype=dtype) + x[:, :16] = 0 + bias = torch.randn(n, device="cuda", dtype=dtype) if with_bias else None + with set_current_vllm_config( + VllmConfig(kernel_config=KernelConfig(linear_backend="emulation")) + ): + kernel = init_nvfp4_linear_kernel() + assert isinstance(kernel, EmulationNvFp4LinearKernel) + kernel.process_weights_after_loading(layer) + original_weight = layer.weight.clone() + original_scale = layer.weight_scale.view(torch.uint8).clone() + actual = kernel.apply_weights(layer, x, bias) + torch.testing.assert_close( + actual.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01 + ) + assert actual.abs().max() > 0.1 + saved = actual.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = kernel.apply_weights(layer, x, bias) + for sign in (-1, 1): + x.mul_(sign * 0.75) + graph.replay() + torch.accelerator.synchronize() + torch.testing.assert_close( + captured.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01 + ) + torch.testing.assert_close(actual, saved, atol=0, rtol=0) + assert torch.equal(layer.weight, original_weight) + assert torch.equal(layer.weight_scale.view(torch.uint8), original_scale)