diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 8183c16e1e..825e9968bb 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2579,7 +2579,7 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.18.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi #--------------------------------------------------------- mi300 ยท samplers ----------------------------------------------------------# @@ -4943,7 +4943,7 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi # - label: Quantized MoE Test (B200-MI355) # TBD @@ -5445,6 +5445,36 @@ steps: - pip install 'modelscope<1.38' - pytest -v -s test_regression.py +- label: ':amd: (MI355) Quantized MoE' + timeout_in_minutes: 45 + working_dir: /vllm-workspace/ + source_file_dependencies: + - tests/quantization/test_rocm_moe.py + - tests/utils.py + - vllm/model_executor/models/deepseek_v2.py + - vllm/model_executor/models/qwen3_moe.py + - vllm/model_executor/models/gpt_oss.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/v1/attention/ + - vllm/v1/worker/gpu/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s tests/quantization/test_rocm_moe.py + dind: false + agent_pool: mi355_1 + num_gpus: 1 + mirror_hardwares: + - amdexperimental + - amdproduction + - amdgfx950nightly + - amdmi355 + optional: true + - label: ':amd: (MI355) Fusion E2E TP2 AsyncTP BF16 Config Sweep' timeout_in_minutes: 30 working_dir: /vllm-workspace/ diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index 702bbe7024..f9a4e86e6e 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -19,7 +19,7 @@ steps: - uv pip install --system conch-triton-kernels # The SM90-only checkpoint currently contains a removed weight_chan_scale # parameter. It was not exercised by the previous L4 job. - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT mirror: amd: label: ":amd: (MI355) Quantization Shard %N" @@ -43,7 +43,7 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi - label: ":nvidia: (H200 MIG 35GB) Quantized Fusions" @@ -76,6 +76,32 @@ steps: - vllm/v1/attention/backends/flashinfer.py commands: - pytest -s -v tests/quantization/test_blackwell_moe.py + mirror: + amd: + label: ":amd: (MI355) Quantized MoE" + dind: false + device: mi355_1 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - tests/quantization/test_rocm_moe.py + - tests/utils.py + - vllm/model_executor/models/deepseek_v2.py + - vllm/model_executor/models/qwen3_moe.py + - vllm/model_executor/models/gpt_oss.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/v1/attention/ + - vllm/v1/worker/gpu/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + commands: + - pytest -v -s tests/quantization/test_rocm_moe.py - label: ":nvidia: (H200 MIG 35GB) Quantized Models" device: h200_35gb diff --git a/tests/quantization/test_rocm_moe.py b/tests/quantization/test_rocm_moe.py new file mode 100644 index 0000000000..f6f75b0c0e --- /dev/null +++ b/tests/quantization/test_rocm_moe.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm equivalents of the quantized-MoE model startup/generation contract. + +Like test_blackwell_moe.py, these tests use reduced models and dummy weights. +They validate model/backend integration, not trained-model accuracy. +""" + +import math + +import pytest + +from tests.utils import RemoteOpenAIServer +from vllm.platforms import current_platform +from vllm.platforms.rocm import on_gfx950 + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm() or not on_gfx950(), + reason="ROCm FP8/MXFP4 model integration requires gfx950", +) + +COMMON_OVERRIDES = { + "num_hidden_layers": 4, + "hidden_size": 512, + "intermediate_size": 1024, + "num_attention_heads": 8, + "num_key_value_heads": 2, +} + +CASES = [ + pytest.param( + "RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8", + "efe1ced428db63e7ccbcf367334596f77e9af140", + "triton", + "TRITON_MLA", + { + "n_routed_experts": 8, + "n_shared_experts": 1, + "num_experts_per_tok": 2, + "moe_intermediate_size": 512, + }, + ["--code-revision", "e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11"], + id="deepseek-fp8-per-tensor-triton", + ), + pytest.param( + "Qwen/Qwen3-30B-A3B-FP8", + "d206ba732169f29bb77fbf80fc2c4b81d4d30782", + "triton", + "TRITON_ATTN", + { + "num_experts": 8, + "num_experts_per_tok": 2, + "moe_intermediate_size": 512, + "head_dim": 64, + }, + [], + id="qwen3-fp8-block-triton", + ), + pytest.param( + "openai/gpt-oss-20b", + "6cee5e81ee83917806bbde320786a8fb61efebee", + "aiter", + "TRITON_ATTN", + { + "num_local_experts": 8, + "num_experts_per_tok": 2, + "experts_per_token": 2, + "layer_types": ["sliding_attention", "full_attention"] * 2, + }, + [], + id="gptoss-mxfp4-aiter", + ), +] + + +@pytest.mark.parametrize("enforce_eager", [True, False], ids=["eager", "compiled"]) +@pytest.mark.parametrize( + "model,revision,moe_backend,attention_backend,overrides,extra_args", CASES +) +def test_rocm_quantized_moe_generation( + model, + revision, + moe_backend, + attention_backend, + overrides, + extra_args, + enforce_eager, +): + """Quantized MoE models warm up and return finite logits through serving.""" + args = [ + "--revision", + revision, + "--load-format", + "dummy", + "--trust-remote-code", + "--dtype", + "bfloat16", + "--max-model-len", + "2048", + "--max-num-batched-tokens", + "256", + "--max-num-seqs", + "8", + "--gpu-memory-utilization", + "0.02", + "--kv-cache-memory-bytes", + str(256 * 1024 * 1024), + "--moe-backend", + moe_backend, + "--attention-backend", + attention_backend, + *extra_args, + ] + if enforce_eager: + args.append("--enforce-eager") + + with RemoteOpenAIServer( + model, + args, + env_dict={"VLLM_ROCM_USE_AITER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1"}, + override_hf_configs={**COMMON_OVERRIDES, **overrides}, + max_wait_seconds=1200, + ) as server: + prompts = [[1, 2, 3], [1, 2, 3, 4] * 4 + [5]] + completion = server.get_client().completions.create( + model=model, + prompt=prompts, + temperature=0, + max_tokens=2, + logprobs=1, + extra_body={"ignore_eos": True, "skip_special_tokens": False}, + ) + assert len(completion.choices) == len(prompts) + assert completion.usage is not None + assert completion.usage.prompt_tokens == sum(map(len, prompts)) + assert completion.usage.completion_tokens == 2 * len(prompts) + for choice in completion.choices: + assert choice.finish_reason == "length" + assert choice.text + assert choice.logprobs is not None + assert choice.logprobs.token_logprobs is not None + assert len(choice.logprobs.token_logprobs) == 2 + assert all( + p is not None and math.isfinite(p) + for p in choice.logprobs.token_logprobs + )