diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 5fd0af69c1..acc1bafa7f 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -93,6 +93,24 @@ steps: commands: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -s -v tests/compile/passes/distributed + mirror: + amd: + label: ':amd: (MI300) Distributed Compile' + dind: false + device: mi300_2 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers + - tests/compile/passes/distributed/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - pytest -s -v tests/compile/passes/distributed - label: ":nvidia: (B200) Fusion and Compile" key: fusion-and-compile-unit-tests-2xb200 @@ -463,3 +481,45 @@ steps: - amd-smi - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_CUSTOM_AR=1 pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k 'test_tp2_ar_rms_fp8_fusions and qwen3 and ROCM_AITER_UNIFIED_ATTN and +quant_fp8 and (dynamo_partition or not +rms_norm)' - pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_moe.py + +- label: ':amd: (MI355) Fusion E2E TP2 AR-RMS (Inductor Partition)' + key: fusion-e2e-tp2-ar-rms-inductor-partition-amd + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - csrc/quantization/ + - vllm/v1/attention/ + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "inductor_partition" + +- label: ':amd: (MI355) Fusion E2E TP2 AR-RMS (Dynamo Partition)' + key: fusion-e2e-tp2-ar-rms-dynamo-partition-amd + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers + - tests/compile/fusions_e2e/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - csrc/quantization/ + - vllm/v1/attention/ + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions -k "dynamo_partition" diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 7138e1e065..3710f80bfd 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -17,6 +17,24 @@ steps: - pytest -v -s distributed/test_shm_broadcast.py - pytest -v -s distributed/test_shm_buffer.py - pytest -v -s distributed/test_shm_storage.py + mirror: + amd: + label: ':amd: (MI300) Distributed Comm Ops' + dind: false + device: mi300_2 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed + - tests/distributed + - vllm/platforms/rocm.py + commands: + - pytest -v -s distributed/test_comm_ops.py + - pytest -v -s distributed/test_shm_broadcast.py + - pytest -v -s distributed/test_shm_buffer.py + - pytest -v -s distributed/test_shm_storage.py - label: ":nvidia: (L4) Sharded RDT Weight Transfer" key: sharded-rdt-weight-transfer @@ -176,6 +194,31 @@ steps: - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown - pytest -v -s v1/worker/test_worker_memory_snapshot.py + mirror: + amd: + label: ':amd: (MI300) Distributed Torchrun + Shutdown' + dind: false + device: mi300_2 + timeout_in_minutes: 35 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - vllm/engine/ + - vllm/v1/executor/ + - vllm/v1/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/distributed/ + - tests/v1/shutdown + - tests/v1/worker/test_worker_memory_snapshot.py + - vllm/platforms/rocm.py + commands: + - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' + - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' + - HIP_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown + - pytest -v -s v1/worker/test_worker_memory_snapshot.py - label: ":nvidia: (L4) Distributed Torchrun + Examples" key: distributed-torchrun-examples-4-gpus @@ -282,6 +325,31 @@ steps: - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp - pytest -v -s distributed/test_utils.py - pytest -v -s distributed/test_engram_dp_shard.py + mirror: + amd: + label: ':amd: (MI355) Distributed DP Extended' + dind: false + device: mi355_4 + timeout_in_minutes: 60 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - tests/v1/distributed + - tests/v1/engine/test_engine_core_client.py + - tests/distributed/test_utils + - vllm/platforms/rocm.py + commands: + - export NCCL_CUMEM_HOST_ENABLE=0 + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py + - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py + - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py + - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp + - pytest -v -s distributed/test_utils.py - label: ":nvidia: (L4) Distributed Compile + Comm" key: distributed-compile-comm-4-gpus @@ -304,6 +372,27 @@ steps: - pytest -v -s distributed/test_symm_mem_allreduce.py # test multi-node TP with multiproc executor (simulated on single node) - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node + mirror: + amd: + label: ':amd: (MI355) Distributed Compile + Comm' + dind: false + device: mi355_4 + timeout_in_minutes: 45 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/distributed/ + - tests/distributed/test_pynccl + - tests/distributed/test_events + - tests/distributed/test_symm_mem_allreduce.py + - tests/distributed/test_multiproc_executor.py + - vllm/platforms/rocm.py + commands: + - pytest -v -s distributed/test_pynccl.py + - pytest -v -s distributed/test_events.py + - pytest -v -s distributed/test_symm_mem_allreduce.py + - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node - label: ":nvidia: (H100) Distributed DP + EP" key: distributed-tests-8xh100 @@ -359,6 +448,28 @@ steps: - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - pytest -v -s -x lora/test_mixtral.py + mirror: + amd: + label: ':amd: (MI355) Distributed' + dind: false + device: mi355_4 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/ + - tests/models/test_vision.py + - vllm/platforms/rocm.py + - tests/distributed/ + - tests/basic_correctness/ + - tests/lora/test_mixtral.py + commands: + - pytest -v -s distributed/test_custom_all_reduce.py + - torchrun --nproc_per_node=2 distributed/test_ca_buffer_sharing.py + - TARGET_TEST_SUITE=MI355 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + - pytest -v -s -x lora/test_mixtral.py + - pytest -v -s models/test_vision.py -m 'distributed(num_gpus=4)' - label: ":nvidia: (H100) Distributed Features" key: distributed-tests-2xh100-2xmi300 @@ -583,3 +694,27 @@ steps: - pytest -v -s distributed/test_ray_v2_executor_e2e.py - pytest -v -s distributed/test_pipeline_parallel.py -k "ray" - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray" + mirror: + amd: + label: ':amd: (MI355) RayExecutorV2' + dind: false + device: mi355_4 + timeout_in_minutes: 75 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/executor/ray_executor_v2.py + - vllm/v1/executor/abstract.py + - vllm/v1/executor/multiproc_executor.py + - tests/distributed/test_ray_v2_executor.py + - tests/distributed/test_ray_v2_executor_e2e.py + - tests/distributed/test_pipeline_parallel.py + - tests/basic_correctness/models/test_basic_correctness.py + - vllm/platforms/rocm.py + commands: + - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 + - pytest -v -s distributed/test_ray_v2_executor.py + - pytest -v -s distributed/test_ray_v2_executor_e2e.py + - pytest -v -s distributed/test_pipeline_parallel.py -k "ray" + - TARGET_TEST_SUITE=MI355 pytest -v -s basic_correctness/models/test_basic_correctness.py -k "ray" diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 9af5423a4b..b211c47e35 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -150,3 +150,27 @@ steps: - .buildkite/scripts/scheduled_integration_test/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py + +- label: ":amd: (MI355) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy" + key: amd-qwen3-next-mtp-async-eplb-accuracy + dind: false + device: mi355_4 + timeout_in_minutes: 45 + depends_on: + - image-build-amd + working_dir: /vllm-workspace + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/spec_decode/ + - vllm/distributed/eplb + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - .buildkite/scripts/scheduled_integration_test/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + num_devices: 4 + commands: + - VLLM_ENGINE_READY_TIMEOUT_S=1800 bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040 diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index a23fcd0a98..04337d03d2 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -753,6 +753,32 @@ steps: parallelism: 4 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200-shard-$$BUILDKITE_PARALLEL_JOB.txt + mirror: + amd: + label: ":amd: (MI355) MoE Refactor BF16 Integration" + dind: false + device: mi355_2 + num_devices: 2 + parallelism: 1 + timeout_in_minutes: 105 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/kernels/linear/ + - vllm/v1/attention/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + env: + VLLM_WORKER_MULTIPROC_METHOD: "spawn" + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200.txt -k "Llama-4-Scout-BF16-triton or Mixtral-8x7B-BF16-triton" - label: ":nvidia: (B200) MoE Refactor DP Integration TEMPORARY" key: moe-refactor-integration-test-b200-dp-temporary @@ -761,6 +787,32 @@ steps: num_devices: 2 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt + mirror: + amd: + label: ":amd: (MI355) MoE Refactor BF16 DP Integration" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 105 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/config/parallel.py + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/kernels/linear/ + - vllm/v1/attention/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + env: + VLLM_WORKER_MULTIPROC_METHOD: "spawn" + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt -k "Qwen3-30B-A3B-BF16-triton" - label: ":nvidia: (L4) LM Eval Humming TEMPORARY" key: lm-eval-humming-l4 @@ -1068,6 +1120,29 @@ steps: - tests/evals/gpt_oss/ commands: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt + mirror: + amd: + label: ":amd: (MI300) GPQA Eval (GPT-OSS)" + dind: false + device: mi300_2 + num_devices: 2 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/model_executor/layers/fused_moe/ + - tests/evals/gpt_oss/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt - label: ":nvidia: (B200) GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-2xb200 @@ -1176,6 +1251,28 @@ steps: - tests/evals/gpt_oss/ commands: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt + mirror: + amd: + label: ":amd: (MI355 DPX) GPQA Eval (GPT-OSS TP1)" + # The Spark config is a backend-neutral GPT-OSS TP1 baseline. + dind: false + device: mi355_dpx + num_devices: 1 + timeout_in_minutes: 40 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/model_executor/layers/fused_moe/ + - tests/evals/gpt_oss/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload" key: kv-offload-small @@ -1292,3 +1389,53 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py - tests/evals/mrcr/ + +- label: ":amd: (MI355) LM Eval Large Models Harness" + key: amd-lm-eval-large-models-harness + dind: false + device: mi355_8 + optional: true + timeout_in_minutes: 85 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/.buildkite/lm-eval-harness + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - .buildkite/lm-eval-harness/ + num_devices: 8 + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + # Qwen2-57B has 28 attention heads and cannot run at TP8. + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4 + +- label: ":amd: (MI300) LM Eval Large Models ROCm Harness" + key: amd-lm-eval-large-models-rocm-harness + dind: false + device: mi300_8 + optional: true + timeout_in_minutes: 40 + depends_on: + - image-build-amd + working_dir: /vllm-workspace/.buildkite/lm-eval-harness + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - .buildkite/lm-eval-harness/ + num_devices: 8 + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8 --timeout=5400 diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 5b6260b534..0ba608e725 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -259,6 +259,25 @@ steps: - tests/compile commands: - pytest -s -v compile/fullgraph/test_full_cudagraph.py + mirror: + amd: + label: ':amd: (MI250) PyTorch Fullgraph CUDAGraph Compatibility' + dind: false + device: mi250_1 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/ + - vllm/v1/attention/ + - vllm/config/compilation.py + - csrc/ + - tests/compile + - vllm/platforms/rocm.py + commands: + - pytest -s -v compile/fullgraph/test_full_cudagraph.py - label: ":nvidia: (H200 MIG 18GB) PyTorch Nightly Dependency Override Check" key: pytorch-nightly-dependency-override-check diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index af7e8f071a..6882c24b37 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -97,6 +97,27 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode/eagle/ + mirror: + amd: + label: ":amd: (MI355) Spec Decode Eagle Nightly" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 120 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - tests/v1/e2e/spec_decode/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py # MTP correctness is one parametrized test split into per-model folders; # each job below runs whole directories. @@ -234,6 +255,28 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode/speculators/ - pytest -v -s v1/e2e/spec_decode/mtp/ + mirror: + amd: + label: ":amd: (MI355) Spec Decode Speculators + MTP Nightly" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 90 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Spec Decode N-Gram + Suffix" key: spec-decode-ngram-suffix @@ -304,6 +347,27 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode/draft_model/ + mirror: + amd: + label: ":amd: (MI355) Spec Decode Draft Model Nightly" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 120 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - tests/v1/e2e/spec_decode/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) Speculators Correctness Nightly" key: speculators-correctness