From 0ae480ff1a1b07a6959c3b19113739ffd2b60820 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 02:06:44 -0500 Subject: [PATCH] [CI][ROCm] Mirror large-model GSM8K evaluations on MI355 (#58683) Signed-off-by: Andreas Karatzas Co-authored-by: Codex --- .buildkite/test-amd.yaml | 165 +++++++++++++++++ .buildkite/test_areas/lm_eval.yaml | 170 ++++++++++++++++++ .../gsm8k/configs/DeepSeek-R1-DP_MI355.yaml | 17 ++ .../gsm8k/configs/DeepSeek-R1-TP_MI355.yaml | 17 ++ .../gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml | 17 ++ .../gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml | 17 ++ ...Nemotron-3-Super-120B-A12B-BF16_MI355.yaml | 15 ++ .../Nemotron-3-Super-120B-A12B-FP8_MI355.yaml | 15 ++ .../configs/models-mi355-deepseek-r1-dp.txt | 1 + .../configs/models-mi355-deepseek-r1-tp.txt | 1 + .../configs/models-mi355-deepseek-v32-dp.txt | 1 + .../configs/models-mi355-deepseek-v32-tp.txt | 1 + .../configs/models-mi355-nemotron-3-super.txt | 2 + tests/evals/gsm8k/test_gsm8k_correctness.py | 12 +- 14 files changed, 446 insertions(+), 5 deletions(-) create mode 100644 tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 16c1e7075b..e77093bb49 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -4248,6 +4248,171 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt +- label: ":amd: (MI355) LM Eval DeepSeek R1 TP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-tp.txt + +- label: ":amd: (MI355) LM Eval DeepSeek R1 DP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt + +- label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt + +- label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-dp.txt + +- label: ":amd: (MI355) LM Eval Nemotron 3 Super" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/mamba/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-nemotron-3-super.txt + #--------------------------------------------------------- mi355 ยท examples ----------------------------------------------------------# - label: ":amd: (MI355 DPX) Examples" # TBD diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 971e308d25..c81bba10da 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -436,6 +436,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-tp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek R1 TP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-tp.txt - label: ":nvidia: (H200) LM Eval DeepSeek R1 DP" key: lm-eval-deepseek-r1-dp-8xh200 @@ -445,6 +479,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-dp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek R1 DP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt - label: ":nvidia: (H200) LM Eval DeepSeek V3.2 TP" key: lm-eval-deepseek-v32-tp-8xh200 @@ -454,6 +522,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-tp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt - label: ":nvidia: (H200) LM Eval DeepSeek V3.2 DP" key: lm-eval-deepseek-v32-dp-8xh200 @@ -463,6 +565,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-dp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-dp.txt - label: ":nvidia: (H200) LM Eval Nemotron 3 Super" key: lm-eval-nemotron-3-super-8xh200 @@ -472,6 +608,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-nemotron-3-super.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval Nemotron 3 Super" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/mamba/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-nemotron-3-super.txt - label: ":amd: (MI300) LM Eval Large Models" key: amd-lm-eval-large-models diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml new file mode 100644 index 0000000000..a482c47bff --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-R1" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --data-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml new file mode 100644 index 0000000000..74776f46b6 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-R1" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml new file mode 100644 index 0000000000..afb4bed274 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-V3.2" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --data-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA_SPARSE + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml new file mode 100644 index 0000000000..5bc5fa9af0 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-V3.2" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA_SPARSE + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml new file mode 100644 index 0000000000..bf67ca1f7d --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml @@ -0,0 +1,15 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --mamba-backend triton + --moe-backend triton + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml new file mode 100644 index 0000000000..9d76e8f3ab --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml @@ -0,0 +1,15 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --mamba-backend triton + --moe-backend triton + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt new file mode 100644 index 0000000000..2481c05f7b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt @@ -0,0 +1 @@ +DeepSeek-R1-DP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt new file mode 100644 index 0000000000..5c7753bc60 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt @@ -0,0 +1 @@ +DeepSeek-R1-TP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt new file mode 100644 index 0000000000..91d687c46b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt @@ -0,0 +1 @@ +DeepSeek-V3.2-DP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt new file mode 100644 index 0000000000..a21dfa4589 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt @@ -0,0 +1 @@ +DeepSeek-V3.2-TP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt b/tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt new file mode 100644 index 0000000000..5cb9401036 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt @@ -0,0 +1,2 @@ +Nemotron-3-Super-120B-A12B-BF16_MI355.yaml +Nemotron-3-Super-120B-A12B-FP8_MI355.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index 7411ca0f40..3b66de6ad0 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -109,15 +109,17 @@ def test_gsm8k_correctness(config_filename): "Its W2A16 (uint2b2) scheme has no kernel outside CUDA." ) - # TODO(akaratza): Enable DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms if current_platform.is_rocm() and ( "deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"] or "deepseek-ai/DeepSeek-R1" in eval_config["model_name"] ): - pytest.skip( - "Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms " - "due to agent pool disk space issues and pod evictions." - ) + from vllm.platforms.rocm import get_cdna_version + + if get_cdna_version() < 4: + pytest.skip( + "Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms below " + "CDNA 4 due to agent pool disk space issues and pod evictions." + ) if current_platform.is_rocm() and ( "Qwen3.5-35B-A3B-MXFP4-AITER-TP2" in config_filename.name ):