[CI][ROCm] Mirror large-model GSM8K evaluations on MI355 (#58683)

Signed-off-by: Andreas Karatzas <andreas.karatzas@protonmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-29 02:06:44 -05:00
committed by GitHub
co-authored by Codex
parent d86cf91240
commit 0ae480ff1a
14 changed files with 446 additions and 5 deletions
+165
View File
@@ -4248,6 +4248,171 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt
- label: ":amd: (MI355) LM Eval DeepSeek R1 TP"
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
optional: true
num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-tp.txt
- label: ":amd: (MI355) LM Eval DeepSeek R1 DP"
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
optional: true
num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt
- label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP"
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
optional: true
num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt
- label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP"
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
optional: true
num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-dp.txt
- label: ":amd: (MI355) LM Eval Nemotron 3 Super"
timeout_in_minutes: 65
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_4
optional: true
num_gpus: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/mamba/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-nemotron-3-super.txt
#--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
- label: ":amd: (MI355 DPX) Examples" # TBD
+170
View File
@@ -436,6 +436,40 @@ steps:
num_devices: 8
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-tp.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval DeepSeek R1 TP"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-tp.txt
- label: ":nvidia: (H200) LM Eval DeepSeek R1 DP"
key: lm-eval-deepseek-r1-dp-8xh200
@@ -445,6 +479,40 @@ steps:
num_devices: 8
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-dp.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval DeepSeek R1 DP"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt
- label: ":nvidia: (H200) LM Eval DeepSeek V3.2 TP"
key: lm-eval-deepseek-v32-tp-8xh200
@@ -454,6 +522,40 @@ steps:
num_devices: 8
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-tp.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt
- label: ":nvidia: (H200) LM Eval DeepSeek V3.2 DP"
key: lm-eval-deepseek-v32-dp-8xh200
@@ -463,6 +565,40 @@ steps:
num_devices: 8
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-dp.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/sparse_attn_indexer.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-dp.txt
- label: ":nvidia: (H200) LM Eval Nemotron 3 Super"
key: lm-eval-nemotron-3-super-8xh200
@@ -472,6 +608,40 @@ steps:
num_devices: 8
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-nemotron-3-super.txt
mirror:
amd:
label: ":amd: (MI355) LM Eval Nemotron 3 Super"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 65
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- .buildkite/test_areas/lm_eval.yaml
- tests/evals/gsm8k/
- tests/utils.py
- tests/conftest.py
- vllm/config/
- vllm/distributed/
- vllm/model_executor/models/
- vllm/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/model_executor/layers/fused_moe/
- vllm/v1/attention/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/core/kv_cache_utils.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/layers/mamba/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-nemotron-3-super.txt
- label: ":amd: (MI300) LM Eval Large Models"
key: amd-lm-eval-large-models
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-R1"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--data-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-R1"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-V3.2"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--data-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA_SPARSE
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,17 @@
model_name: "deepseek-ai/DeepSeek-V3.2"
accuracy_threshold: 0.95
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MLA: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--attention-backend ROCM_AITER_MLA_SPARSE
--moe-backend aiter
--speculative-config '{"method":"mtp","num_speculative_tokens":3}'
@@ -0,0 +1,15 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
accuracy_threshold: 0.93
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--mamba-backend triton
--moe-backend triton
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -0,0 +1,15 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
accuracy_threshold: 0.93
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
env:
VLLM_ROCM_USE_AITER: "1"
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 4
--enable-expert-parallel
--mamba-backend triton
--moe-backend triton
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -0,0 +1 @@
DeepSeek-R1-DP_MI355.yaml
@@ -0,0 +1 @@
DeepSeek-R1-TP_MI355.yaml
@@ -0,0 +1 @@
DeepSeek-V3.2-DP_MI355.yaml
@@ -0,0 +1 @@
DeepSeek-V3.2-TP_MI355.yaml
@@ -0,0 +1,2 @@
Nemotron-3-Super-120B-A12B-BF16_MI355.yaml
Nemotron-3-Super-120B-A12B-FP8_MI355.yaml
+7 -5
View File
@@ -109,15 +109,17 @@ def test_gsm8k_correctness(config_filename):
"Its W2A16 (uint2b2) scheme has no kernel outside CUDA."
)
# TODO(akaratza): Enable DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms
if current_platform.is_rocm() and (
"deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"]
or "deepseek-ai/DeepSeek-R1" in eval_config["model_name"]
):
pytest.skip(
"Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms "
"due to agent pool disk space issues and pod evictions."
)
from vllm.platforms.rocm import get_cdna_version
if get_cdna_version() < 4:
pytest.skip(
"Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms below "
"CDNA 4 due to agent pool disk space issues and pod evictions."
)
if current_platform.is_rocm() and (
"Qwen3.5-35B-A3B-MXFP4-AITER-TP2" in config_filename.name
):