[CI] Add GSM8K accuracy test for amd/DeepSeek-V4-Flash-MXFP4 (#50632)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>

.buildkite/lm-eval-harness/configs/DeepSeek-V4-Flash-MXFP4.yaml

@@ -0,0 +1,25 @@
+# DeepSeek-V4-Flash MXFP4 (AMD Quark) accuracy test.
+# Runs on ROCm (gfx950) with the AITER MoE + fused shared-experts path.
+# Reference GSM8K (8-shot, lm-eval): strict 0.9522 / flexible 0.9515 measured
+# on gfx950 TP=4 (AITER_MXFP4_BF16 backend); default rtol=0.08.
+model_name: "amd/DeepSeek-V4-Flash-MXFP4"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.95
+  - name: "exact_match,flexible-extract"
+    value: 0.95
+limit: 1319
+num_fewshot: 8
+max_model_len: 8192
+kv_cache_dtype: fp8
+moe_backend: aiter
+enforce_eager: false
+trust_remote_code: true
+tokenizer_mode: deepseek_v4
+required_gpu_arch:
+- gfx950
+env_vars:
+  VLLM_ROCM_USE_AITER: "1"
+  VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: "1"

.buildkite/lm-eval-harness/configs/models-large-rocm-tp4.txt

@@ -1 +1,2 @@
 Qwen3-235B-A22B-Instruct-2507-FP8.yaml
+DeepSeek-V4-Flash-MXFP4.yaml

.buildkite/lm-eval-harness/test_lm_eval_correctness.py

@@ -77,6 +77,10 @@ def launch_lm_eval(eval_config, tp_size):
         if rocm_load_strategy is not None:
             model_args += f"safetensors_load_strategy={rocm_load_strategy},"
 
+    tokenizer_mode = eval_config.get("tokenizer_mode", None)
+    if tokenizer_mode is not None:
+        model_args += f"tokenizer_mode={tokenizer_mode},"
+
     env_vars = eval_config.get("env_vars", None)
     with scoped_env_vars(env_vars):
         results = lm_eval.simple_evaluate(

.buildkite/test-amd.yaml

@@ -1488,7 +1488,7 @@ steps:
   - vllm/platforms/rocm.py
   commands:
   - export VLLM_USE_DEEP_GEMM=0
-  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
+  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
 
 #---------------------------------------------------------  mi300 · examples  ----------------------------------------------------------#
 
@@ -3374,7 +3374,7 @@ steps:
   - vllm/platforms/rocm.py
   commands:
   - export VLLM_USE_DEEP_GEMM=0
-  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
+  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-tp4.txt --tp-size=4
 
 - label: ":amd: (MI355) LM Eval Large Models" # TBD
   timeout_in_minutes: 180