[CI][ROCm] Add DSpark evals (#54852)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>

.buildkite/test_areas/lm_eval.yaml

@@ -123,6 +123,45 @@ steps:
   - vllm/v1/worker/gpu/spec_decode/speculator.py
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt
+  mirror:
+    amd:
+      label: ":amd: (MI355) LM Eval Spec Decode"
+      dind: false
+      device: mi355_4
+      timeout_in_minutes: 120
+      depends_on:
+      - image-build-amd
+      source_file_dependencies:
+      - csrc/
+      - tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-AITER-TP4.yaml
+      - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml
+      - tests/evals/gsm8k/configs/models-spec-decode-rocm.txt
+      - tests/evals/gsm8k/test_gsm8k_correctness.py
+      - tests/evals/gsm8k/gsm8k_eval.py
+      - tests/evals/gsm8k/conftest.py
+      - vllm/config/
+      - vllm/envs.py
+      - vllm/models/kimi_k3/
+      - vllm/models/deepseek_v4/
+      - vllm/model_executor/kernels/mhc/
+      - vllm/model_executor/layers/mhc.py
+      - vllm/model_executor/layers/quantization/
+      - vllm/model_executor/layers/sparse_attn_indexer.py
+      - vllm/model_executor/models/config.py
+      - vllm/model_executor/models/registry.py
+      - vllm/model_executor/models/qwen3_dspark.py
+      - vllm/model_executor/model_loader/
+      - vllm/model_executor/warmup/
+      - vllm/model_executor/layers/fused_moe/
+      - vllm/tokenizers/
+      - vllm/v1/attention/
+      - vllm/v1/core/kv_cache_utils.py
+      - vllm/v1/spec_decode/metrics.py
+      - vllm/v1/worker/
+      - vllm/_aiter_ops.py
+      - vllm/platforms/rocm.py
+      commands:
+      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode-rocm.txt
 
 - label: ":nvidia: (B200) LM Eval Large Models EP"
   key: lm-eval-large-models-ep-2xb200

tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml

@@ -0,0 +1,29 @@
+model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark"
+accuracy_threshold: 0.92
+# Loose lower bound for smoke testing fixed-length DSpark verification.
+min_acceptance_length: 1.2
+num_questions: 1319
+num_fewshot: 5
+startup_max_wait_seconds: 1800
+server_args: >-
+  --tokenizer-mode deepseek_v4
+  --trust-remote-code
+  --max-model-len 8192
+  --tensor-parallel-size 4
+  --enable-expert-parallel
+  --block-size 256
+  --gpu-memory-utilization 0.5
+  --kv-cache-dtype fp8
+  --attention_config.indexer_kv_dtype=fp8
+  --max-num-batched-tokens 16384
+  --max-num-seqs 128
+  --moe-backend aiter
+  --speculative-config '{"method":"dspark",
+  "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark",
+  "num_speculative_tokens":7,
+  "draft_sample_method":"probabilistic",
+  "enable_adaptive_verification":false}'
+env:
+  VLLM_USE_V2_MODEL_RUNNER: "1"
+  VLLM_ROCM_USE_AITER: "1"
+  VLLM_ROCM_USE_AITER_MOE: "1"

tests/evals/gsm8k/configs/Kimi-K3-pruned75-DSpark-AITER-TP4.yaml

@@ -0,0 +1,28 @@
+model_name: "mgoin/Kimi-K3-pruned75"
+accuracy_threshold: 0.33
+min_acceptance_length: 4.8
+num_questions: 1319
+num_fewshot: 5
+startup_max_wait_seconds: 1800
+server_args: >-
+  --trust-remote-code
+  --gpu-memory-utilization 0.90
+  --tensor-parallel-size 4
+  --load-format fastsafetensors
+  --max-model-len 1048576
+  --max-num-batched-tokens 16384
+  --max-num-seqs 128
+  --kv-cache-dtype fp8
+  --moe-backend aiter
+  --attention-config '{"backend":"ROCM_AITER_MLA","mla_prefill_backend":"ROCM_AITER_FA"}'
+  --enable-prefix-caching
+  --enable-auto-tool-choice
+  --tool-call-parser kimi_k3
+  --reasoning-parser kimi_k3
+  --speculative-config '{"method":"dspark",
+  "model":"RedHatAI/Kimi-K3-speculator.dspark",
+  "attention_backend":"ROCM_AITER_FA","num_speculative_tokens":8,
+  "draft_sample_method":"probabilistic",
+  "rejection_sample_method":"block"}'
+env:
+  VLLM_ROCM_USE_AITER: "1"

tests/evals/gsm8k/configs/models-spec-decode-rocm.txt

@@ -0,0 +1,2 @@
+Kimi-K3-pruned75-DSpark-AITER-TP4.yaml
+DeepSeek-V4-Flash-DSpark-AITER-TEP4.yaml