From 27da2a2ac4776faa4265cba38ba86dd3a7119c4f Mon Sep 17 00:00:00 2001 From: Matt <156021403+mawong-amd@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:08:04 -0500 Subject: [PATCH] [Hardware][AMD][CI] Use Triton-based AITER MHA for LM Eval Qwen-3.5 Models Tests (#46691) Signed-off-by: Matthew Wong --- .buildkite/test-amd.yaml | 7 +++---- .../gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml | 2 +- vllm/v1/attention/ops/vit_attn_wrappers.py | 6 +++++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 09d8a33cc3..3a6568c2e0 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2790,13 +2790,12 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt -- label: LM Eval Qwen3-5 Models (B200-MI355) %N # TBD - timeout_in_minutes: 180 +- label: LM Eval Qwen3-5 Models (B200-MI355) # TBD + timeout_in_minutes: 120 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_2 num_gpus: 2 optional: true - parallelism: 4 working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/model_executor/models/qwen3_5.py @@ -2811,7 +2810,7 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt - label: LM Eval Small Models (2xB200-2xMI355) # TBD timeout_in_minutes: 180 diff --git a/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml b/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml index 2c0431747d..ca5cc450c0 100644 --- a/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml +++ b/tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml @@ -3,7 +3,6 @@ accuracy_threshold: 0.89 tolerance: 0.03 num_questions: 1319 num_fewshot: 5 -startup_max_wait_seconds: 3600 server_args: >- --max-model-len 4096 --tensor-parallel-size 2 @@ -11,3 +10,4 @@ server_args: >- --moe-backend aiter env: VLLM_ROCM_USE_AITER: "1" + ENABLE_CK: "0" # Avoid AITER CK-based MHA JIT compilation to save time diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index 4506f452cf..5bbcc3386e 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -12,6 +12,8 @@ latencies by ~7% (see qwen2_5_vl for example usage) To use these ops, you must have a recent version of PyTorch installed (>= 2.4.0) """ +from typing import Any + import einops import torch import torch.nn.functional as F @@ -31,9 +33,11 @@ def flash_attn_maxseqlen_wrapper( cu_seqlens: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None, ) -> torch.Tensor: - kwargs = {} + kwargs: dict[str, Any] = {} if is_rocm_aiter: from aiter import flash_attn_varlen_func + + kwargs["window_size"] = (-1, -1) else: from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func