[Hardware][AMD][CI] Use Triton-based AITER MHA for LM Eval Qwen-3.5 Models Tests (#46691)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
@@ -2790,13 +2790,12 @@ steps:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
|
||||
|
||||
- label: LM Eval Qwen3-5 Models (B200-MI355) %N # TBD
|
||||
timeout_in_minutes: 180
|
||||
- label: LM Eval Qwen3-5 Models (B200-MI355) # TBD
|
||||
timeout_in_minutes: 120
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
agent_pool: mi355_2
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
parallelism: 4
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/qwen3_5.py
|
||||
@@ -2811,7 +2810,7 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
|
||||
|
||||
- label: LM Eval Small Models (2xB200-2xMI355) # TBD
|
||||
timeout_in_minutes: 180
|
||||
|
||||
@@ -3,7 +3,6 @@ accuracy_threshold: 0.89
|
||||
tolerance: 0.03
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 3600
|
||||
server_args: >-
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 2
|
||||
@@ -11,3 +10,4 @@ server_args: >-
|
||||
--moe-backend aiter
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
ENABLE_CK: "0" # Avoid AITER CK-based MHA JIT compilation to save time
|
||||
|
||||
@@ -12,6 +12,8 @@ latencies by ~7% (see qwen2_5_vl for example usage)
|
||||
To use these ops, you must have a recent version of PyTorch installed (>= 2.4.0)
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import einops
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -31,9 +33,11 @@ def flash_attn_maxseqlen_wrapper(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
kwargs = {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
if is_rocm_aiter:
|
||||
from aiter import flash_attn_varlen_func
|
||||
|
||||
kwargs["window_size"] = (-1, -1)
|
||||
else:
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
||||
|
||||
|
||||
Reference in New Issue
Block a user