diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 0b06aab230..fe4e4fd9f2 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3869,6 +3869,84 @@ steps: - uv pip install --system 'evalscope==1.10.0' - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt +- label: ":amd: (MI355) LM Eval TurboQuant k8v4" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt + +- label: ":amd: (MI355) LM Eval TurboQuant t4nc" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt + +- label: ":amd: (MI355) LM Eval TurboQuant k3v4nc" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt + - label: ":amd: (MI355) LM Eval Small Models Distributed" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index eaa8218047..f1ef63b4ed 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -646,6 +646,34 @@ steps: - vllm/v1/attention/ops/triton_turboquant_store.py commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant k8v4" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt - label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t4nc" key: lm-eval-turboquant-t4nc @@ -658,6 +686,34 @@ steps: - vllm/v1/attention/ops/triton_turboquant_store.py commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant t4nc" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt - label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k3v4nc" key: lm-eval-turboquant-k3v4nc @@ -670,6 +726,34 @@ steps: - vllm/v1/attention/ops/triton_turboquant_store.py commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval TurboQuant k3v4nc" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 30 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/backends/triton_attn.py + - vllm/v1/attention/ops/triton_unified_attention.py + - vllm/v1/attention/selector.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + - vllm/v1/attention/ops/flydsl_turboquant_decode.py + - vllm/v1/attention/ops/flydsl_kernels/ + - vllm/v1/attention/ops/turboquant_soa/ + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt - label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t3nc" key: lm-eval-turboquant-t3nc diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc-ROCm.yaml new file mode 100644 index 0000000000..879ad57c20 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.78 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_k3v4_nc --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4-ROCm.yaml new file mode 100644 index 0000000000..aa66561223 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_k8v4 --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc-ROCm.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc-ROCm.yaml new file mode 100644 index 0000000000..1348661b47 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc-ROCm.yaml @@ -0,0 +1,6 @@ +model_name: "Qwen/Qwen3-4B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +# Triton serves the unquantized boundary layers with TurboQuant-compatible layout. +server_args: "--kv-cache-dtype turboquant_4bit_nc --max-model-len 4096 --attention-backend TRITON_ATTN" diff --git a/tests/evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt new file mode 100644 index 0000000000..9a1e252a4b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k3v4nc-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt new file mode 100644 index 0000000000..458fe69f00 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k8v4-ROCm.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt b/tests/evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt new file mode 100644 index 0000000000..cd599778b6 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t4nc-ROCm.yaml