From 4ec0b95c447fe70b92b2d12164c31255bb178e21 Mon Sep 17 00:00:00 2001 From: Thang Nguyen <69278249+Thangnguyenvn98@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:49:06 -0400 Subject: [PATCH] [CI] Split LM Eval TurboQuant KV Cache into per-config jobs (#57113) Signed-off-by: Thang Nguyen Co-authored-by: Kimi --- .buildkite/test_areas/lm_eval.yaml | 44 +++++++++++++++++-- .../configs/models-turboquant-k3v4nc.txt | 1 + .../gsm8k/configs/models-turboquant-k8v4.txt | 1 + .../gsm8k/configs/models-turboquant-t3nc.txt | 1 + .../gsm8k/configs/models-turboquant-t4nc.txt | 1 + .../evals/gsm8k/configs/models-turboquant.txt | 4 -- 6 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt create mode 100644 tests/evals/gsm8k/configs/models-turboquant-k8v4.txt create mode 100644 tests/evals/gsm8k/configs/models-turboquant-t3nc.txt create mode 100644 tests/evals/gsm8k/configs/models-turboquant-t4nc.txt delete mode 100644 tests/evals/gsm8k/configs/models-turboquant.txt diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 3450b70c28..c9dc4741e1 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -587,9 +587,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt -- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant KV Cache" - key: lm-eval-turboquant-kv-cache - timeout_in_minutes: 55 +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k8v4" + key: lm-eval-turboquant-k8v4 + timeout_in_minutes: 30 device: h200_18gb source_file_dependencies: - vllm/model_executor/layers/quantization/turboquant/ @@ -597,7 +597,43 @@ steps: - vllm/v1/attention/ops/triton_turboquant_decode.py - vllm/v1/attention/ops/triton_turboquant_store.py commands: - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4.txt + +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t4nc" + key: lm-eval-turboquant-t4nc + timeout_in_minutes: 30 + device: h200_18gb + source_file_dependencies: + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc.txt + +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant k3v4nc" + key: lm-eval-turboquant-k3v4nc + timeout_in_minutes: 30 + device: h200_18gb + source_file_dependencies: + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc.txt + +- label: ":nvidia: (H200 MIG 18GB) LM Eval TurboQuant t3nc" + key: lm-eval-turboquant-t3nc + timeout_in_minutes: 30 + device: h200_18gb + source_file_dependencies: + - vllm/model_executor/layers/quantization/turboquant/ + - vllm/v1/attention/backends/turboquant_attn.py + - vllm/v1/attention/ops/triton_turboquant_decode.py + - vllm/v1/attention/ops/triton_turboquant_store.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc.txt - label: ":nvidia: (H100) GPQA Eval (GPT-OSS)" key: gpqa-eval-gpt-oss-2xh100 diff --git a/tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt new file mode 100644 index 0000000000..836993e0fb --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k3v4nc.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k3v4nc.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-k8v4.txt b/tests/evals/gsm8k/configs/models-turboquant-k8v4.txt new file mode 100644 index 0000000000..3c0f67d9bf --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-k8v4.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-k8v4.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t3nc.txt b/tests/evals/gsm8k/configs/models-turboquant-t3nc.txt new file mode 100644 index 0000000000..49b31f737d --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t3nc.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t3nc.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant-t4nc.txt b/tests/evals/gsm8k/configs/models-turboquant-t4nc.txt new file mode 100644 index 0000000000..8cf140fa69 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-turboquant-t4nc.txt @@ -0,0 +1 @@ +Qwen3-4B-TQ-t4nc.yaml diff --git a/tests/evals/gsm8k/configs/models-turboquant.txt b/tests/evals/gsm8k/configs/models-turboquant.txt deleted file mode 100644 index 518aac780b..0000000000 --- a/tests/evals/gsm8k/configs/models-turboquant.txt +++ /dev/null @@ -1,4 +0,0 @@ -Qwen3-4B-TQ-k8v4.yaml -Qwen3-4B-TQ-t4nc.yaml -Qwen3-4B-TQ-k3v4nc.yaml -Qwen3-4B-TQ-t3nc.yaml