From bb233626caa31602728f7ee4625f3d2a4d1a3ad5 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Mon, 17 Aug 2026 02:03:33 -0700 Subject: [PATCH] [CI] Shard Humming A100 eval (#52325) Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> --- .buildkite/test_areas/lm_eval.yaml | 5 +++-- tests/evals/gsm8k/configs/humming/config-a100-shard-0.txt | 4 ++++ tests/evals/gsm8k/configs/humming/config-a100-shard-1.txt | 5 +++++ tests/evals/gsm8k/configs/humming/config-a100-shard-2.txt | 5 +++++ 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 tests/evals/gsm8k/configs/humming/config-a100-shard-0.txt create mode 100644 tests/evals/gsm8k/configs/humming/config-a100-shard-1.txt create mode 100644 tests/evals/gsm8k/configs/humming/config-a100-shard-2.txt diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 4f72645730..f9c21a5421 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -195,12 +195,13 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt -- label: LM Eval Humming f16 (A100 - TEMPORARY) +- label: LM Eval Humming f16 (A100 - TEMPORARY) %N key: lm-eval-humming-f16-a100 timeout_in_minutes: 75 device: a100 optional: true num_devices: 1 + parallelism: 3 source_file_dependencies: - vllm/model_executor/layers/quantization/humming.py - vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -208,7 +209,7 @@ steps: - vllm/model_executor/layers/fused_moe/oracle/ - vllm/model_executor/kernels/linear/ commands: - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-a100-shard-$$BUILDKITE_PARALLEL_JOB.txt - label: LM Eval Humming Act int8 (A100 - TEMPORARY) key: lm-eval-humming-act-a100 diff --git a/tests/evals/gsm8k/configs/humming/config-a100-shard-0.txt b/tests/evals/gsm8k/configs/humming/config-a100-shard-0.txt new file mode 100644 index 0000000000..63b428ac0e --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/config-a100-shard-0.txt @@ -0,0 +1,4 @@ +Qwen3.5-35B-A3B-experts-int8-humming.yaml +gpt-oss-20b-humming.yaml +Qwen3-30B-A3B-GPTQ-Int4-humming.yaml +Qwen3-30B-A3B-FP8-block-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-a100-shard-1.txt b/tests/evals/gsm8k/configs/humming/config-a100-shard-1.txt new file mode 100644 index 0000000000..e0260c504e --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/config-a100-shard-1.txt @@ -0,0 +1,5 @@ +Qwen3.6-35B-A3B-NVFP4-humming.yaml +Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml +Qwen3-30B-A3B-Fp8-v1-humming.yaml +Qwen3-30B-A3B-AWQ-humming.yaml +Qwen3-0.6B-MXFP8-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-a100-shard-2.txt b/tests/evals/gsm8k/configs/humming/config-a100-shard-2.txt new file mode 100644 index 0000000000..8453e5d73c --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/config-a100-shard-2.txt @@ -0,0 +1,5 @@ +Qwen3.5-35B-A3B-FP8-humming.yaml +NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml +Qwen3-30B-A3B-MXFP4A16-humming.yaml +Qwen3-30B-A3B-NVFP4-humming.yaml +Qwen2-1.5B-Instruct-FP8W8-humming.yaml