[CI] Shard quantization, pooling, multimodal APIs, core kernels and PCP eval automatically (#61477)

Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>

.buildkite/test_areas/entrypoints.yaml

@@ -215,6 +215,7 @@ steps:
 - label: ":nvidia: (H200 MIG 35GB) Entrypoints Integration (Multimodal)"
   device: h200_35gb
   key: entrypoints-integration-multimodal
+  automatic_shard: true
   timeout_in_minutes: 45
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:

.buildkite/test_areas/kernels.yaml

@@ -30,9 +30,10 @@ steps:
       - vllm/_aiter_ops.py
       - vllm/platforms/rocm.py
 
-- label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels Shard %N"
+- label: ":nvidia: (H200 MIG 35GB) Core Operation Kernels"
   device: h200_35gb
   key: kernels-core-operation-test
+  automatic_shard: true
   timeout_in_minutes: 120
   source_file_dependencies:
   - csrc/
@@ -40,10 +41,10 @@ steps:
   - tests/kernels/test_concat_mla_q.py
   - tests/kernels/test_fused_qk_norm_rope_gate.py
   commands:
-    - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-  parallelism: 3
+    - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py
   mirror:
     amd:
+      parallelism: 3
       label: ":amd: (MI300) Core Operation Kernels Shard %N"
       dind: false
       device: mi300_1

.buildkite/test_areas/lm_eval.yaml

@@ -236,6 +236,7 @@ steps:
 
 - label: ":nvidia: (B200) LM Eval PCP"
   key: lm-eval-pcp-4xb200
+  automatic_shard: true
   timeout_in_minutes: 360
   device: b200-k8s
   num_devices: 4

.buildkite/test_areas/models_language.yaml

@@ -199,20 +199,23 @@ steps:
       - vllm/platforms/rocm.py
       - tests/models/language/generation_ppl_test
 
-- label: ":nvidia: (H200 MIG 35GB) Language Models (Extended Pooling) Shard %N"
+- label: ":nvidia: (H200 MIG 35GB) Language Models (Extended Pooling)"
   device: h200_35gb
   key: language-models-test-extended-pooling
+  automatic_shard: true
   timeout_in_minutes: 120
-  parallelism: 4
   optional: true
   source_file_dependencies:
   - vllm/
   - "!vllm/distributed/kv_transfer/"
   - tests/models/language/pooling
   commands:
-    - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
+    - pytest -v -s models/language/pooling -m 'not core_model'
   mirror:
     amd:
+      parallelism: 4
+      commands:
+        - "pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
       label: ":amd: (MI300) Language Models (Extended Pooling) Shard %N"
       dind: false
       device: mi300_1

.buildkite/test_areas/quantization.yaml

@@ -2,11 +2,11 @@ group: Quantization
 depends_on:
   - image-build
 steps:
-- label: ":nvidia: (H200 MIG 35GB) Quantization Shard %N"
+- label: ":nvidia: (H200 MIG 35GB) Quantization"
   device: h200_35gb
   key: quantization
+  automatic_shard: true
   timeout_in_minutes: 40
-  parallelism: 4
   source_file_dependencies:
   - csrc/
   - vllm/model_executor/layers/quantization
@@ -18,9 +18,10 @@ steps:
   - uv pip install --system conch-triton-kernels
   # The SM90-only checkpoint currently contains a removed weight_chan_scale
   # parameter. It was not exercised by the previous L4 job.
-  - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
+  - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py -k 'not test_compressed_tensors_w4a8_fp8'
   mirror:
     amd:
+      parallelism: 4
       label: ":amd: (MI300) Quantization Shard %N"
       dind: false
       device: mi300_1