diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 2d54c1bda0..e61e57e710 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -40,7 +40,7 @@ steps: - label: LM Eval Small Models (B200) key: lm-eval-small-models-b200 timeout_in_minutes: 120 - device: b200 + device: b200-k8s optional: true source_file_dependencies: - csrc/ @@ -48,6 +48,18 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt +- label: LM Eval Large Models (B200, EP) + key: lm-eval-large-models-b200-ep + timeout_in_minutes: 120 + device: b200-k8s + optional: true + num_devices: 2 + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt + - label: LM Eval Qwen3.5 Models (B200) key: lm-eval-qwen3-5-models-b200 timeout_in_minutes: 120 diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 3746957664..98c26b0e11 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -16,7 +16,7 @@ steps: - label: Spec Decode Eagle Nightly B200 key: spec-decode-eagle-nightly-b200 timeout_in_minutes: 30 - device: b200 + device: b200-k8s optional: true source_file_dependencies: - vllm/v1/spec_decode/ @@ -40,7 +40,7 @@ steps: - label: Spec Decode Speculators + MTP Nightly B200 key: spec-decode-speculators-mtp-nightly-b200 timeout_in_minutes: 30 - device: b200 + device: b200-k8s optional: true source_file_dependencies: - vllm/v1/spec_decode/ diff --git a/tests/evals/gsm8k/configs/models-blackwell-ep.txt b/tests/evals/gsm8k/configs/models-blackwell-ep.txt new file mode 100644 index 0000000000..c2a1905198 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-blackwell-ep.txt @@ -0,0 +1,3 @@ +Qwen3-Next-80B-A3B-NVFP4-EP2.yaml +Qwen3-Next-FP8-EP2.yaml +Nemotron-3-Super-120B-A12B-NVFP4.yaml diff --git a/tests/evals/gsm8k/configs/models-blackwell.txt b/tests/evals/gsm8k/configs/models-blackwell.txt index 2936fa891e..3c9b1084de 100644 --- a/tests/evals/gsm8k/configs/models-blackwell.txt +++ b/tests/evals/gsm8k/configs/models-blackwell.txt @@ -3,6 +3,3 @@ Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml Qwen1.5-MoE-W4A16-CT.yaml DeepSeek-V2-Lite-Instruct-FP8.yaml Qwen3-30B-A3B-NVFP4.yaml -Qwen3-Next-80B-A3B-NVFP4-EP2.yaml -Qwen3-Next-FP8-EP2.yaml -Nemotron-3-Super-120B-A12B-NVFP4.yaml diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index e14f7681bf..f820008670 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -488,6 +488,10 @@ def _run_eagle_correctness( @single_gpu_only +@pytest.mark.skipif( + current_platform.is_device_capability_family(100), + reason="DeepSeek head_dim=192 not supported on SM100/SM110 (Blackwell)", +) @pytest.mark.parametrize( [ "model_setup", @@ -718,7 +722,15 @@ def test_eagle_correctness_heavy( ["model_setup", "mm_enabled", "expected_accuracy_threshold"], [ (("mtp", "XiaomiMiMo/MiMo-7B-Base", 1), False, 0.5), # ref: 65%-70% - (("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1), False, 0.0), # dummy model + pytest.param( + ("mtp", "ZixiQi/DeepSeek-V3-4layers-MTP-FP8", 1), + False, + 0.0, + marks=pytest.mark.skipif( + current_platform.is_device_capability_family(100), + reason="DeepSeek MTP: TRTLLM MoE top_k check fails on Blackwell", + ), + ), # dummy model ( ("mtp", "Qwen/Qwen3.5-0.8B-Base", 1), False,