From 9842d701450214d4b78cd9aefb8eee0c616bce33 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Tue, 18 Aug 2026 12:00:04 -0700 Subject: [PATCH] [DBO][CI] Increase the coverage of prefill DBO in test_dbo.py (#48628) Signed-off-by: Sage Moore --- tests/v1/distributed/test_dbo.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/v1/distributed/test_dbo.py b/tests/v1/distributed/test_dbo.py index e5cbe1ce85..a4e3d92be6 100644 --- a/tests/v1/distributed/test_dbo.py +++ b/tests/v1/distributed/test_dbo.py @@ -11,7 +11,6 @@ correctly with the DeepSeek-V2-Lite model using GSM8K evaluation. import pytest import torch -from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k from tests.utils import RemoteOpenAIServer from vllm.utils.import_utils import has_deep_ep @@ -58,6 +57,7 @@ def test_dbo_dp_ep_gsm8k(all2all_backend: str, num_gpus_available): """ Test DBO with DP+EP using GSM8K evaluation. """ + lm_eval = pytest.importorskip("lm_eval") required_gpus = DP_SIZE if num_gpus_available < required_gpus: @@ -79,7 +79,7 @@ def test_dbo_dp_ep_gsm8k(all2all_backend: str, num_gpus_available): "--dbo-decode-token-threshold", "16", "--dbo-prefill-token-threshold", - "256", + "32", "--all2all-backend", all2all_backend, ] @@ -90,19 +90,28 @@ def test_dbo_dp_ep_gsm8k(all2all_backend: str, num_gpus_available): max_wait_seconds=600, # Allow time for model loading with DP+EP ) as remote_server: # Use host and port directly from RemoteOpenAIServer - host = f"http://{remote_server.host}" - port = remote_server.port + + base_url = f"http://{remote_server.host}:{remote_server.port}/v1/completions" # Run GSM8K evaluation - results = evaluate_gsm8k( - num_questions=NUM_QUESTIONS, - num_shots=NUM_SHOTS, - host=host, - port=port, + results = lm_eval.simple_evaluate( + model="local-completions", + model_args=( + f"pretrained={MODEL_NAME}," + f"base_url={base_url}," + "num_concurrent=512,max_retries=3" + ), + tasks=["gsm8k"], + num_fewshot=NUM_SHOTS, + limit=NUM_QUESTIONS, ) - # Validate accuracy is reasonable - accuracy = results["accuracy"] + gsm8k = results["results"]["gsm8k"] + accuracy = gsm8k.get( + "exact_match,strict-match", + gsm8k.get("exact_match,flexible-extract"), + ) + assert accuracy is not None, f"gsm8k exact_match missing: {gsm8k}" assert accuracy >= MIN_ACCURACY, ( f"DBO+DP+EP accuracy too low ({all2all_backend}): " f"{accuracy:.3f} < {MIN_ACCURACY:.3f} "