From 8bc31bd21075c5ca8a070e6da84e96e2eaf816ad Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 30 Sep 2026 17:22:58 -0700 Subject: [PATCH] [CI] Deflake the multi-API-server metrics test and Mooncake PD ports (#59507) Signed-off-by: khluu Co-authored-by: Claude Opus 5.5 --- .../dev/rlhf/test_weight_operation_metrics.py | 14 +++++--- .../mooncake_integration/run_accuracy_test.sh | 32 +++++++++++++------ .../mooncake_integration/test_accuracy.py | 2 +- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py b/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py index 6c7cb60bbb..063b307767 100644 --- a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py +++ b/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py @@ -21,8 +21,10 @@ from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo DURATION = "vllm:rl_weight_update_operation_duration_seconds" IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight" -# Enough requests that the kernel spreads them over both API servers. -SYNCS = 8 +# The API servers accept from one shared listening socket, so the kernel may +# hand one process most connections. Sync until both have recorded, within a cap. +MIN_SYNCS = 8 +MAX_SYNCS = 64 def scrape(url: str) -> dict[tuple[str, str], float]: @@ -75,11 +77,15 @@ def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path): client=HTTPVLLMWeightSyncClient(url), source=ModuleSource(trainer), ) - for _ in range(SYNCS): + syncs = 0 + while syncs < MIN_SYNCS or ( + len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS + ): engine.send_weights() + syncs += 1 metrics = scrape(url) - calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS} + calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs} for operation, count in calls.items(): assert metrics[(f"{DURATION}_count", operation)] == count, operation assert metrics[(IN_FLIGHT, operation)] == 0, operation diff --git a/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh b/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh index 9b3d129804..6436ccdb13 100755 --- a/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh +++ b/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh @@ -78,6 +78,20 @@ wait_for_server() { done" && return 0 || return 1 } +# b200-k8s pods use host networking, so a fixed port can already be taken by +# another job on the node. Bind all the ports at once so they are distinct. +free_ports() { + python3 - "$1" <<'EOF' +import socket +import sys + +socks = [socket.socket() for _ in range(int(sys.argv[1]))] +for s in socks: + s.bind(("", 0)) +print(" ".join(str(s.getsockname()[1]) for s in socks)) +EOF +} + # Function to clean up previous instances cleanup_instances() { echo "Cleaning up any running vLLM instances..." @@ -107,6 +121,9 @@ run_tests_for_model() { # Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL. PROXY_ARGS=() + # Per prefiller: server and bootstrap. Per decoder: server. Then the proxy. + read -r -a PORTS <<< "$(free_ports $((2 * NUM_PREFILL_INSTANCES + NUM_DECODE_INSTANCES + 1)))" + PROXY_PORT=${PORTS[${#PORTS[@]} - 1]} # Start prefill instances for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do @@ -119,10 +136,8 @@ run_tests_for_model() { GPU_ID="${GPU_ID},${NEXT_GPU}" done - # Calculate port number (base port + instance number) - PORT=$((8100 + i)) - # Bootstrap server port for this prefiller. Avoid clash across instances. - BOOTSTRAP_PORT=$((8998 + i)) + PORT=${PORTS[$((2 * i))]} + BOOTSTRAP_PORT=${PORTS[$((2 * i + 1))]} echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT" @@ -162,8 +177,7 @@ run_tests_for_model() { NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus))) GPU_ID="${GPU_ID},${NEXT_GPU}" done - # Calculate port number (base port + instance number) - PORT=$((8200 + i)) + PORT=${PORTS[$((2 * NUM_PREFILL_INSTANCES + i))]} echo "Starting decode instance $i on GPU $GPU_ID, port $PORT" @@ -205,16 +219,16 @@ run_tests_for_model() { done # Start the proxy server - echo "Starting proxy server on port 8192" + echo "Starting proxy server on port $PROXY_PORT" python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \ - --port 8192 "${PROXY_ARGS[@]}" & + --port "$PROXY_PORT" "${PROXY_ARGS[@]}" & # Wait for the proxy to start sleep 5 # Run lm eval for this model echo "Running tests for $model_name" - TEST_MODEL=$model_name python3 -m pytest -s -x \ + TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \ "${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py" # Clean up before running next model diff --git a/tests/v1/kv_connector/mooncake_integration/test_accuracy.py b/tests/v1/kv_connector/mooncake_integration/test_accuracy.py index 25033399e7..aa4c72faf4 100644 --- a/tests/v1/kv_connector/mooncake_integration/test_accuracy.py +++ b/tests/v1/kv_connector/mooncake_integration/test_accuracy.py @@ -5,7 +5,7 @@ import os import lm_eval import openai -BASE_URL = "http://localhost:8192/v1" +BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1" NUM_CONCURRENT = 100 TASK = "gsm8k" FILTER = "exact_match,strict-match"