[CI] Deflake the multi-API-server metrics test and Mooncake PD ports (#59507)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
21c08ac458
commit
8bc31bd210
@@ -21,8 +21,10 @@ from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo
|
|||||||
|
|
||||||
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
DURATION = "vllm:rl_weight_update_operation_duration_seconds"
|
||||||
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight"
|
||||||
# Enough requests that the kernel spreads them over both API servers.
|
# The API servers accept from one shared listening socket, so the kernel may
|
||||||
SYNCS = 8
|
# hand one process most connections. Sync until both have recorded, within a cap.
|
||||||
|
MIN_SYNCS = 8
|
||||||
|
MAX_SYNCS = 64
|
||||||
|
|
||||||
|
|
||||||
def scrape(url: str) -> dict[tuple[str, str], float]:
|
def scrape(url: str) -> dict[tuple[str, str], float]:
|
||||||
@@ -75,11 +77,15 @@ def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path):
|
|||||||
client=HTTPVLLMWeightSyncClient(url),
|
client=HTTPVLLMWeightSyncClient(url),
|
||||||
source=ModuleSource(trainer),
|
source=ModuleSource(trainer),
|
||||||
)
|
)
|
||||||
for _ in range(SYNCS):
|
syncs = 0
|
||||||
|
while syncs < MIN_SYNCS or (
|
||||||
|
len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS
|
||||||
|
):
|
||||||
engine.send_weights()
|
engine.send_weights()
|
||||||
|
syncs += 1
|
||||||
metrics = scrape(url)
|
metrics = scrape(url)
|
||||||
|
|
||||||
calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS}
|
calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs}
|
||||||
for operation, count in calls.items():
|
for operation, count in calls.items():
|
||||||
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
assert metrics[(f"{DURATION}_count", operation)] == count, operation
|
||||||
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
assert metrics[(IN_FLIGHT, operation)] == 0, operation
|
||||||
|
|||||||
@@ -78,6 +78,20 @@ wait_for_server() {
|
|||||||
done" && return 0 || return 1
|
done" && return 0 || return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# b200-k8s pods use host networking, so a fixed port can already be taken by
|
||||||
|
# another job on the node. Bind all the ports at once so they are distinct.
|
||||||
|
free_ports() {
|
||||||
|
python3 - "$1" <<'EOF'
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
|
||||||
|
socks = [socket.socket() for _ in range(int(sys.argv[1]))]
|
||||||
|
for s in socks:
|
||||||
|
s.bind(("", 0))
|
||||||
|
print(" ".join(str(s.getsockname()[1]) for s in socks))
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
# Function to clean up previous instances
|
# Function to clean up previous instances
|
||||||
cleanup_instances() {
|
cleanup_instances() {
|
||||||
echo "Cleaning up any running vLLM instances..."
|
echo "Cleaning up any running vLLM instances..."
|
||||||
@@ -107,6 +121,9 @@ run_tests_for_model() {
|
|||||||
|
|
||||||
# Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL.
|
# Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL.
|
||||||
PROXY_ARGS=()
|
PROXY_ARGS=()
|
||||||
|
# Per prefiller: server and bootstrap. Per decoder: server. Then the proxy.
|
||||||
|
read -r -a PORTS <<< "$(free_ports $((2 * NUM_PREFILL_INSTANCES + NUM_DECODE_INSTANCES + 1)))"
|
||||||
|
PROXY_PORT=${PORTS[${#PORTS[@]} - 1]}
|
||||||
|
|
||||||
# Start prefill instances
|
# Start prefill instances
|
||||||
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
|
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
|
||||||
@@ -119,10 +136,8 @@ run_tests_for_model() {
|
|||||||
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
||||||
done
|
done
|
||||||
|
|
||||||
# Calculate port number (base port + instance number)
|
PORT=${PORTS[$((2 * i))]}
|
||||||
PORT=$((8100 + i))
|
BOOTSTRAP_PORT=${PORTS[$((2 * i + 1))]}
|
||||||
# Bootstrap server port for this prefiller. Avoid clash across instances.
|
|
||||||
BOOTSTRAP_PORT=$((8998 + i))
|
|
||||||
|
|
||||||
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT"
|
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT"
|
||||||
|
|
||||||
@@ -162,8 +177,7 @@ run_tests_for_model() {
|
|||||||
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
|
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
|
||||||
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
||||||
done
|
done
|
||||||
# Calculate port number (base port + instance number)
|
PORT=${PORTS[$((2 * NUM_PREFILL_INSTANCES + i))]}
|
||||||
PORT=$((8200 + i))
|
|
||||||
|
|
||||||
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
|
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
|
||||||
|
|
||||||
@@ -205,16 +219,16 @@ run_tests_for_model() {
|
|||||||
done
|
done
|
||||||
|
|
||||||
# Start the proxy server
|
# Start the proxy server
|
||||||
echo "Starting proxy server on port 8192"
|
echo "Starting proxy server on port $PROXY_PORT"
|
||||||
python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \
|
python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \
|
||||||
--port 8192 "${PROXY_ARGS[@]}" &
|
--port "$PROXY_PORT" "${PROXY_ARGS[@]}" &
|
||||||
|
|
||||||
# Wait for the proxy to start
|
# Wait for the proxy to start
|
||||||
sleep 5
|
sleep 5
|
||||||
|
|
||||||
# Run lm eval for this model
|
# Run lm eval for this model
|
||||||
echo "Running tests for $model_name"
|
echo "Running tests for $model_name"
|
||||||
TEST_MODEL=$model_name python3 -m pytest -s -x \
|
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \
|
||||||
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
|
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
|
||||||
|
|
||||||
# Clean up before running next model
|
# Clean up before running next model
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import os
|
|||||||
import lm_eval
|
import lm_eval
|
||||||
import openai
|
import openai
|
||||||
|
|
||||||
BASE_URL = "http://localhost:8192/v1"
|
BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1"
|
||||||
NUM_CONCURRENT = 100
|
NUM_CONCURRENT = 100
|
||||||
TASK = "gsm8k"
|
TASK = "gsm8k"
|
||||||
FILTER = "exact_match,strict-match"
|
FILTER = "exact_match,strict-match"
|
||||||
|
|||||||
Reference in New Issue
Block a user