[CI][BugFix] ShellCheck cleanup to remove baseline and preserve runtime behavior (#34514)
Signed-off-by: junuxyz <216036880+junuxyz@users.noreply.github.com>
This commit is contained in:
@@ -6,7 +6,7 @@ set -e
|
||||
|
||||
merge_base_commit=$(git merge-base HEAD origin/main)
|
||||
echo "INFO: current merge base commit with main: $merge_base_commit"
|
||||
git show --oneline -s $merge_base_commit
|
||||
git show --oneline -s "$merge_base_commit"
|
||||
|
||||
# test whether the metadata.json url is valid, retry each 3 minutes up to 5 times
|
||||
# this avoids cumbersome error messages & manual retries in case the precompiled wheel
|
||||
@@ -40,7 +40,7 @@ for i in {1..5}; do
|
||||
fi
|
||||
fi
|
||||
# failure handling & retry logic
|
||||
if [ $i -eq 5 ]; then
|
||||
if [ "$i" -eq 5 ]; then
|
||||
echo "ERROR: metadata is still not available after 5 attempts."
|
||||
echo "ERROR: Please check whether the precompiled wheel for commit $merge_base_commit is available."
|
||||
echo " NOTE: If $merge_base_commit is a new commit on main, maybe try again after its release pipeline finishes."
|
||||
|
||||
@@ -24,7 +24,7 @@ MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
|
||||
# Set 1 to use multimodal prompts; else to use text-only
|
||||
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
|
||||
MM_FLAG=""
|
||||
if [ $USE_MM_PROMPTS = "1" ]; then
|
||||
if [ "$USE_MM_PROMPTS" = "1" ]; then
|
||||
MM_FLAG="--use_mm_prompts"
|
||||
fi
|
||||
|
||||
@@ -51,7 +51,7 @@ LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_baseline.txt}"
|
||||
BASELINE_PD_FILE="${BASELINE_PD_FILE:-/tmp/vllm_epd_baseline.txt}"
|
||||
|
||||
mkdir -p $LOG_PATH
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
||||
@@ -87,20 +87,20 @@ run_baseline() {
|
||||
# Start baseline instance
|
||||
echo "Starting baseline instance on GPU $GPU_SINGLE, port $PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \
|
||||
--port $PORT \
|
||||
--port "$PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.7 \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
> $LOG_PATH/baseline.log 2>&1 &
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
> "$LOG_PATH"/baseline.log 2>&1 &
|
||||
|
||||
local BASELINE_PID=$!
|
||||
|
||||
# Wait for baseline to start
|
||||
echo "Waiting for baseline instance to start..."
|
||||
wait_for_server $PORT
|
||||
wait_for_server "$PORT"
|
||||
|
||||
curl http://127.0.0.1:$PORT/v1/models
|
||||
curl http://127.0.0.1:"$PORT"/v1/models
|
||||
echo ""
|
||||
|
||||
# Run test in baseline mode
|
||||
@@ -139,14 +139,14 @@ run_epd_1e_1pd() {
|
||||
# Start encoder instance
|
||||
echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
|
||||
--port $ENCODE_PORT \
|
||||
--port "$ENCODE_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.01 \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_producer",
|
||||
@@ -154,18 +154,18 @@ run_epd_1e_1pd() {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> $LOG_PATH/1e1pd_encoder.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1pd_encoder.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start prefill+decode instance
|
||||
echo "Starting PD instance on GPU $GPU_PD, port $PREFILL_DECODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_PD" vllm serve "$MODEL" \
|
||||
--port $PREFILL_DECODE_PORT \
|
||||
--port "$PREFILL_DECODE_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.7 \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
@@ -173,32 +173,32 @@ run_epd_1e_1pd() {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> $LOG_PATH/1e1pd_pd.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1pd_pd.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for instances to start
|
||||
echo "Waiting for encoder instance..."
|
||||
wait_for_server $ENCODE_PORT
|
||||
wait_for_server "$ENCODE_PORT"
|
||||
echo "Waiting for PD instance..."
|
||||
wait_for_server $PREFILL_DECODE_PORT
|
||||
wait_for_server "$PREFILL_DECODE_PORT"
|
||||
|
||||
# Start proxy
|
||||
echo "Starting EPD proxy on port $PROXY_PORT"
|
||||
python "${GIT_ROOT}/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port $PROXY_PORT \
|
||||
--port "$PROXY_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "disable" \
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
|
||||
> $LOG_PATH/1e1pd_proxy.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1pd_proxy.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for proxy
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server $PROXY_PORT
|
||||
wait_for_server "$PROXY_PORT"
|
||||
|
||||
curl http://127.0.0.1:$PROXY_PORT/v1/models
|
||||
curl http://127.0.0.1:$PROXY_PORT/health
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/v1/models
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/health
|
||||
echo ""
|
||||
|
||||
echo "All EPD (1E+1PD) services are up!"
|
||||
@@ -217,7 +217,7 @@ run_epd_1e_1pd() {
|
||||
echo "✓✓ 1E+1PD Correctness Test finished"
|
||||
echo "Stopping EPD (1E+1PD) instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill $pid 2>/dev/null || true
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
@@ -244,17 +244,17 @@ run_baseline_1p_1d() {
|
||||
CUDA_VISIBLE_DEVICES="$GPU_P" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
|
||||
vllm serve "$MODEL" \
|
||||
--port $PREFILL_PORT \
|
||||
--port "$PREFILL_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.7 \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_producer"
|
||||
}' \
|
||||
> $LOG_PATH/1p1d_prefill.log 2>&1 &
|
||||
> "$LOG_PATH"/1p1d_prefill.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start decode instance
|
||||
@@ -262,40 +262,40 @@ run_baseline_1p_1d() {
|
||||
CUDA_VISIBLE_DEVICES="$GPU_D" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
|
||||
vllm serve "$MODEL" \
|
||||
--port $DECODE_PORT \
|
||||
--port "$DECODE_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.7 \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_consumer"
|
||||
}' \
|
||||
> $LOG_PATH/1p1d_decode.log 2>&1 &
|
||||
> "$LOG_PATH"/1p1d_decode.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for instances to start
|
||||
echo "Waiting for prefill instance..."
|
||||
wait_for_server $PREFILL_PORT
|
||||
wait_for_server "$PREFILL_PORT"
|
||||
echo "Waiting for decode instance..."
|
||||
wait_for_server $DECODE_PORT
|
||||
wait_for_server "$DECODE_PORT"
|
||||
|
||||
# Start proxy
|
||||
echo "Starting EPD proxy on port $PROXY_PORT"
|
||||
python "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port $PROXY_PORT \
|
||||
--prefiller-ports $PREFILL_PORT \
|
||||
--decoder-ports $DECODE_PORT \
|
||||
> $LOG_PATH/1p1d_proxy.log 2>&1 &
|
||||
--port "$PROXY_PORT" \
|
||||
--prefiller-ports "$PREFILL_PORT" \
|
||||
--decoder-ports "$DECODE_PORT" \
|
||||
> "$LOG_PATH"/1p1d_proxy.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for proxy
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server $PROXY_PORT
|
||||
wait_for_server "$PROXY_PORT"
|
||||
|
||||
curl http://127.0.0.1:$PROXY_PORT/healthcheck
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/healthcheck
|
||||
echo ""
|
||||
|
||||
echo "All PD (1P+1D) services are up!"
|
||||
@@ -313,7 +313,7 @@ run_baseline_1p_1d() {
|
||||
# Cleanup
|
||||
echo "Stopping PD (1P+1D) instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill $pid 2>/dev/null || true
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
@@ -339,14 +339,14 @@ run_epd_1e_1p_1d() {
|
||||
# Start encoder instance
|
||||
echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_E" vllm serve "$MODEL" \
|
||||
--port $ENCODE_PORT \
|
||||
--port "$ENCODE_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.01 \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_producer",
|
||||
@@ -354,7 +354,7 @@ run_epd_1e_1p_1d() {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> $LOG_PATH/1e1p1d_encoder.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1p1d_encoder.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start prefill instance
|
||||
@@ -362,12 +362,12 @@ run_epd_1e_1p_1d() {
|
||||
CUDA_VISIBLE_DEVICES="$GPU_P" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
|
||||
vllm serve "$MODEL" \
|
||||
--port $PREFILL_PORT \
|
||||
--port "$PREFILL_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.7 \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
@@ -379,7 +379,7 @@ run_epd_1e_1p_1d() {
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_producer"
|
||||
}' \
|
||||
> $LOG_PATH/1e1p1d_prefill.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1p1d_prefill.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start decode instance
|
||||
@@ -387,44 +387,44 @@ run_epd_1e_1p_1d() {
|
||||
CUDA_VISIBLE_DEVICES="$GPU_D" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
|
||||
vllm serve "$MODEL" \
|
||||
--port $DECODE_PORT \
|
||||
--port "$DECODE_PORT" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.7 \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 128 \
|
||||
--allowed-local-media-path ${GIT_ROOT}/tests/v1/ec_connector/integration \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_consumer"
|
||||
}' \
|
||||
> $LOG_PATH/1e1p1d_decode.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1p1d_decode.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for instances to start
|
||||
echo "Waiting for encoder instance..."
|
||||
wait_for_server $ENCODE_PORT
|
||||
wait_for_server "$ENCODE_PORT"
|
||||
echo "Waiting for prefill instance..."
|
||||
wait_for_server $PREFILL_PORT
|
||||
wait_for_server "$PREFILL_PORT"
|
||||
echo "Waiting for decode instance..."
|
||||
wait_for_server $DECODE_PORT
|
||||
wait_for_server "$DECODE_PORT"
|
||||
|
||||
# Start proxy
|
||||
echo "Starting EPD proxy on port $PROXY_PORT"
|
||||
python "${GIT_ROOT}/examples/online_serving/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port $PROXY_PORT \
|
||||
--port "$PROXY_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "http://localhost:$PREFILL_PORT" \
|
||||
--decode-servers-urls "http://localhost:$DECODE_PORT" \
|
||||
> $LOG_PATH/1e1p1d_proxy.log 2>&1 &
|
||||
> "$LOG_PATH"/1e1p1d_proxy.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for proxy
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server $PROXY_PORT
|
||||
wait_for_server "$PROXY_PORT"
|
||||
|
||||
curl http://127.0.0.1:$PROXY_PORT/v1/models
|
||||
curl http://127.0.0.1:$PROXY_PORT/health
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/v1/models
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/health
|
||||
echo ""
|
||||
|
||||
echo "All EPD (1E+1P+1D) services are up!"
|
||||
@@ -443,7 +443,7 @@ run_epd_1e_1p_1d() {
|
||||
echo "✓✓ 1E+1P+1D Correctness Test finished"
|
||||
echo "Stopping EPD (1E+1P+1D) instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill $pid 2>/dev/null || true
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
|
||||
@@ -32,9 +32,14 @@ run_tests() {
|
||||
|
||||
echo "=== Running tests (${label}) ==="
|
||||
for cfg in "${configs[@]}"; do
|
||||
local -a cfg_parts extra_args_parts
|
||||
read -r -a cfg_parts <<< "$cfg"
|
||||
read -r -a extra_args_parts <<< "$extra_args"
|
||||
|
||||
echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
|
||||
# Use 'env' to safely set variables without eval
|
||||
if ! env ${cfg} bash "${SCRIPT}" ${extra_args}; then
|
||||
# keep argv splitting safe and SC2086-clean via arrays.
|
||||
if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
|
||||
echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -109,9 +109,9 @@ get_model_args() {
|
||||
|
||||
get_num_gpus() {
|
||||
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
|
||||
echo "$($SMI_BIN --query-gpu=name --format=csv,noheader | wc -l)"
|
||||
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
|
||||
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
|
||||
echo "$($SMI_BIN -l | grep GPU | wc -l)"
|
||||
$SMI_BIN -l | grep -c GPU
|
||||
else
|
||||
# works for non-cuda platforms,
|
||||
# assuming at least 1 device and
|
||||
@@ -182,7 +182,7 @@ run_tests_for_model() {
|
||||
|
||||
# Store host and port for proxy configuration
|
||||
PREFILL_HOSTS+=("localhost")
|
||||
PREFILL_PORTS+=($PORT)
|
||||
PREFILL_PORTS+=("$PORT")
|
||||
done
|
||||
|
||||
# Start decode instances
|
||||
@@ -237,30 +237,30 @@ run_tests_for_model() {
|
||||
|
||||
# Store host and port for proxy configuration
|
||||
DECODE_HOSTS+=("localhost")
|
||||
DECODE_PORTS+=($PORT)
|
||||
DECODE_PORTS+=("$PORT")
|
||||
done
|
||||
|
||||
# Wait for all instances to start
|
||||
for PORT in "${PREFILL_PORTS[@]}"; do
|
||||
echo "Waiting for prefill instance on port $PORT to start..."
|
||||
wait_for_server $PORT
|
||||
wait_for_server "$PORT"
|
||||
done
|
||||
|
||||
for PORT in "${DECODE_PORTS[@]}"; do
|
||||
echo "Waiting for decode instance on port $PORT to start..."
|
||||
wait_for_server $PORT
|
||||
wait_for_server "$PORT"
|
||||
done
|
||||
|
||||
# Build the command for the proxy server with all the hosts and ports
|
||||
PROXY_CMD="python3 ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port 8192"
|
||||
|
||||
# Add all prefill hosts and ports
|
||||
PROXY_CMD+=" --prefiller-hosts ${PREFILL_HOSTS[@]}"
|
||||
PROXY_CMD+=" --prefiller-ports ${PREFILL_PORTS[@]}"
|
||||
PROXY_CMD+=" --prefiller-hosts ${PREFILL_HOSTS[*]}"
|
||||
PROXY_CMD+=" --prefiller-ports ${PREFILL_PORTS[*]}"
|
||||
|
||||
# Add all decode hosts and ports
|
||||
PROXY_CMD+=" --decoder-hosts ${DECODE_HOSTS[@]}"
|
||||
PROXY_CMD+=" --decoder-ports ${DECODE_PORTS[@]}"
|
||||
PROXY_CMD+=" --decoder-hosts ${DECODE_HOSTS[*]}"
|
||||
PROXY_CMD+=" --decoder-ports ${DECODE_PORTS[*]}"
|
||||
|
||||
# Start the proxy server
|
||||
echo "Starting proxy server with command: $PROXY_CMD"
|
||||
@@ -271,7 +271,7 @@ run_tests_for_model() {
|
||||
|
||||
# Run lm eval for this model
|
||||
echo "Running tests for $model_name"
|
||||
TEST_MODEL=$model_name python3 -m pytest -s -x ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py
|
||||
TEST_MODEL=$model_name python3 -m pytest -s -x "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_accuracy.py
|
||||
|
||||
# Clean up before running next model
|
||||
cleanup_instances
|
||||
|
||||
@@ -114,10 +114,10 @@ run_tests_for_model() {
|
||||
eval "$FULL_CMD &"
|
||||
|
||||
# Wait for all instances to start
|
||||
echo "Waiting for prefill instance on port $PORT to start..."
|
||||
wait_for_server $PREFILL_PORT
|
||||
echo "Waiting for decode instance on port $PORT to start..."
|
||||
wait_for_server $DECODE_PORT
|
||||
echo "Waiting for prefill instance on port $PREFILL_PORT to start..."
|
||||
wait_for_server "$PREFILL_PORT"
|
||||
echo "Waiting for decode instance on port $DECODE_PORT to start..."
|
||||
wait_for_server "$DECODE_PORT"
|
||||
|
||||
# Build the command for the proxy server with all the hosts and ports
|
||||
PROXY_PORT=8192
|
||||
@@ -133,7 +133,7 @@ run_tests_for_model() {
|
||||
|
||||
# Run lm eval for this model
|
||||
echo "Running tests for $model_name"
|
||||
PREFILL_PORT=$PREFILL_PORT DECODE_PORT=$DECODE_PORT PROXY_PORT=$PROXY_PORT python -m pytest -s -v ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
|
||||
PREFILL_PORT=$PREFILL_PORT DECODE_PORT=$DECODE_PORT PROXY_PORT=$PROXY_PORT python -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
|
||||
|
||||
# Clean up before running next model
|
||||
cleanup_instances
|
||||
|
||||
@@ -63,8 +63,8 @@ launch_baseline() {
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--enforce-eager"
|
||||
echo ${BASELINE_BASE_CMD}
|
||||
ssh -tt ${BASELINE_HOST} "${BASELINE_BASE_CMD}" &
|
||||
echo "${BASELINE_BASE_CMD}"
|
||||
ssh -tt "${BASELINE_HOST}" "${BASELINE_BASE_CMD}" &
|
||||
}
|
||||
|
||||
launch_pd() {
|
||||
@@ -103,17 +103,17 @@ launch_pd() {
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
|
||||
|
||||
echo ${PREFILL_BASE_CMD}
|
||||
echo ${DECODE_BASE_CMD}
|
||||
echo "${PREFILL_BASE_CMD}"
|
||||
echo "${DECODE_BASE_CMD}"
|
||||
sleep 2
|
||||
|
||||
# execute on hosts
|
||||
ssh -tt ${PREFILL_HOST} "${PREFILL_BASE_CMD}" &
|
||||
ssh -tt ${DECODE_HOST} "${DECODE_BASE_CMD}" &
|
||||
ssh -tt "${PREFILL_HOST}" "${PREFILL_BASE_CMD}" &
|
||||
ssh -tt "${DECODE_HOST}" "${DECODE_BASE_CMD}" &
|
||||
sleep 1
|
||||
wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
|
||||
wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}"
|
||||
sleep 1
|
||||
wait_for_server ${DECODE_HOST} ${DECODE_PORT}
|
||||
wait_for_server "${DECODE_HOST}" "${DECODE_PORT}"
|
||||
sleep 1
|
||||
}
|
||||
|
||||
@@ -123,21 +123,21 @@ launch_pd_proxy(){
|
||||
--prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
|
||||
--decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
|
||||
--host=${PROXY_HOST} --port ${PROXY_PORT}"
|
||||
echo ${PROXY_BASE_CMD}
|
||||
ssh -tt ${PROXY_HOST} "${PROXY_BASE_CMD}" &
|
||||
echo "${PROXY_BASE_CMD}"
|
||||
ssh -tt "${PROXY_HOST}" "${PROXY_BASE_CMD}" &
|
||||
}
|
||||
|
||||
run_tests(){
|
||||
local service_url=$1
|
||||
local mode=$2
|
||||
python3 ${EXP_ROOT}/test_disagg_accuracy.py --service_url=${service_url} --model_name=${MODEL_NAME} --mode=${mode} --file_name=${OUTPUT_FILE}
|
||||
python3 "${EXP_ROOT}"/test_disagg_accuracy.py --service_url="${service_url}" --model_name="${MODEL_NAME}" --mode="${mode}" --file_name="${OUTPUT_FILE}"
|
||||
}
|
||||
|
||||
|
||||
# run non-disagg. baseline & save outputs
|
||||
launch_baseline
|
||||
sleep 2
|
||||
wait_for_server ${BASELINE_HOST} ${BASELINE_PORT}
|
||||
wait_for_server "${BASELINE_HOST}" "${BASELINE_PORT}"
|
||||
run_tests "http://${BASELINE_HOST}:${BASELINE_PORT}" "baseline"
|
||||
cleanup
|
||||
sleep 10
|
||||
@@ -150,7 +150,7 @@ sleep 10
|
||||
run_tests "http://${PROXY_HOST}:${PROXY_PORT}" "disagg"
|
||||
echo "-----P/D success----"
|
||||
|
||||
rm ${OUTPUT_FILE}
|
||||
rm "${OUTPUT_FILE}"
|
||||
cleanup
|
||||
|
||||
exit 0
|
||||
@@ -86,17 +86,17 @@ launch_pd() {
|
||||
--gpu-memory-utilization 0.5 \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"cpu\"}'"
|
||||
|
||||
echo ${PREFILL_BASE_CMD}
|
||||
echo ${DECODE_BASE_CMD}
|
||||
echo "${PREFILL_BASE_CMD}"
|
||||
echo "${DECODE_BASE_CMD}"
|
||||
sleep 2
|
||||
|
||||
# execute on hosts
|
||||
ssh -tt ${PREFILL_HOST} "${PREFILL_BASE_CMD}" &
|
||||
ssh -tt ${DECODE_HOST} "${DECODE_BASE_CMD}" &
|
||||
ssh -tt "${PREFILL_HOST}" "${PREFILL_BASE_CMD}" &
|
||||
ssh -tt "${DECODE_HOST}" "${DECODE_BASE_CMD}" &
|
||||
sleep 1
|
||||
wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
|
||||
wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}"
|
||||
sleep 1
|
||||
wait_for_server ${DECODE_HOST} ${DECODE_PORT}
|
||||
wait_for_server "${DECODE_HOST}" "${DECODE_PORT}"
|
||||
sleep 1
|
||||
}
|
||||
|
||||
@@ -106,8 +106,8 @@ launch_pd_proxy(){
|
||||
--prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
|
||||
--decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
|
||||
--host=${PROXY_HOST} --port ${PROXY_PORT}"
|
||||
echo ${PROXY_BASE_CMD}
|
||||
ssh -tt ${PROXY_HOST} "${PROXY_BASE_CMD}" &
|
||||
echo "${PROXY_BASE_CMD}"
|
||||
ssh -tt "${PROXY_HOST}" "${PROXY_BASE_CMD}" &
|
||||
}
|
||||
|
||||
|
||||
@@ -121,4 +121,4 @@ PREFILL_PORT=${PREFILL_PORT} \
|
||||
DECODE_HOST=${DECODE_HOST} \
|
||||
DECODE_PORT=${DECODE_PORT} \
|
||||
PROXY_HOST=${PROXY_HOST} \
|
||||
PROXY_PORT=${PROXY_PORT} python -m pytest -s -v ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
|
||||
PROXY_PORT=${PROXY_PORT} python -m pytest -s -v "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_edge_cases.py
|
||||
|
||||
Reference in New Issue
Block a user