[CI] replace shellcheck script with shellcheck-py hook (#52572)
Signed-off-by: Turner <doubleujabbour@gmail.com> Signed-off-by: Turner Jabbour <doubleujabbour@gmail.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
5b1e7a812b
commit
1eab6fef01
@@ -52,10 +52,10 @@ CUDA_VISIBLE_DEVICES=$PREFILL_GPU_ID \
|
||||
VLLM_SSM_CONV_STATE_LAYOUT=DS \
|
||||
VLLM_KV_CACHE_LAYOUT=HND \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
|
||||
vllm serve $MODEL \
|
||||
vllm serve "$MODEL" \
|
||||
--port $PREFILL_PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--max-model-len 16384 \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
@@ -70,10 +70,10 @@ CUDA_VISIBLE_DEVICES=$DECODE_GPU_ID \
|
||||
VLLM_SSM_CONV_STATE_LAYOUT=DS \
|
||||
VLLM_KV_CACHE_LAYOUT=HND \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
|
||||
vllm serve $MODEL \
|
||||
vllm serve "$MODEL" \
|
||||
--port $DECODE_PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--max-model-len 16384 \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
|
||||
@@ -86,10 +86,10 @@ launch_baseline() {
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--dtype float16 \
|
||||
--enforce-eager"
|
||||
echo ${BASELINE_BASE_CMD}
|
||||
echo "${BASELINE_BASE_CMD}"
|
||||
bash -c "${BASELINE_BASE_CMD}" &
|
||||
sleep 10
|
||||
wait_for_server ${BASELINE_HOST} ${BASELINE_PORT}
|
||||
wait_for_server "${BASELINE_HOST}" "${BASELINE_PORT}"
|
||||
}
|
||||
|
||||
launch_pd() {
|
||||
@@ -97,53 +97,50 @@ launch_pd() {
|
||||
PREFILL_GPU_ID=$(compute_gpu_id 0 "${PREFILLER_TP_SIZE}")
|
||||
local DECODE_GPU_ID
|
||||
DECODE_GPU_ID=$(compute_gpu_id "${PREFILLER_TP_SIZE}" "${DECODER_TP_SIZE}")
|
||||
local kv_config="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"${KV_BUFFER_DEVICE}\"}"
|
||||
|
||||
PREFILL_BASE_CMD="
|
||||
ZE_AFFINITY_MASK=$PREFILL_GPU_ID \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 vllm serve $MODEL_NAME \
|
||||
--host ${PREFILL_HOST} \
|
||||
--port ${PREFILL_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
--dtype float16 \
|
||||
-tp ${PREFILLER_TP_SIZE} \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\"}'"
|
||||
|
||||
|
||||
DECODE_BASE_CMD="
|
||||
ZE_AFFINITY_MASK=$DECODE_GPU_ID \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 vllm serve $MODEL_NAME \
|
||||
--host ${DECODE_HOST} \
|
||||
--port ${DECODE_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
-tp ${DECODER_TP_SIZE} \
|
||||
--dtype float16 \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\"}'"
|
||||
|
||||
echo ${PREFILL_BASE_CMD}
|
||||
echo ${DECODE_BASE_CMD}
|
||||
sleep 2
|
||||
|
||||
# execute on hosts
|
||||
bash -c "${PREFILL_BASE_CMD}" &
|
||||
bash -c "${DECODE_BASE_CMD}" &
|
||||
echo "Starting prefill on GPU ${PREFILL_GPU_ID}, port ${PREFILL_PORT}"
|
||||
ZE_AFFINITY_MASK="${PREFILL_GPU_ID}" \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST="${PREFILL_HOST}" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT="${PREFILL_NIXL_SIDE_PORT}" \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--host "${PREFILL_HOST}" \
|
||||
--port "${PREFILL_PORT}" \
|
||||
--max-model-len "${MAX_MODEL_LEN}" \
|
||||
--seed 42 \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
--enforce-eager \
|
||||
--dtype float16 \
|
||||
-tp "${PREFILLER_TP_SIZE}" \
|
||||
--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
|
||||
--kv-transfer-config "${kv_config}" &
|
||||
|
||||
echo "Starting decode on GPU ${DECODE_GPU_ID}, port ${DECODE_PORT}"
|
||||
ZE_AFFINITY_MASK="${DECODE_GPU_ID}" \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--host "${DECODE_HOST}" \
|
||||
--port "${DECODE_PORT}" \
|
||||
--max-model-len "${MAX_MODEL_LEN}" \
|
||||
--seed 42 \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
--enforce-eager \
|
||||
-tp "${DECODER_TP_SIZE}" \
|
||||
--dtype float16 \
|
||||
--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
|
||||
--kv-transfer-config "${kv_config}" &
|
||||
|
||||
sleep 1
|
||||
wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
|
||||
wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}"
|
||||
sleep 1
|
||||
wait_for_server ${DECODE_HOST} ${DECODE_PORT}
|
||||
wait_for_server "${DECODE_HOST}" "${DECODE_PORT}"
|
||||
sleep 1
|
||||
}
|
||||
|
||||
@@ -153,7 +150,7 @@ launch_pd_proxy(){
|
||||
--prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
|
||||
--decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
|
||||
--host=${PROXY_HOST} --port ${PROXY_PORT}"
|
||||
echo ${PROXY_BASE_CMD}
|
||||
echo "${PROXY_BASE_CMD}"
|
||||
bash -c "${PROXY_BASE_CMD}" &
|
||||
sleep 2
|
||||
}
|
||||
@@ -161,7 +158,7 @@ launch_pd_proxy(){
|
||||
run_tests(){
|
||||
local service_url=$1
|
||||
local mode=$2
|
||||
python3 ${EXP_ROOT}/test_disagg_accuracy.py --service_url=${service_url} --model_name=${MODEL_NAME} --mode=${mode} --file_name=${OUTPUT_FILE}
|
||||
python3 "${EXP_ROOT}/test_disagg_accuracy.py" --service_url="${service_url}" --model_name="${MODEL_NAME}" --mode="${mode}" --file_name="${OUTPUT_FILE}"
|
||||
}
|
||||
|
||||
|
||||
@@ -178,7 +175,7 @@ launch_pd_proxy
|
||||
run_tests "http://${PROXY_HOST}:${PROXY_PORT}" "disagg"
|
||||
echo "-----P/D success----"
|
||||
|
||||
rm ${OUTPUT_FILE}
|
||||
rm "${OUTPUT_FILE}"
|
||||
cleanup
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -103,8 +103,10 @@ fi
|
||||
cleanup_instances() {
|
||||
echo ""
|
||||
echo "Cleaning up..."
|
||||
# shellcheck disable=SC2046 # word splitting is intentional for multiple PIDs
|
||||
kill $(jobs -pr) 2>/dev/null || true
|
||||
sleep 1
|
||||
# shellcheck disable=SC2046
|
||||
kill -9 $(jobs -pr) 2>/dev/null || true
|
||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
||||
pkill -9 -f "toy_proxy_server.*8192" 2>/dev/null || true
|
||||
@@ -122,7 +124,7 @@ wait_for_server() {
|
||||
local deadline=${5:-600}
|
||||
local elapsed=0
|
||||
echo "Waiting for ${server_name} on port ${port}..."
|
||||
while [ $elapsed -lt $deadline ]; do
|
||||
while [ "$elapsed" -lt "$deadline" ]; do
|
||||
if ! ps -p "$server_pid" > /dev/null 2>&1; then
|
||||
local status=0
|
||||
wait "$server_pid" || status=$?
|
||||
@@ -186,7 +188,7 @@ else
|
||||
else
|
||||
num=1
|
||||
fi
|
||||
for (( g=0; g<num; g++ )); do ALL_GPUS+=($g); done
|
||||
for (( g=0; g<num; g++ )); do ALL_GPUS+=("$g"); done
|
||||
fi
|
||||
|
||||
TOTAL_GPUS_NEEDED=$(( (NUM_PREFILL_INSTANCES * PREFILLER_TP_SIZE) + (NUM_DECODE_INSTANCES * DECODER_TP_SIZE) ))
|
||||
@@ -246,22 +248,22 @@ run_test_for_device() {
|
||||
|
||||
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT"
|
||||
env \
|
||||
${GPU_DEVICE_VAR}=$GPU_ID \
|
||||
${GPU_DEVICE_VAR}="$GPU_ID" \
|
||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||
UCX_NET_DEVICES=all \
|
||||
${VLLM_SSM_CONV_STATE_LAYOUT:+VLLM_SSM_CONV_STATE_LAYOUT=$VLLM_SSM_CONV_STATE_LAYOUT} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST="$NIXL_SIDE_CHANNEL_HOST" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $MODEL_NAME \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--max-model-len $MAX_MODEL_LEN \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
"${KV_CACHE_ARGS[@]}" \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--tensor-parallel-size "$PREFILLER_TP_SIZE" \
|
||||
--kv-transfer-config "$kv_config_p" \
|
||||
--speculative-config "$PREFILL_SPEC_CONFIG" \
|
||||
--attention-backend $ATTENTION_BACKEND \
|
||||
--attention-backend "$ATTENTION_BACKEND" \
|
||||
${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &
|
||||
local SERVER_PID=$!
|
||||
|
||||
@@ -285,22 +287,22 @@ run_test_for_device() {
|
||||
|
||||
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
|
||||
env \
|
||||
${GPU_DEVICE_VAR}=$GPU_ID \
|
||||
${GPU_DEVICE_VAR}="$GPU_ID" \
|
||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||
UCX_NET_DEVICES=all \
|
||||
${VLLM_SSM_CONV_STATE_LAYOUT:+VLLM_SSM_CONV_STATE_LAYOUT=$VLLM_SSM_CONV_STATE_LAYOUT} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST="$NIXL_SIDE_CHANNEL_HOST" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $MODEL_NAME \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--max-model-len $MAX_MODEL_LEN \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
"${KV_CACHE_ARGS[@]}" \
|
||||
--tensor-parallel-size $DECODER_TP_SIZE \
|
||||
--tensor-parallel-size "$DECODER_TP_SIZE" \
|
||||
--kv-transfer-config "$kv_config_d" \
|
||||
--speculative-config "$DECODE_SPEC_CONFIG" \
|
||||
--attention-backend $ATTENTION_BACKEND \
|
||||
--attention-backend "$ATTENTION_BACKEND" \
|
||||
${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &
|
||||
local SERVER_PID=$!
|
||||
|
||||
@@ -315,10 +317,10 @@ run_test_for_device() {
|
||||
echo "Starting proxy server on port $PROXY_PORT..."
|
||||
python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
|
||||
--port $PROXY_PORT \
|
||||
--prefiller-hosts ${PREFILL_HOSTS[*]} \
|
||||
--prefiller-ports ${PREFILL_PORTS[*]} \
|
||||
--decoder-hosts ${DECODE_HOSTS[*]} \
|
||||
--decoder-ports ${DECODE_PORTS[*]} &
|
||||
--prefiller-hosts "${PREFILL_HOSTS[@]}" \
|
||||
--prefiller-ports "${PREFILL_PORTS[@]}" \
|
||||
--decoder-hosts "${DECODE_HOSTS[@]}" \
|
||||
--decoder-ports "${DECODE_PORTS[@]}" &
|
||||
local PROXY_PID=$!
|
||||
|
||||
wait_for_server "$PROXY_PORT" "$PROXY_PID" "proxy" "/healthcheck" 60
|
||||
|
||||
Reference in New Issue
Block a user