[CI] Recover empty multi-node Docker networks and finish partial cleanup (#55454)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Kevin H. Luu
2026-09-07 12:53:13 -07:00
committed by GitHub
co-authored by Codex
parent 58f09211dd
commit 537af2c3a4
2 changed files with 39 additions and 29 deletions
@@ -121,18 +121,6 @@ clear_ci_orchestration_env() {
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
} }
cleanup_network() {
local max_nodes=${NUM_NODES:-2}
for node in $(seq 0 $((max_nodes - 1))); do
if docker ps -a -q -f name="node${node}" | grep -q .; then
docker stop "node${node}" || true
fi
done
if docker network ls | grep -q docker-net; then
docker network rm docker-net || true
fi
}
amd_ci_teardown_log() { amd_ci_teardown_log() {
local event=$1 local event=$1
shift shift
@@ -1704,13 +1692,11 @@ if is_multi_node "$commands"; then
/bin/bash -c "${composite_command}" /bin/bash -c "${composite_command}"
exit_code=$? exit_code=$?
cleanup_network
handle_pytest_exit "$exit_code" handle_pytest_exit "$exit_code"
else else
echo "Multi-node job detected but failed to parse bracket command syntax." echo "Multi-node job detected but failed to parse bracket command syntax."
echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]" echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]"
echo "Got: $commands" echo "Got: $commands"
cleanup_network
exit 111 exit 111
fi fi
else else
+39 -15
View File
@@ -38,8 +38,21 @@ for command in "${COMMANDS[@]}"; do
done done
STARTED_CONTAINERS=()
NETWORK_CREATED=0
RUN_DIR=$(mktemp -d)
NETWORK_NAME="docker-net-${RUN_DIR##*/}"
start_network() { start_network() {
docker network create --subnet=192.168.10.0/24 docker-net if docker network inspect docker-net > /dev/null 2>&1; then
if [ "$(docker network inspect --format '{{len .Containers}}' docker-net)" != 0 ]; then
echo "docker-net still has attached containers; refusing to reuse it." >&2
return 1
fi
docker network rm docker-net
fi
docker network create --subnet=192.168.10.0/24 "$NETWORK_NAME"
NETWORK_CREATED=1
} }
start_nodes() { start_nodes() {
@@ -67,20 +80,21 @@ start_nodes() {
# 3. map the huggingface cache directory to the container # 3. map the huggingface cache directory to the container
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes: # 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
# starting from 192.168.10.11) # starting from 192.168.10.11)
docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \ CONTAINER_ID=$(docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
-v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \ -v ~/.cache/huggingface:/root/.cache/huggingface --name "$NETWORK_NAME-node$node" \
--network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \ --network "$NETWORK_NAME" --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
/bin/bash -c "tail -f /dev/null" /bin/bash -c "tail -f /dev/null")
STARTED_CONTAINERS+=("$CONTAINER_ID")
# organize containers into a ray cluster # organize containers into a ray cluster
if [ "$node" -eq 0 ]; then if [ "$node" -eq 0 ]; then
# start the ray head node # start the ray head node
docker exec -d "node$node" /bin/bash -c "ray start --head --port=6379 --block" docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --head --port=6379 --block"
# wait for the head node to be ready # wait for the head node to be ready
sleep 10 sleep 10
else else
# start the ray worker nodes, and connect them to the head node # start the ray worker nodes, and connect them to the head node
docker exec -d "node$node" /bin/bash -c "ray start --address=192.168.10.10:6379 --block" docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
fi fi
done done
@@ -88,7 +102,7 @@ start_nodes() {
sleep 10 sleep 10
# print the cluster status # print the cluster status
docker exec node0 /bin/bash -c "ray status" docker exec "${STARTED_CONTAINERS[0]}" /bin/bash -c "ray status"
} }
run_nodes() { run_nodes() {
@@ -106,22 +120,32 @@ run_nodes() {
done done
echo "Running node$node with GPU devices: $DEVICE_LIST" echo "Running node$node with GPU devices: $DEVICE_LIST"
if [ "$node" -ne 0 ]; then if [ "$node" -ne 0 ]; then
docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}" docker exec -d "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
else else
# Allocate a TTY (-t -i) for the foreground head node so its output # Allocate a TTY (-t -i) for the foreground head node so its output
# keeps ANSI color in the Buildkite log (see run-amd-test.sh). # keeps ANSI color in the Buildkite log (see run-amd-test.sh).
docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}" docker exec -t -i "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
fi fi
done done
} }
cleanup() { cleanup() {
for node in $(seq 0 $(($NUM_NODES-1))); do local status=$?
docker stop "node$node" local cleanup_status=0
done if [ "${#STARTED_CONTAINERS[@]}" -gt 0 ]; then
docker network rm docker-net for container_id in "${STARTED_CONTAINERS[@]}"; do
docker stop "$container_id" || cleanup_status=$?
done
fi
if [ "$NETWORK_CREATED" -eq 1 ]; then
docker network rm "$NETWORK_NAME" || cleanup_status=$?
fi
rmdir "$RUN_DIR" || cleanup_status=$?
if [ "$status" -ne 0 ]; then
return "$status"
fi
return "$cleanup_status"
} }
trap cleanup EXIT trap cleanup EXIT
start_network start_network
start_nodes start_nodes
run_nodes run_nodes