[CI] Recover empty multi-node Docker networks and finish partial cleanup (#55454)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -121,18 +121,6 @@ clear_ci_orchestration_env() {
|
|||||||
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
|
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup_network() {
|
|
||||||
local max_nodes=${NUM_NODES:-2}
|
|
||||||
for node in $(seq 0 $((max_nodes - 1))); do
|
|
||||||
if docker ps -a -q -f name="node${node}" | grep -q .; then
|
|
||||||
docker stop "node${node}" || true
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if docker network ls | grep -q docker-net; then
|
|
||||||
docker network rm docker-net || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
amd_ci_teardown_log() {
|
amd_ci_teardown_log() {
|
||||||
local event=$1
|
local event=$1
|
||||||
shift
|
shift
|
||||||
@@ -1704,13 +1692,11 @@ if is_multi_node "$commands"; then
|
|||||||
|
|
||||||
/bin/bash -c "${composite_command}"
|
/bin/bash -c "${composite_command}"
|
||||||
exit_code=$?
|
exit_code=$?
|
||||||
cleanup_network
|
|
||||||
handle_pytest_exit "$exit_code"
|
handle_pytest_exit "$exit_code"
|
||||||
else
|
else
|
||||||
echo "Multi-node job detected but failed to parse bracket command syntax."
|
echo "Multi-node job detected but failed to parse bracket command syntax."
|
||||||
echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]"
|
echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]"
|
||||||
echo "Got: $commands"
|
echo "Got: $commands"
|
||||||
cleanup_network
|
|
||||||
exit 111
|
exit 111
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -38,8 +38,21 @@ for command in "${COMMANDS[@]}"; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
|
|
||||||
|
STARTED_CONTAINERS=()
|
||||||
|
NETWORK_CREATED=0
|
||||||
|
RUN_DIR=$(mktemp -d)
|
||||||
|
NETWORK_NAME="docker-net-${RUN_DIR##*/}"
|
||||||
|
|
||||||
start_network() {
|
start_network() {
|
||||||
docker network create --subnet=192.168.10.0/24 docker-net
|
if docker network inspect docker-net > /dev/null 2>&1; then
|
||||||
|
if [ "$(docker network inspect --format '{{len .Containers}}' docker-net)" != 0 ]; then
|
||||||
|
echo "docker-net still has attached containers; refusing to reuse it." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
docker network rm docker-net
|
||||||
|
fi
|
||||||
|
docker network create --subnet=192.168.10.0/24 "$NETWORK_NAME"
|
||||||
|
NETWORK_CREATED=1
|
||||||
}
|
}
|
||||||
|
|
||||||
start_nodes() {
|
start_nodes() {
|
||||||
@@ -67,20 +80,21 @@ start_nodes() {
|
|||||||
# 3. map the huggingface cache directory to the container
|
# 3. map the huggingface cache directory to the container
|
||||||
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
|
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
|
||||||
# starting from 192.168.10.11)
|
# starting from 192.168.10.11)
|
||||||
docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
|
CONTAINER_ID=$(docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
|
||||||
-v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
|
-v ~/.cache/huggingface:/root/.cache/huggingface --name "$NETWORK_NAME-node$node" \
|
||||||
--network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
|
--network "$NETWORK_NAME" --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
|
||||||
/bin/bash -c "tail -f /dev/null"
|
/bin/bash -c "tail -f /dev/null")
|
||||||
|
STARTED_CONTAINERS+=("$CONTAINER_ID")
|
||||||
|
|
||||||
# organize containers into a ray cluster
|
# organize containers into a ray cluster
|
||||||
if [ "$node" -eq 0 ]; then
|
if [ "$node" -eq 0 ]; then
|
||||||
# start the ray head node
|
# start the ray head node
|
||||||
docker exec -d "node$node" /bin/bash -c "ray start --head --port=6379 --block"
|
docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --head --port=6379 --block"
|
||||||
# wait for the head node to be ready
|
# wait for the head node to be ready
|
||||||
sleep 10
|
sleep 10
|
||||||
else
|
else
|
||||||
# start the ray worker nodes, and connect them to the head node
|
# start the ray worker nodes, and connect them to the head node
|
||||||
docker exec -d "node$node" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
|
docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -88,7 +102,7 @@ start_nodes() {
|
|||||||
sleep 10
|
sleep 10
|
||||||
|
|
||||||
# print the cluster status
|
# print the cluster status
|
||||||
docker exec node0 /bin/bash -c "ray status"
|
docker exec "${STARTED_CONTAINERS[0]}" /bin/bash -c "ray status"
|
||||||
}
|
}
|
||||||
|
|
||||||
run_nodes() {
|
run_nodes() {
|
||||||
@@ -106,22 +120,32 @@ run_nodes() {
|
|||||||
done
|
done
|
||||||
echo "Running node$node with GPU devices: $DEVICE_LIST"
|
echo "Running node$node with GPU devices: $DEVICE_LIST"
|
||||||
if [ "$node" -ne 0 ]; then
|
if [ "$node" -ne 0 ]; then
|
||||||
docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
docker exec -d "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||||
else
|
else
|
||||||
# Allocate a TTY (-t -i) for the foreground head node so its output
|
# Allocate a TTY (-t -i) for the foreground head node so its output
|
||||||
# keeps ANSI color in the Buildkite log (see run-amd-test.sh).
|
# keeps ANSI color in the Buildkite log (see run-amd-test.sh).
|
||||||
docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
docker exec -t -i "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
cleanup() {
|
cleanup() {
|
||||||
for node in $(seq 0 $(($NUM_NODES-1))); do
|
local status=$?
|
||||||
docker stop "node$node"
|
local cleanup_status=0
|
||||||
done
|
if [ "${#STARTED_CONTAINERS[@]}" -gt 0 ]; then
|
||||||
docker network rm docker-net
|
for container_id in "${STARTED_CONTAINERS[@]}"; do
|
||||||
|
docker stop "$container_id" || cleanup_status=$?
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
if [ "$NETWORK_CREATED" -eq 1 ]; then
|
||||||
|
docker network rm "$NETWORK_NAME" || cleanup_status=$?
|
||||||
|
fi
|
||||||
|
rmdir "$RUN_DIR" || cleanup_status=$?
|
||||||
|
if [ "$status" -ne 0 ]; then
|
||||||
|
return "$status"
|
||||||
|
fi
|
||||||
|
return "$cleanup_status"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
start_network
|
start_network
|
||||||
start_nodes
|
start_nodes
|
||||||
run_nodes
|
run_nodes
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user