[CI] Recover empty multi-node Docker networks and finish partial cleanup (#55454)

Signed-off-by: khluu <khluu000@gmail.com>
Co-authored-by: Codex <noreply@openai.com>

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -121,18 +121,6 @@ clear_ci_orchestration_env() {
     VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
 }
 
-cleanup_network() {
-  local max_nodes=${NUM_NODES:-2}
-  for node in $(seq 0 $((max_nodes - 1))); do
-    if docker ps -a -q -f name="node${node}" | grep -q .; then
-      docker stop "node${node}" || true
-    fi
-  done
-  if docker network ls | grep -q docker-net; then
-    docker network rm docker-net || true
-  fi
-}
-
 amd_ci_teardown_log() {
   local event=$1
   shift
@@ -1704,13 +1692,11 @@ if is_multi_node "$commands"; then
 
     /bin/bash -c "${composite_command}"
     exit_code=$?
-    cleanup_network
     handle_pytest_exit "$exit_code"
   else
     echo "Multi-node job detected but failed to parse bracket command syntax."
     echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]"
     echo "Got: $commands"
-    cleanup_network
     exit 111
   fi
 else

.buildkite/scripts/run-multi-node-test.sh

@@ -38,8 +38,21 @@ for command in "${COMMANDS[@]}"; do
 done
 
 
+STARTED_CONTAINERS=()
+NETWORK_CREATED=0
+RUN_DIR=$(mktemp -d)
+NETWORK_NAME="docker-net-${RUN_DIR##*/}"
+
 start_network() {
-    docker network create --subnet=192.168.10.0/24 docker-net
+    if docker network inspect docker-net > /dev/null 2>&1; then
+        if [ "$(docker network inspect --format '{{len .Containers}}' docker-net)" != 0 ]; then
+            echo "docker-net still has attached containers; refusing to reuse it." >&2
+            return 1
+        fi
+        docker network rm docker-net
+    fi
+    docker network create --subnet=192.168.10.0/24 "$NETWORK_NAME"
+    NETWORK_CREATED=1
 }
 
 start_nodes() {
@@ -67,20 +80,21 @@ start_nodes() {
         # 3. map the huggingface cache directory to the container
         # 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
         #    starting from 192.168.10.11)
-        docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
-            -v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
-            --network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
-            /bin/bash -c "tail -f /dev/null"
+        CONTAINER_ID=$(docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
+            -v ~/.cache/huggingface:/root/.cache/huggingface --name "$NETWORK_NAME-node$node" \
+            --network "$NETWORK_NAME" --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
+            /bin/bash -c "tail -f /dev/null")
+        STARTED_CONTAINERS+=("$CONTAINER_ID")
 
         # organize containers into a ray cluster
         if [ "$node" -eq 0 ]; then
             # start the ray head node
-            docker exec -d "node$node" /bin/bash -c "ray start --head --port=6379 --block"
+            docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --head --port=6379 --block"
             # wait for the head node to be ready
             sleep 10
         else
             # start the ray worker nodes, and connect them to the head node
-            docker exec -d "node$node" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
+            docker exec -d "$CONTAINER_ID" /bin/bash -c "ray start --address=192.168.10.10:6379 --block"
         fi
     done
 
@@ -88,7 +102,7 @@ start_nodes() {
     sleep 10
 
     # print the cluster status
-    docker exec node0 /bin/bash -c "ray status"
+    docker exec "${STARTED_CONTAINERS[0]}" /bin/bash -c "ray status"
 }
 
 run_nodes() {
@@ -106,22 +120,32 @@ run_nodes() {
         done
         echo "Running node$node with GPU devices: $DEVICE_LIST"
         if [ "$node" -ne 0 ]; then
-            docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
+            docker exec -d "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
         else
             # Allocate a TTY (-t -i) for the foreground head node so its output
             # keeps ANSI color in the Buildkite log (see run-amd-test.sh).
-            docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
+            docker exec -t -i "${STARTED_CONTAINERS[$node]}" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
         fi
     done
 }
 cleanup() {
-    for node in $(seq 0 $(($NUM_NODES-1))); do
-        docker stop "node$node"
-    done
-    docker network rm docker-net
+    local status=$?
+    local cleanup_status=0
+    if [ "${#STARTED_CONTAINERS[@]}" -gt 0 ]; then
+        for container_id in "${STARTED_CONTAINERS[@]}"; do
+            docker stop "$container_id" || cleanup_status=$?
+        done
+    fi
+    if [ "$NETWORK_CREATED" -eq 1 ]; then
+        docker network rm "$NETWORK_NAME" || cleanup_status=$?
+    fi
+    rmdir "$RUN_DIR" || cleanup_status=$?
+    if [ "$status" -ne 0 ]; then
+        return "$status"
+    fi
+    return "$cleanup_status"
 }
 trap cleanup EXIT
 start_network
 start_nodes
 run_nodes
-