[CI] replace shellcheck script with shellcheck-py hook (#52572)
Signed-off-by: Turner <doubleujabbour@gmail.com> Signed-off-by: Turner Jabbour <doubleujabbour@gmail.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
5b1e7a812b
commit
1eab6fef01
@@ -44,20 +44,20 @@ start_network() {
|
||||
|
||||
start_nodes() {
|
||||
for node in $(seq 0 $(($NUM_NODES-1))); do
|
||||
if [ "$IS_ROCM" -eq 1 ]; then
|
||||
GPU_DEVICES='--device /dev/kfd --device /dev/dri -e HIP_VISIBLE_DEVICES='
|
||||
else
|
||||
GPU_DEVICES='--gpus "device='
|
||||
fi
|
||||
DEVICE_LIST=""
|
||||
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
|
||||
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
|
||||
GPU_DEVICES+=$(($DEVICE_NUM))
|
||||
DEVICE_LIST+=$(($DEVICE_NUM))
|
||||
if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
|
||||
GPU_DEVICES+=','
|
||||
DEVICE_LIST+=','
|
||||
fi
|
||||
done
|
||||
if [ "$IS_ROCM" -eq 0 ]; then
|
||||
GPU_DEVICES+='"'
|
||||
if [ "$IS_ROCM" -eq 1 ]; then
|
||||
GPU_DEVICES=(--device /dev/kfd --device /dev/dri -e "HIP_VISIBLE_DEVICES=${DEVICE_LIST}")
|
||||
else
|
||||
# The literal quotes around device=... are required by docker's
|
||||
# --gpus value parser when the device list itself contains commas.
|
||||
GPU_DEVICES=(--gpus "\"device=${DEVICE_LIST}\"")
|
||||
fi
|
||||
|
||||
# start the container in detached mode
|
||||
@@ -67,7 +67,7 @@ start_nodes() {
|
||||
# 3. map the huggingface cache directory to the container
|
||||
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
|
||||
# starting from 192.168.10.11)
|
||||
docker run -d $GPU_DEVICES --shm-size=10.24gb -e HF_TOKEN \
|
||||
docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
|
||||
--network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
|
||||
/bin/bash -c "tail -f /dev/null"
|
||||
@@ -96,16 +96,15 @@ run_nodes() {
|
||||
# we start the worker nodes first, in detached mode, and then start the head node
|
||||
# in the foreground, so that the output of the head node is visible in the buildkite logs
|
||||
for node in $(seq $(($NUM_NODES - 1)) -1 0); do
|
||||
GPU_DEVICES='"device='
|
||||
DEVICE_LIST=""
|
||||
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
|
||||
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
|
||||
GPU_DEVICES+=$(($DEVICE_NUM))
|
||||
DEVICE_LIST+=$(($DEVICE_NUM))
|
||||
if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
|
||||
GPU_DEVICES+=','
|
||||
DEVICE_LIST+=','
|
||||
fi
|
||||
done
|
||||
GPU_DEVICES+='"'
|
||||
echo "Running node$node with GPU devices: $GPU_DEVICES"
|
||||
echo "Running node$node with GPU devices: $DEVICE_LIST"
|
||||
if [ "$node" -ne 0 ]; then
|
||||
docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user