[CI] replace shellcheck script with shellcheck-py hook (#52572)
Signed-off-by: Turner <doubleujabbour@gmail.com> Signed-off-by: Turner Jabbour <doubleujabbour@gmail.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Turner Jabbour
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
5b1e7a812b
commit
1eab6fef01
@@ -143,6 +143,7 @@ fi
|
||||
# Build-wide mode: fetch finished jobs matching $SCOPE.
|
||||
[ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
|
||||
|
||||
# shellcheck disable=SC2016 # single-quoted: these are awk field refs ($3/$4/$5), not shell vars
|
||||
case "$SCOPE" in
|
||||
failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
|
||||
soft) FILTER='$3 == "True" && $5 == "True"' ;;
|
||||
|
||||
@@ -29,7 +29,7 @@ if [[ -n "${BUILDKITE:-}" || -n "${BUILDKITE_COMMIT:-}" ]]; then
|
||||
image_tag="${image_name}:nightly-${tag_commit}"
|
||||
elif [[ "${variant}" == cu* ]]; then
|
||||
cuda_variant="${variant%%-*}"
|
||||
remaining_variant="${variant#${cuda_variant}}"
|
||||
remaining_variant="${variant#"${cuda_variant}"}"
|
||||
image_tag="${image_name}:${cuda_variant}-nightly-${tag_commit}${remaining_variant}"
|
||||
else
|
||||
image_tag="${image_name}:nightly-${tag_commit}${variant_suffix}"
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
# shellcheck disable=SC2329 # Every function in this file is only ever reached
|
||||
# transitively through handle_amd_runner_exit, the EXIT trap handler - shellcheck's
|
||||
# unused-function check doesn't do full reachability analysis from an indirectly
|
||||
# invoked entry point, so it flags the whole diagnostic-collection call chain as
|
||||
# dead code even though it runs live on every nonzero exit.
|
||||
|
||||
# This script runs ROCm tests either directly in a native CI pod or inside the
|
||||
# corresponding Docker container. Multi-node tests continue to use Docker.
|
||||
#
|
||||
@@ -1523,13 +1529,15 @@ fi
|
||||
clear_ci_orchestration_env
|
||||
if is_multi_node "$commands"; then
|
||||
echo "--- Multi-node job detected"
|
||||
export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
|
||||
DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
|
||||
export DCKR_VER
|
||||
|
||||
# Parse the bracket syntax: prefix ; [node0_cmds] && [node1_cmds]
|
||||
# BASH_REMATCH[1] = prefix (everything before first bracket)
|
||||
# BASH_REMATCH[2] = comma-separated node0 commands
|
||||
# BASH_REMATCH[3] = comma-separated node1 commands
|
||||
if [[ "$commands" =~ ^(.*)\[(.*)"] && ["(.*)\]$ ]]; then
|
||||
# shellcheck disable=SC2001 # verified equivalent behavior; TODO: switch to param expansion in a follow-up cleanup PR
|
||||
prefix=$(echo "${BASH_REMATCH[1]}" | sed 's/;//g')
|
||||
echo "PREFIX: ${prefix}"
|
||||
|
||||
@@ -1545,7 +1553,9 @@ if is_multi_node "$commands"; then
|
||||
fi
|
||||
|
||||
for i in "${!node0[@]}"; do
|
||||
# shellcheck disable=SC2001 # verified equivalent behavior; TODO: switch to param expansion in a follow-up cleanup PR
|
||||
command_node_0=$(echo "${node0[i]}" | sed 's/\"//g')
|
||||
# shellcheck disable=SC2001 # verified equivalent behavior; TODO: switch to param expansion in a follow-up cleanup PR
|
||||
command_node_1=$(echo "${node1[i]}" | sed 's/\"//g')
|
||||
|
||||
step_cmd="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'"
|
||||
@@ -1581,14 +1591,15 @@ else
|
||||
ulimit_core_hard="-1"
|
||||
fi
|
||||
# Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
|
||||
coredump_flags="--ulimit core=0:$ulimit_core_hard"
|
||||
coredump_flags=(--ulimit "core=0:$ulimit_core_hard")
|
||||
if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
|
||||
# Works around https://github.com/rocm/rocm-systems/issues/6206
|
||||
coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
|
||||
coredump_flags=(-e 'HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"')
|
||||
else
|
||||
echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086 # word splitting is intentional: both hold multiple docker flags
|
||||
docker run \
|
||||
"${docker_run_terminal_args[@]}" \
|
||||
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
|
||||
@@ -1597,7 +1608,7 @@ else
|
||||
--shm-size=16gb \
|
||||
--group-add "$render_gid" \
|
||||
--rm \
|
||||
$coredump_flags \
|
||||
"${coredump_flags[@]}" \
|
||||
-e HF_TOKEN \
|
||||
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
|
||||
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
|
||||
|
||||
@@ -42,97 +42,97 @@ aws ecr-public get-login-password --region us-east-1 | \
|
||||
# ---- CUDA (default: 13.0) ----
|
||||
|
||||
if target_enabled cuda-13-0; then
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64"
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64 vllm/vllm-openai:latest-x86_64
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64" vllm/vllm-openai:latest-x86_64
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64"
|
||||
docker push vllm/vllm-openai:latest-x86_64
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64 vllm/vllm-openai:latest-aarch64
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64" vllm/vllm-openai:latest-aarch64
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64"
|
||||
docker push vllm/vllm-openai:latest-aarch64
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64"
|
||||
|
||||
docker manifest rm vllm/vllm-openai:latest || true
|
||||
docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION} || true
|
||||
docker manifest rm "vllm/vllm-openai:v${RELEASE_VERSION}" || true
|
||||
docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64
|
||||
docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
|
||||
docker manifest create "vllm/vllm-openai:v${RELEASE_VERSION}" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64"
|
||||
docker manifest push vllm/vllm-openai:latest
|
||||
docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
|
||||
docker manifest push "vllm/vllm-openai:v${RELEASE_VERSION}"
|
||||
fi
|
||||
|
||||
# ---- CUDA 12.9 ----
|
||||
|
||||
if target_enabled cuda-12-9; then
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129"
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129 vllm/vllm-openai:latest-x86_64-cu129
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129" vllm/vllm-openai:latest-x86_64-cu129
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129"
|
||||
docker push vllm/vllm-openai:latest-x86_64-cu129
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129 vllm/vllm-openai:latest-aarch64-cu129
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129" vllm/vllm-openai:latest-aarch64-cu129
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129"
|
||||
docker push vllm/vllm-openai:latest-aarch64-cu129
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129"
|
||||
|
||||
docker manifest rm vllm/vllm-openai:latest-cu129 || true
|
||||
docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-cu129 || true
|
||||
docker manifest rm "vllm/vllm-openai:v${RELEASE_VERSION}-cu129" || true
|
||||
docker manifest create vllm/vllm-openai:latest-cu129 vllm/vllm-openai:latest-x86_64-cu129 vllm/vllm-openai:latest-aarch64-cu129
|
||||
docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
|
||||
docker manifest create "vllm/vllm-openai:v${RELEASE_VERSION}-cu129" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129"
|
||||
docker manifest push vllm/vllm-openai:latest-cu129
|
||||
docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129
|
||||
docker manifest push "vllm/vllm-openai:v${RELEASE_VERSION}-cu129"
|
||||
fi
|
||||
|
||||
# ---- Ubuntu 24.04 (CUDA 13.0) ----
|
||||
|
||||
if target_enabled cuda-13-0-ubuntu-24-04; then
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404"
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404 vllm/vllm-openai:latest-x86_64-ubuntu2404
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404" vllm/vllm-openai:latest-x86_64-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404"
|
||||
docker push vllm/vllm-openai:latest-x86_64-ubuntu2404
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404 vllm/vllm-openai:latest-aarch64-ubuntu2404
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404" vllm/vllm-openai:latest-aarch64-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404"
|
||||
docker push vllm/vllm-openai:latest-aarch64-ubuntu2404
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404"
|
||||
|
||||
docker manifest rm vllm/vllm-openai:latest-ubuntu2404 || true
|
||||
docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404 || true
|
||||
docker manifest rm "vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404" || true
|
||||
docker manifest create vllm/vllm-openai:latest-ubuntu2404 vllm/vllm-openai:latest-x86_64-ubuntu2404 vllm/vllm-openai:latest-aarch64-ubuntu2404
|
||||
docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
|
||||
docker manifest create "vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404"
|
||||
docker manifest push vllm/vllm-openai:latest-ubuntu2404
|
||||
docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404
|
||||
docker manifest push "vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404"
|
||||
fi
|
||||
|
||||
# ---- Ubuntu 24.04 (CUDA 12.9) ----
|
||||
|
||||
if target_enabled cuda-12-9-ubuntu-24-04; then
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404"
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404" vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404"
|
||||
docker push vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404 vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404" vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404"
|
||||
docker push vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
|
||||
docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
|
||||
docker push "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404"
|
||||
|
||||
docker manifest rm vllm/vllm-openai:latest-cu129-ubuntu2404 || true
|
||||
docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404 || true
|
||||
docker manifest rm "vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404" || true
|
||||
docker manifest create vllm/vllm-openai:latest-cu129-ubuntu2404 vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404 vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
|
||||
docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
|
||||
docker manifest create "vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404"
|
||||
docker manifest push vllm/vllm-openai:latest-cu129-ubuntu2404
|
||||
docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404
|
||||
docker manifest push "vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404"
|
||||
fi
|
||||
|
||||
# ---- ROCm ----
|
||||
@@ -141,36 +141,36 @@ if target_enabled rocm; then
|
||||
ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
|
||||
echo "ROCm base cache key: ${ROCM_BASE_CACHE_KEY}"
|
||||
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm"
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm vllm/vllm-openai-rocm:latest
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm vllm/vllm-openai-rocm:v${RELEASE_VERSION}
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" vllm/vllm-openai-rocm:latest
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}"
|
||||
docker push vllm/vllm-openai-rocm:latest
|
||||
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
|
||||
docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:latest-base
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" vllm/vllm-openai-rocm:latest-base
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base"
|
||||
docker push vllm/vllm-openai-rocm:latest-base
|
||||
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
|
||||
docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base"
|
||||
fi
|
||||
|
||||
# ---- XPU ----
|
||||
|
||||
if target_enabled xpu; then
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu
|
||||
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu"
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:latest-x86_64
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu" vllm/vllm-openai-xpu:latest-x86_64
|
||||
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu" "vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64"
|
||||
docker push vllm/vllm-openai-xpu:latest-x86_64
|
||||
docker push vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
|
||||
docker push "vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64"
|
||||
|
||||
docker manifest rm vllm/vllm-openai-xpu:latest || true
|
||||
docker manifest rm vllm/vllm-openai-xpu:v${RELEASE_VERSION} || true
|
||||
docker manifest rm "vllm/vllm-openai-xpu:v${RELEASE_VERSION}" || true
|
||||
docker manifest create vllm/vllm-openai-xpu:latest vllm/vllm-openai-xpu:latest-x86_64 --amend
|
||||
docker manifest create vllm/vllm-openai-xpu:v${RELEASE_VERSION} vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64 --amend
|
||||
docker manifest create "vllm/vllm-openai-xpu:v${RELEASE_VERSION}" "vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64" --amend
|
||||
docker manifest push vllm/vllm-openai-xpu:latest
|
||||
docker manifest push vllm/vllm-openai-xpu:v${RELEASE_VERSION}
|
||||
docker manifest push "vllm/vllm-openai-xpu:v${RELEASE_VERSION}"
|
||||
fi
|
||||
|
||||
# ---- CPU ----
|
||||
@@ -192,22 +192,22 @@ if target_enabled cpu; then
|
||||
if [ "$CPU_X86_AVAILABLE" = "true" ] && [ "$CPU_ARM_AVAILABLE" = "true" ]; then
|
||||
docker pull "${CPU_X86_TAG}"
|
||||
docker tag "${CPU_X86_TAG}" vllm/vllm-openai-cpu:latest-x86_64
|
||||
docker tag "${CPU_X86_TAG}" vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
|
||||
docker tag "${CPU_X86_TAG}" "vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64"
|
||||
docker push vllm/vllm-openai-cpu:latest-x86_64
|
||||
docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
|
||||
docker push "vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64"
|
||||
|
||||
docker pull "${CPU_ARM_TAG}"
|
||||
docker tag "${CPU_ARM_TAG}" vllm/vllm-openai-cpu:latest-arm64
|
||||
docker tag "${CPU_ARM_TAG}" vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
|
||||
docker tag "${CPU_ARM_TAG}" "vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64"
|
||||
docker push vllm/vllm-openai-cpu:latest-arm64
|
||||
docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
|
||||
docker push "vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64"
|
||||
|
||||
docker manifest rm vllm/vllm-openai-cpu:latest || true
|
||||
docker manifest rm vllm/vllm-openai-cpu:v${RELEASE_VERSION} || true
|
||||
docker manifest rm "vllm/vllm-openai-cpu:v${RELEASE_VERSION}" || true
|
||||
docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64
|
||||
docker manifest create vllm/vllm-openai-cpu:v${RELEASE_VERSION} vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
|
||||
docker manifest create "vllm/vllm-openai-cpu:v${RELEASE_VERSION}" "vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64" "vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64"
|
||||
docker manifest push vllm/vllm-openai-cpu:latest
|
||||
docker manifest push vllm/vllm-openai-cpu:v${RELEASE_VERSION}
|
||||
docker manifest push "vllm/vllm-openai-cpu:v${RELEASE_VERSION}"
|
||||
elif [ "$CPU_X86_AVAILABLE" = "false" ] && [ "$CPU_ARM_AVAILABLE" = "false" ]; then
|
||||
echo "WARNING: Neither CPU image found in ECR, skipping CPU publish (ensure block-cpu-release-image-build and block-arm64-cpu-release-image-build were unblocked and the builds finished pushing)"
|
||||
else
|
||||
|
||||
@@ -44,20 +44,20 @@ start_network() {
|
||||
|
||||
start_nodes() {
|
||||
for node in $(seq 0 $(($NUM_NODES-1))); do
|
||||
if [ "$IS_ROCM" -eq 1 ]; then
|
||||
GPU_DEVICES='--device /dev/kfd --device /dev/dri -e HIP_VISIBLE_DEVICES='
|
||||
else
|
||||
GPU_DEVICES='--gpus "device='
|
||||
fi
|
||||
DEVICE_LIST=""
|
||||
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
|
||||
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
|
||||
GPU_DEVICES+=$(($DEVICE_NUM))
|
||||
DEVICE_LIST+=$(($DEVICE_NUM))
|
||||
if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
|
||||
GPU_DEVICES+=','
|
||||
DEVICE_LIST+=','
|
||||
fi
|
||||
done
|
||||
if [ "$IS_ROCM" -eq 0 ]; then
|
||||
GPU_DEVICES+='"'
|
||||
if [ "$IS_ROCM" -eq 1 ]; then
|
||||
GPU_DEVICES=(--device /dev/kfd --device /dev/dri -e "HIP_VISIBLE_DEVICES=${DEVICE_LIST}")
|
||||
else
|
||||
# The literal quotes around device=... are required by docker's
|
||||
# --gpus value parser when the device list itself contains commas.
|
||||
GPU_DEVICES=(--gpus "\"device=${DEVICE_LIST}\"")
|
||||
fi
|
||||
|
||||
# start the container in detached mode
|
||||
@@ -67,7 +67,7 @@ start_nodes() {
|
||||
# 3. map the huggingface cache directory to the container
|
||||
# 3. assign ip addresses to the containers (head node: 192.168.10.10, worker nodes:
|
||||
# starting from 192.168.10.11)
|
||||
docker run -d $GPU_DEVICES --shm-size=10.24gb -e HF_TOKEN \
|
||||
docker run -d "${GPU_DEVICES[@]}" --shm-size=10.24gb -e HF_TOKEN \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface --name "node$node" \
|
||||
--network docker-net --ip 192.168.10.$((10 + $node)) --rm "$DOCKER_IMAGE" \
|
||||
/bin/bash -c "tail -f /dev/null"
|
||||
@@ -96,16 +96,15 @@ run_nodes() {
|
||||
# we start the worker nodes first, in detached mode, and then start the head node
|
||||
# in the foreground, so that the output of the head node is visible in the buildkite logs
|
||||
for node in $(seq $(($NUM_NODES - 1)) -1 0); do
|
||||
GPU_DEVICES='"device='
|
||||
DEVICE_LIST=""
|
||||
for node_gpu in $(seq 0 $(($NUM_GPUS - 1))); do
|
||||
DEVICE_NUM=$(($node * $NUM_GPUS + $node_gpu))
|
||||
GPU_DEVICES+=$(($DEVICE_NUM))
|
||||
DEVICE_LIST+=$(($DEVICE_NUM))
|
||||
if [ "$node_gpu" -lt $(($NUM_GPUS - 1)) ]; then
|
||||
GPU_DEVICES+=','
|
||||
DEVICE_LIST+=','
|
||||
fi
|
||||
done
|
||||
GPU_DEVICES+='"'
|
||||
echo "Running node$node with GPU devices: $GPU_DEVICES"
|
||||
echo "Running node$node with GPU devices: $DEVICE_LIST"
|
||||
if [ "$node" -ne 0 ]; then
|
||||
docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||
else
|
||||
|
||||
@@ -75,6 +75,7 @@ fi
|
||||
|
||||
# ---- Cleanup handler ----
|
||||
SERVER_PID=""
|
||||
# shellcheck disable=SC2329 # invoked via `trap cleanup EXIT` below
|
||||
cleanup() {
|
||||
if [ -n "$SERVER_PID" ]; then
|
||||
echo "Stopping vLLM server (pid=$SERVER_PID)..."
|
||||
|
||||
@@ -7,7 +7,7 @@ REPO="vllm-release-repo"
|
||||
ARCH_TAG="${BUILDKITE_COMMIT}-$(uname -m)-xpu"
|
||||
PLATFORM_TAG="${BUILDKITE_COMMIT}-xpu"
|
||||
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${REGISTRY}
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "${REGISTRY}"
|
||||
docker manifest rm "${REGISTRY}/${REPO}:${PLATFORM_TAG}" || true
|
||||
docker manifest create "${REGISTRY}/${REPO}:${PLATFORM_TAG}" "${REGISTRY}/${REPO}:${ARCH_TAG}" --amend
|
||||
docker manifest push ${REGISTRY}/${REPO}:${PLATFORM_TAG}
|
||||
docker manifest push "${REGISTRY}/${REPO}:${PLATFORM_TAG}"
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"problemMatcher": [
|
||||
{
|
||||
"owner": "shellcheck",
|
||||
"pattern": [
|
||||
{
|
||||
"regexp": "^(.+):(\\d+):(\\d+): (error|warning|note|info): (.+) \\[(SC\\d+)\\]$",
|
||||
"file": 1,
|
||||
"line": 2,
|
||||
"column": 3,
|
||||
"severity": 4,
|
||||
"message": 5,
|
||||
"code": 6
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -53,13 +53,10 @@ jobs:
|
||||
- uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
# Provide shellcheck on PATH so tools/pre_commit/shellcheck.sh skips its
|
||||
# wget + tar -xJ self-download, which the self-hosted runner image lacks
|
||||
# (no wget/xz). Pinned to shellcheck 0.10.0 to match the script's "stable".
|
||||
- run: python -m pip install shellcheck-py==0.10.0.1
|
||||
- run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
|
||||
- run: echo "::add-matcher::.github/workflows/matchers/markdownlint.json"
|
||||
- run: echo "::add-matcher::.github/workflows/matchers/mypy.json"
|
||||
- run: echo "::add-matcher::.github/workflows/matchers/shellcheck.json"
|
||||
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
|
||||
with:
|
||||
extra_args: --all-files --hook-stage manual
|
||||
|
||||
@@ -35,6 +35,11 @@ repos:
|
||||
rev: v1.7.7
|
||||
hooks:
|
||||
- id: actionlint
|
||||
- repo: https://github.com/shellcheck-py/shellcheck-py
|
||||
rev: 745eface02aef23e168a8afb6b5737818efbea95 # v0.11.0.1
|
||||
hooks:
|
||||
- id: shellcheck
|
||||
args: ["-s", "bash", "--format=gcc"]
|
||||
- repo: https://github.com/astral-sh/uv-pre-commit
|
||||
rev: 0.11.1
|
||||
hooks:
|
||||
@@ -195,11 +200,6 @@ repos:
|
||||
entry: python tools/pre_commit/mypy.py "3.13"
|
||||
<<: *mypy_common
|
||||
stages: [manual] # Only run in CI
|
||||
- id: shellcheck
|
||||
name: Lint shell scripts
|
||||
entry: tools/pre_commit/shellcheck.sh
|
||||
language: script
|
||||
types: [shell]
|
||||
- id: png-lint
|
||||
name: Lint PNG exports from excalidraw
|
||||
entry: tools/pre_commit/png-lint.sh
|
||||
|
||||
@@ -130,7 +130,7 @@ TEMP_BUILD_DIR=$(mktemp -d)
|
||||
cd "${TEMP_BUILD_DIR}"
|
||||
export BUILD_SOX=1 BUILD_KALDI=1 BUILD_RNNT=1 USE_FFMPEG=0 USE_ROCM=0 USE_CUDA=0
|
||||
export TORCHAUDIO_TEST_ALLOW_SKIP_IF_NO_FFMPEG=1
|
||||
git clone --recursive https://github.com/pytorch/audio.git -b v${TORCHAUDIO_VERSION}
|
||||
git clone --recursive https://github.com/pytorch/audio.git -b "v${TORCHAUDIO_VERSION}"
|
||||
cd audio
|
||||
#patching
|
||||
sed -i '
|
||||
|
||||
@@ -43,6 +43,7 @@ run_wrapper() {
|
||||
_env="$_env $1"; shift
|
||||
done
|
||||
shift
|
||||
# shellcheck disable=SC2086 # word splitting is intentional: $_env holds multiple NAME=value assignments
|
||||
env -i PATH="$WORKDIR/bin:/usr/bin:/bin" $_env "$WRAPPER" "$@" > "$_out"
|
||||
}
|
||||
|
||||
@@ -140,9 +141,8 @@ fake_passwd="$WORKDIR/fake-passwd-prepopulated"
|
||||
printf 'vllm:x:%s:%s:vllm:/home/vllm:/bin/bash\n' "$current_uid" "$current_gid" > "$fake_passwd"
|
||||
out="$WORKDIR/case6.out"
|
||||
run_wrapper "$out" "HOME=$case5_home" "VLLM_PASSWD_FILE=$fake_passwd" -- --model foo
|
||||
line_count="$(wc -l < "$fake_passwd")"
|
||||
# NOTE: wc may count 0 or 1 depending on trailing newline; accept 1.
|
||||
# More robust: count lines matching our UID.
|
||||
# wc may count 0 or 1 depending on trailing newline, so count lines
|
||||
# matching our UID instead for a robust check.
|
||||
uid_lines="$(grep -c ":${current_uid}:" "$fake_passwd" || true)"
|
||||
[ "$uid_lines" = "1" ] \
|
||||
|| { echo "FAIL: case6: expected exactly one entry for UID $current_uid, got $uid_lines"; cat "$fake_passwd"; exit 1; }
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# shellcheck disable=SC2317 # code after the exit/exit-183 gate below is intentionally dead pending re-enablement (see line comment)
|
||||
if [ "$READTHEDOCS_VERSION_TYPE" != "external" ]; then
|
||||
echo "Not a PR build (version type=$READTHEDOCS_VERSION_TYPE); skipping pre-run-check gate."
|
||||
exit 0
|
||||
|
||||
@@ -52,10 +52,10 @@ CUDA_VISIBLE_DEVICES=$PREFILL_GPU_ID \
|
||||
VLLM_SSM_CONV_STATE_LAYOUT=DS \
|
||||
VLLM_KV_CACHE_LAYOUT=HND \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
|
||||
vllm serve $MODEL \
|
||||
vllm serve "$MODEL" \
|
||||
--port $PREFILL_PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--max-model-len 16384 \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
@@ -70,10 +70,10 @@ CUDA_VISIBLE_DEVICES=$DECODE_GPU_ID \
|
||||
VLLM_SSM_CONV_STATE_LAYOUT=DS \
|
||||
VLLM_KV_CACHE_LAYOUT=HND \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
|
||||
vllm serve $MODEL \
|
||||
vllm serve "$MODEL" \
|
||||
--port $DECODE_PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--max-model-len 16384 \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
|
||||
@@ -86,10 +86,10 @@ launch_baseline() {
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--dtype float16 \
|
||||
--enforce-eager"
|
||||
echo ${BASELINE_BASE_CMD}
|
||||
echo "${BASELINE_BASE_CMD}"
|
||||
bash -c "${BASELINE_BASE_CMD}" &
|
||||
sleep 10
|
||||
wait_for_server ${BASELINE_HOST} ${BASELINE_PORT}
|
||||
wait_for_server "${BASELINE_HOST}" "${BASELINE_PORT}"
|
||||
}
|
||||
|
||||
launch_pd() {
|
||||
@@ -97,53 +97,50 @@ launch_pd() {
|
||||
PREFILL_GPU_ID=$(compute_gpu_id 0 "${PREFILLER_TP_SIZE}")
|
||||
local DECODE_GPU_ID
|
||||
DECODE_GPU_ID=$(compute_gpu_id "${PREFILLER_TP_SIZE}" "${DECODER_TP_SIZE}")
|
||||
local kv_config="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"${KV_BUFFER_DEVICE}\"}"
|
||||
|
||||
PREFILL_BASE_CMD="
|
||||
ZE_AFFINITY_MASK=$PREFILL_GPU_ID \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=${PREFILL_HOST} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=${PREFILL_NIXL_SIDE_PORT} \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 vllm serve $MODEL_NAME \
|
||||
--host ${PREFILL_HOST} \
|
||||
--port ${PREFILL_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
--dtype float16 \
|
||||
-tp ${PREFILLER_TP_SIZE} \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\"}'"
|
||||
|
||||
|
||||
DECODE_BASE_CMD="
|
||||
ZE_AFFINITY_MASK=$DECODE_GPU_ID \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 vllm serve $MODEL_NAME \
|
||||
--host ${DECODE_HOST} \
|
||||
--port ${DECODE_PORT} \
|
||||
--max-model-len ${MAX_MODEL_LEN}\
|
||||
--seed 42 \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--enforce-eager \
|
||||
-tp ${DECODER_TP_SIZE} \
|
||||
--dtype float16 \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\"}'"
|
||||
|
||||
echo ${PREFILL_BASE_CMD}
|
||||
echo ${DECODE_BASE_CMD}
|
||||
sleep 2
|
||||
|
||||
# execute on hosts
|
||||
bash -c "${PREFILL_BASE_CMD}" &
|
||||
bash -c "${DECODE_BASE_CMD}" &
|
||||
echo "Starting prefill on GPU ${PREFILL_GPU_ID}, port ${PREFILL_PORT}"
|
||||
ZE_AFFINITY_MASK="${PREFILL_GPU_ID}" \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST="${PREFILL_HOST}" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT="${PREFILL_NIXL_SIDE_PORT}" \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--host "${PREFILL_HOST}" \
|
||||
--port "${PREFILL_PORT}" \
|
||||
--max-model-len "${MAX_MODEL_LEN}" \
|
||||
--seed 42 \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
--enforce-eager \
|
||||
--dtype float16 \
|
||||
-tp "${PREFILLER_TP_SIZE}" \
|
||||
--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
|
||||
--kv-transfer-config "${kv_config}" &
|
||||
|
||||
echo "Starting decode on GPU ${DECODE_GPU_ID}, port ${DECODE_PORT}"
|
||||
ZE_AFFINITY_MASK="${DECODE_GPU_ID}" \
|
||||
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=200 \
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
VLLM_ENABLE_V1_MULTIPROCESSING=1 \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--host "${DECODE_HOST}" \
|
||||
--port "${DECODE_PORT}" \
|
||||
--max-model-len "${MAX_MODEL_LEN}" \
|
||||
--seed 42 \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
--enforce-eager \
|
||||
-tp "${DECODER_TP_SIZE}" \
|
||||
--dtype float16 \
|
||||
--gpu-memory-utilization "${GPU_MEMORY_UTILIZATION}" \
|
||||
--kv-transfer-config "${kv_config}" &
|
||||
|
||||
sleep 1
|
||||
wait_for_server ${PREFILL_HOST} ${PREFILL_PORT}
|
||||
wait_for_server "${PREFILL_HOST}" "${PREFILL_PORT}"
|
||||
sleep 1
|
||||
wait_for_server ${DECODE_HOST} ${DECODE_PORT}
|
||||
wait_for_server "${DECODE_HOST}" "${DECODE_PORT}"
|
||||
sleep 1
|
||||
}
|
||||
|
||||
@@ -153,7 +150,7 @@ launch_pd_proxy(){
|
||||
--prefiller-host ${PREFILL_HOST} --prefiller-port ${PREFILL_PORT} \
|
||||
--decoder-host ${DECODE_HOST} --decoder-port ${DECODE_PORT} \
|
||||
--host=${PROXY_HOST} --port ${PROXY_PORT}"
|
||||
echo ${PROXY_BASE_CMD}
|
||||
echo "${PROXY_BASE_CMD}"
|
||||
bash -c "${PROXY_BASE_CMD}" &
|
||||
sleep 2
|
||||
}
|
||||
@@ -161,7 +158,7 @@ launch_pd_proxy(){
|
||||
run_tests(){
|
||||
local service_url=$1
|
||||
local mode=$2
|
||||
python3 ${EXP_ROOT}/test_disagg_accuracy.py --service_url=${service_url} --model_name=${MODEL_NAME} --mode=${mode} --file_name=${OUTPUT_FILE}
|
||||
python3 "${EXP_ROOT}/test_disagg_accuracy.py" --service_url="${service_url}" --model_name="${MODEL_NAME}" --mode="${mode}" --file_name="${OUTPUT_FILE}"
|
||||
}
|
||||
|
||||
|
||||
@@ -178,7 +175,7 @@ launch_pd_proxy
|
||||
run_tests "http://${PROXY_HOST}:${PROXY_PORT}" "disagg"
|
||||
echo "-----P/D success----"
|
||||
|
||||
rm ${OUTPUT_FILE}
|
||||
rm "${OUTPUT_FILE}"
|
||||
cleanup
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -103,8 +103,10 @@ fi
|
||||
cleanup_instances() {
|
||||
echo ""
|
||||
echo "Cleaning up..."
|
||||
# shellcheck disable=SC2046 # word splitting is intentional for multiple PIDs
|
||||
kill $(jobs -pr) 2>/dev/null || true
|
||||
sleep 1
|
||||
# shellcheck disable=SC2046
|
||||
kill -9 $(jobs -pr) 2>/dev/null || true
|
||||
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
||||
pkill -9 -f "toy_proxy_server.*8192" 2>/dev/null || true
|
||||
@@ -122,7 +124,7 @@ wait_for_server() {
|
||||
local deadline=${5:-600}
|
||||
local elapsed=0
|
||||
echo "Waiting for ${server_name} on port ${port}..."
|
||||
while [ $elapsed -lt $deadline ]; do
|
||||
while [ "$elapsed" -lt "$deadline" ]; do
|
||||
if ! ps -p "$server_pid" > /dev/null 2>&1; then
|
||||
local status=0
|
||||
wait "$server_pid" || status=$?
|
||||
@@ -186,7 +188,7 @@ else
|
||||
else
|
||||
num=1
|
||||
fi
|
||||
for (( g=0; g<num; g++ )); do ALL_GPUS+=($g); done
|
||||
for (( g=0; g<num; g++ )); do ALL_GPUS+=("$g"); done
|
||||
fi
|
||||
|
||||
TOTAL_GPUS_NEEDED=$(( (NUM_PREFILL_INSTANCES * PREFILLER_TP_SIZE) + (NUM_DECODE_INSTANCES * DECODER_TP_SIZE) ))
|
||||
@@ -246,22 +248,22 @@ run_test_for_device() {
|
||||
|
||||
echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT"
|
||||
env \
|
||||
${GPU_DEVICE_VAR}=$GPU_ID \
|
||||
${GPU_DEVICE_VAR}="$GPU_ID" \
|
||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||
UCX_NET_DEVICES=all \
|
||||
${VLLM_SSM_CONV_STATE_LAYOUT:+VLLM_SSM_CONV_STATE_LAYOUT=$VLLM_SSM_CONV_STATE_LAYOUT} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST="$NIXL_SIDE_CHANNEL_HOST" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $MODEL_NAME \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--max-model-len $MAX_MODEL_LEN \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
"${KV_CACHE_ARGS[@]}" \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--tensor-parallel-size "$PREFILLER_TP_SIZE" \
|
||||
--kv-transfer-config "$kv_config_p" \
|
||||
--speculative-config "$PREFILL_SPEC_CONFIG" \
|
||||
--attention-backend $ATTENTION_BACKEND \
|
||||
--attention-backend "$ATTENTION_BACKEND" \
|
||||
${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &
|
||||
local SERVER_PID=$!
|
||||
|
||||
@@ -285,22 +287,22 @@ run_test_for_device() {
|
||||
|
||||
echo "Starting decode instance $i on GPU $GPU_ID, port $PORT"
|
||||
env \
|
||||
${GPU_DEVICE_VAR}=$GPU_ID \
|
||||
${GPU_DEVICE_VAR}="$GPU_ID" \
|
||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||
UCX_NET_DEVICES=all \
|
||||
${VLLM_SSM_CONV_STATE_LAYOUT:+VLLM_SSM_CONV_STATE_LAYOUT=$VLLM_SSM_CONV_STATE_LAYOUT} \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST="$NIXL_SIDE_CHANNEL_HOST" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $MODEL_NAME \
|
||||
vllm serve "$MODEL_NAME" \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--max-model-len $MAX_MODEL_LEN \
|
||||
--block-size ${BLOCK_SIZE} \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--block-size "${BLOCK_SIZE}" \
|
||||
"${KV_CACHE_ARGS[@]}" \
|
||||
--tensor-parallel-size $DECODER_TP_SIZE \
|
||||
--tensor-parallel-size "$DECODER_TP_SIZE" \
|
||||
--kv-transfer-config "$kv_config_d" \
|
||||
--speculative-config "$DECODE_SPEC_CONFIG" \
|
||||
--attention-backend $ATTENTION_BACKEND \
|
||||
--attention-backend "$ATTENTION_BACKEND" \
|
||||
${EXTRA_SERVE_ARGS[@]+"${EXTRA_SERVE_ARGS[@]}"} &
|
||||
local SERVER_PID=$!
|
||||
|
||||
@@ -315,10 +317,10 @@ run_test_for_device() {
|
||||
echo "Starting proxy server on port $PROXY_PORT..."
|
||||
python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
|
||||
--port $PROXY_PORT \
|
||||
--prefiller-hosts ${PREFILL_HOSTS[*]} \
|
||||
--prefiller-ports ${PREFILL_PORTS[*]} \
|
||||
--decoder-hosts ${DECODE_HOSTS[*]} \
|
||||
--decoder-ports ${DECODE_PORTS[*]} &
|
||||
--prefiller-hosts "${PREFILL_HOSTS[@]}" \
|
||||
--prefiller-ports "${PREFILL_PORTS[@]}" \
|
||||
--decoder-hosts "${DECODE_HOSTS[@]}" \
|
||||
--decoder-ports "${DECODE_PORTS[@]}" &
|
||||
local PROXY_PID=$!
|
||||
|
||||
wait_for_server "$PROXY_PORT" "$PROXY_PID" "proxy" "/healthcheck" 60
|
||||
|
||||
@@ -117,6 +117,7 @@ echo "Building TorchCodec (MAX_JOBS=$MAX_JOBS)..."
|
||||
pip wheel . --no-build-isolation --no-deps -w "$BUILD_DIR/dist"
|
||||
|
||||
# Install the built wheel
|
||||
# shellcheck disable=SC2012 # `ls` sorting is relied on here; TODO: switch to `find` in a follow-up cleanup PR
|
||||
BUILT_WHEEL=$(ls "$BUILD_DIR/dist"/torchcodec-*.whl 2>/dev/null | head -1)
|
||||
if [ -z "$BUILT_WHEEL" ]; then
|
||||
echo "Error: No wheel produced"
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
scversion="stable"
|
||||
|
||||
if [ -d "shellcheck-${scversion}" ]; then
|
||||
export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
|
||||
fi
|
||||
|
||||
if ! [ -x "$(command -v shellcheck)" ]; then
|
||||
if [ "$(uname -s)" != "Linux" ] || [ "$(uname -m)" != "x86_64" ]; then
|
||||
echo "Please install shellcheck: https://github.com/koalaman/shellcheck?tab=readme-ov-file#installing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# automatic local install if linux x86_64
|
||||
wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
|
||||
export PATH="$PATH:$(pwd)/shellcheck-${scversion}"
|
||||
fi
|
||||
|
||||
# TODO - fix warnings in .buildkite/scripts/hardware_ci/run-amd-test.sh
|
||||
find . -path ./.git -prune -o -name "*.sh" \
|
||||
-not -path "./.buildkite/scripts/hardware_ci/run-amd-test.sh" -print0 | \
|
||||
xargs -0 sh -c "for f in \"\$@\"; do git check-ignore -q \"\$f\" || shellcheck -s bash \"\$f\"; done" --
|
||||
@@ -17,7 +17,8 @@ if [ "$#" -eq 0 ]; then
|
||||
| grep -oE '>=3\.[0-9]+,<3\.[0-9]+')
|
||||
lo=${spec#>=3.}; lo=${lo%%,*}
|
||||
hi=${spec##*<3.}
|
||||
set -- $(seq "$lo" $((hi - 1)) | sed 's/^/3./')
|
||||
readarray -t versions < <(seq "$lo" $((hi - 1)) | sed 's/^/3./')
|
||||
set -- "${versions[@]}"
|
||||
fi
|
||||
|
||||
prefix="${DEEPGEMM_VENV_PREFIX:-/tmp/dgenv}"
|
||||
|
||||
@@ -15,6 +15,7 @@ if ! command -v numactl >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1001 # verified equivalent behavior; TODO: revisit escaping in a follow-up cleanup PR
|
||||
case "${_VLLM_INTERNAL_NUMACTL_ARGS}" in
|
||||
*[![:alnum:]\ \-\_=,./]*)
|
||||
echo "Invalid characters in _VLLM_INTERNAL_NUMACTL_ARGS" >&2
|
||||
@@ -22,4 +23,5 @@ case "${_VLLM_INTERNAL_NUMACTL_ARGS}" in
|
||||
;;
|
||||
esac
|
||||
|
||||
# shellcheck disable=SC2086 # word splitting is intentional: this expands into multiple numactl flags
|
||||
exec numactl ${_VLLM_INTERNAL_NUMACTL_ARGS} "${_VLLM_INTERNAL_NUMACTL_PYTHON_EXECUTABLE}" "$@"
|
||||
|
||||
Reference in New Issue
Block a user