[CI][AMD] Configure MI300 tests for native execution without DinD (#48387)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -1898,8 +1898,8 @@ confirm_remote_image_push() {
|
||||
fi
|
||||
|
||||
if [[ -z "${remote_revision}" \
|
||||
&& ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 \
|
||||
&& image_tag_is_commit_scoped ]]; then
|
||||
&& ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \
|
||||
&& image_tag_is_commit_scoped; then
|
||||
echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label."
|
||||
return 0
|
||||
fi
|
||||
@@ -2029,36 +2029,57 @@ upload_wheel_artifacts_if_present() {
|
||||
local wheel_dir="./wheel-export"
|
||||
local artifact_dir="artifacts/vllm-rocm-install"
|
||||
local archive_name="vllm-rocm-install.tar.gz"
|
||||
local metadata_dir="${wheel_dir}/.vllm-ci-artifact"
|
||||
local native_base_image=""
|
||||
local whl=""
|
||||
local whl_name=""
|
||||
local -a wheels=()
|
||||
|
||||
if ! should_upload_wheel_artifacts; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ ! -d "${wheel_dir}" ]] || ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then
|
||||
echo "No ROCm wheel artifacts found in ${wheel_dir}"
|
||||
return 0
|
||||
if [[ -d "${wheel_dir}" ]]; then
|
||||
mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print)
|
||||
fi
|
||||
if [[ ${#wheels[@]} -ne 1 ]]; then
|
||||
echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
whl="${wheels[0]}"
|
||||
whl_name=$(basename "${whl}")
|
||||
native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}"
|
||||
if [[ -z "${native_base_image}" ]]; then
|
||||
echo "Native ROCm artifact requires a ci_base image reference" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "--- :package: Uploading ROCm vLLM install artifact"
|
||||
mkdir -p "${artifact_dir}"
|
||||
rm -rf "${artifact_dir}" "${metadata_dir}"
|
||||
mkdir -p "${artifact_dir}" "${metadata_dir}"
|
||||
|
||||
printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt"
|
||||
printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt"
|
||||
printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt"
|
||||
printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt"
|
||||
printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt"
|
||||
|
||||
tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" .
|
||||
(
|
||||
cd "${artifact_dir}"
|
||||
sha256sum "${archive_name}" > "${archive_name}.sha256"
|
||||
)
|
||||
echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)"
|
||||
printf '%s\n' "${CI_BASE_IMAGE:-}" > "${artifact_dir}/ci-base-image.txt"
|
||||
printf '%s\n' "${IMAGE_TAG:-}" > "${artifact_dir}/fallback-image.txt"
|
||||
|
||||
for whl in "${wheel_dir}"/*.whl; do
|
||||
[[ -f "${whl}" ]] || continue
|
||||
whl_name=$(basename "${whl}")
|
||||
cp "${whl}" "${artifact_dir}/${whl_name}"
|
||||
echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
|
||||
done
|
||||
cp "${metadata_dir}"/*.txt "${artifact_dir}/"
|
||||
cp "${whl}" "${artifact_dir}/${whl_name}"
|
||||
echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
|
||||
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent artifact upload "${artifact_dir}/*"
|
||||
buildkite-agent artifact upload "${artifact_dir}/*" || return 1
|
||||
echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/"
|
||||
elif [[ "${BUILDKITE:-false}" == "true" ]]; then
|
||||
echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2
|
||||
return 1
|
||||
else
|
||||
echo "Not in Buildkite, skipping artifact upload"
|
||||
fi
|
||||
@@ -2090,6 +2111,11 @@ main() {
|
||||
echo "BAKE_PRINT_ONLY=1 set; skipping build"
|
||||
return 0
|
||||
fi
|
||||
if should_upload_wheel_artifacts; then
|
||||
# wheel-export is an output directory, not a BuildKit cache. Starting
|
||||
# clean prevents a failed/retried export from packaging a stale wheel.
|
||||
rm -rf ./wheel-export
|
||||
fi
|
||||
seed_dependency_caches_if_needed
|
||||
run_bake
|
||||
upload_wheel_artifacts_if_present
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script runs tests inside the corresponding ROCm docker container.
|
||||
# It handles both single-node and multi-node test configurations.
|
||||
# This script runs ROCm tests either directly in a native CI pod or inside the
|
||||
# corresponding Docker container. Multi-node tests continue to use Docker.
|
||||
#
|
||||
# Multi-node detection: Instead of matching on fragile group names, we detect
|
||||
# multi-node jobs structurally by looking for the bracket command syntax
|
||||
@@ -34,10 +34,27 @@ set -o pipefail
|
||||
: "${CLICOLOR_FORCE:=1}"
|
||||
: "${PY_COLORS:=1}"
|
||||
: "${ROCM_DOCKER_TTY:=1}"
|
||||
: "${PYTHONFAULTHANDLER:=1}"
|
||||
: "${PYTEST_TIMEOUT:=2100}"
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
|
||||
fi
|
||||
export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS ROCM_DOCKER_TTY
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25"
|
||||
fi
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0"
|
||||
fi
|
||||
# Dump stacks after 15 minutes, then stop an individual test after 35 minutes.
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900"
|
||||
fi
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* &&
|
||||
" ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread"
|
||||
fi
|
||||
export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY
|
||||
export PYTHONFAULTHANDLER
|
||||
|
||||
# Export Python path for commands that run directly on the host. Containerized
|
||||
# tests set this to /vllm-workspace below so spawned Python processes do not
|
||||
@@ -53,6 +70,28 @@ report_docker_usage() {
|
||||
docker system df || true
|
||||
}
|
||||
|
||||
clear_ci_orchestration_env() {
|
||||
unset -v \
|
||||
VLLM_TEST_GROUP_NAME \
|
||||
VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \
|
||||
VLLM_CI_ARTIFACT_STEP \
|
||||
VLLM_TEST_CACHE \
|
||||
VLLM_CI_EXECUTION_MODE \
|
||||
VLLM_CI_WORKSPACE \
|
||||
VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
|
||||
VLLM_TEST_COMMANDS \
|
||||
VLLM_CI_BRANCH \
|
||||
VLLM_CI_BASE_IMAGE \
|
||||
VLLM_CI_FALLBACK_IMAGE \
|
||||
VLLM_CI_DOCKER_DISABLED \
|
||||
VLLM_CI_ARTIFACT_GLOB \
|
||||
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
|
||||
VLLM_CI_EXPECTED_GPU_COUNT \
|
||||
VLLM_CI_USE_ARTIFACTS \
|
||||
VLLM_CI_RESULTS_ROOT \
|
||||
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
|
||||
}
|
||||
|
||||
cleanup_network() {
|
||||
local max_nodes=${NUM_NODES:-2}
|
||||
for node in $(seq 0 $((max_nodes - 1))); do
|
||||
@@ -145,7 +184,11 @@ prepare_artifact_image() {
|
||||
fi
|
||||
|
||||
cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1
|
||||
tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \
|
||||
tar -C "${wheel_dir}" \
|
||||
--exclude='*.whl' \
|
||||
--exclude='.vllm-ci-artifact' \
|
||||
--exclude='./.vllm-ci-artifact' \
|
||||
-cf - . \
|
||||
| tar -C "${workspace_dir}" -xf - || return 1
|
||||
cat > "${context_dir}/Dockerfile" <<'EOF'
|
||||
ARG BASE_IMAGE
|
||||
@@ -168,6 +211,257 @@ EOF
|
||||
return 0
|
||||
}
|
||||
|
||||
is_native_runtime() {
|
||||
[[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]]
|
||||
}
|
||||
|
||||
validate_native_workspace() {
|
||||
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
local workspace_real=""
|
||||
local checkout_real=""
|
||||
local workspace_mount=""
|
||||
|
||||
mkdir -p "${workspace_dir}" || return 1
|
||||
workspace_real=$(readlink -m "${workspace_dir}") || return 1
|
||||
if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then
|
||||
checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1
|
||||
if [[ "${checkout_real}" == "${workspace_real}" \
|
||||
|| "${checkout_real}" == "${workspace_real}/"* \
|
||||
|| "${workspace_real}" == "${checkout_real}/"* ]]; then
|
||||
echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then
|
||||
if ! command -v findmnt >/dev/null 2>&1; then
|
||||
echo "findmnt is required to verify the native workspace mount" >&2
|
||||
return 1
|
||||
fi
|
||||
workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true)
|
||||
if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then
|
||||
echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
prepare_native_workspace() {
|
||||
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
|
||||
echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
|
||||
return 1
|
||||
fi
|
||||
if ! command -v buildkite-agent >/dev/null 2>&1; then
|
||||
echo "buildkite-agent not found; cannot download ROCm wheel artifact"
|
||||
return 1
|
||||
fi
|
||||
validate_native_workspace || return 1
|
||||
|
||||
local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}"
|
||||
local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}"
|
||||
local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}"
|
||||
local archive=""
|
||||
local checksum=""
|
||||
local download_dir=""
|
||||
local metadata_dir=""
|
||||
local recorded_base=""
|
||||
local recorded_commit=""
|
||||
local recorded_wheel=""
|
||||
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
local wheel_dir=""
|
||||
local attempt=0
|
||||
local attempt_dir=""
|
||||
local -a archives=()
|
||||
local -a checksums=()
|
||||
local -a wheels=()
|
||||
|
||||
artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1
|
||||
wheel_dir="${artifact_work_dir}/wheels"
|
||||
mkdir -p "${wheel_dir}" || return 1
|
||||
|
||||
echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)"
|
||||
for attempt in 1 2 3; do
|
||||
attempt_dir="${artifact_work_dir}/download-${attempt}"
|
||||
rm -rf "${attempt_dir}" || return 1
|
||||
mkdir -p "${attempt_dir}" || return 1
|
||||
if buildkite-agent artifact download \
|
||||
"${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \
|
||||
&& buildkite-agent artifact download \
|
||||
"${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then
|
||||
download_dir="${attempt_dir}"
|
||||
break
|
||||
fi
|
||||
echo "Artifact download attempt ${attempt}/3 failed"
|
||||
if [[ "${attempt}" -lt 3 ]]; then
|
||||
sleep $((attempt * 2))
|
||||
fi
|
||||
done
|
||||
if [[ -z "${download_dir}" ]]; then
|
||||
echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
mapfile -t archives < <(
|
||||
find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print
|
||||
)
|
||||
mapfile -t checksums < <(
|
||||
find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print
|
||||
)
|
||||
if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then
|
||||
echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2
|
||||
return 1
|
||||
fi
|
||||
archive="${archives[0]}"
|
||||
checksum="${checksums[0]}"
|
||||
if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then
|
||||
echo "ROCm archive and checksum were downloaded to different directories" >&2
|
||||
return 1
|
||||
fi
|
||||
(
|
||||
cd "$(dirname "${archive}")"
|
||||
sha256sum -c "$(basename "${checksum}")"
|
||||
) || return 1
|
||||
|
||||
tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1
|
||||
mapfile -t wheels < <(
|
||||
find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print
|
||||
)
|
||||
if [[ ${#wheels[@]} -ne 1 ]]; then
|
||||
echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
metadata_dir="${wheel_dir}/.vllm-ci-artifact"
|
||||
for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do
|
||||
if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then
|
||||
echo "ROCm artifact metadata is missing ${metadata_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
for metadata_file in ci-base-image.txt fallback-image.txt; do
|
||||
if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then
|
||||
echo "ROCm artifact metadata is missing ${metadata_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt")
|
||||
recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt")
|
||||
recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt")
|
||||
if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then
|
||||
echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
|
||||
echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then
|
||||
echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2
|
||||
return 1
|
||||
fi
|
||||
for required_dir in tests .buildkite requirements; do
|
||||
if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then
|
||||
echo "ROCm wheel artifact did not contain ${required_dir}/" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- Installing ROCm wheel into pod environment"
|
||||
python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1
|
||||
|
||||
echo "--- Preparing ${workspace_dir} from artifact"
|
||||
find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1
|
||||
tar -C "${wheel_dir}" \
|
||||
--exclude='*.whl' \
|
||||
--exclude='.vllm-ci-artifact' \
|
||||
--exclude='./.vllm-ci-artifact' \
|
||||
-cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
|
||||
if [[ ! -d "${workspace_dir}/tests" ]]; then
|
||||
echo "Failed to stage the native test workspace" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
initialize_native_environment() {
|
||||
local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
|
||||
local job_id_suffix=""
|
||||
local native_root=""
|
||||
local hf_mount=""
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "Native ROCm CI currently requires the ci_base container to run as root" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
|
||||
job_id_suffix="${job_id##*-}"
|
||||
job_id_suffix="${job_id_suffix:0:12}"
|
||||
native_root="/tmp/vllm-native-${job_id}"
|
||||
TMPDIR="/tmp/vllm-${job_id_suffix}/tmp"
|
||||
VLLM_RPC_BASE_PATH="/tmp"
|
||||
: "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}"
|
||||
: "${TRITON_CACHE_DIR:=${native_root}/cache/triton}"
|
||||
: "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}"
|
||||
: "${XDG_CACHE_HOME:=${native_root}/cache/xdg}"
|
||||
: "${HF_HOME:=/home/buildkite-agent/huggingface}"
|
||||
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
|
||||
: "${HF_HUB_ETAG_TIMEOUT:=60}"
|
||||
export TMPDIR VLLM_RPC_BASE_PATH
|
||||
export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
|
||||
export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
|
||||
export PYTORCH_ROCM_ARCH=""
|
||||
|
||||
mkdir -p "${TMPDIR}" \
|
||||
"${TORCHINDUCTOR_CACHE_DIR}" \
|
||||
"${TRITON_CACHE_DIR}" \
|
||||
"${VLLM_CACHE_ROOT}" \
|
||||
"${XDG_CACHE_HOME}" \
|
||||
"${HF_HOME}" || return 1
|
||||
|
||||
if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
|
||||
if ! command -v findmnt >/dev/null 2>&1; then
|
||||
echo "findmnt is required to verify the native Hugging Face cache mount" >&2
|
||||
return 1
|
||||
fi
|
||||
hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
|
||||
if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
|
||||
echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
run_native_preflight() {
|
||||
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
|
||||
|
||||
if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then
|
||||
echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1
|
||||
|
||||
if [[ "${expected_gpus}" == "0" ]]; then
|
||||
echo "Native CPU-only AMD job: skipping ROCm device validation"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "--- ROCm info"
|
||||
rocminfo || return 1
|
||||
VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY'
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"])
|
||||
assert torch.version.hip, "PyTorch is not a ROCm build"
|
||||
assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
|
||||
actual = torch.cuda.device_count()
|
||||
assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
|
||||
PY
|
||||
}
|
||||
|
||||
is_multi_node() {
|
||||
local cmds="$1"
|
||||
# Primary signal: NUM_NODES environment variable set by the pipeline
|
||||
@@ -350,7 +644,58 @@ re_quote_pytest_markers() {
|
||||
# Main
|
||||
###############################################################################
|
||||
|
||||
# --- GPU initialization ---
|
||||
if is_native_runtime; then
|
||||
echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
|
||||
artifact_work_dir=""
|
||||
|
||||
cleanup_native_workspace() {
|
||||
if [[ -n "${artifact_work_dir}" ]]; then
|
||||
rm -rf "${artifact_work_dir}"
|
||||
fi
|
||||
}
|
||||
trap cleanup_native_workspace EXIT
|
||||
|
||||
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
|
||||
commands="${VLLM_TEST_COMMANDS}"
|
||||
commands_source="env"
|
||||
else
|
||||
commands="$*"
|
||||
commands_source="argv"
|
||||
if [[ -z "$commands" ]]; then
|
||||
echo "Error: No test commands provided for native CI." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$commands_source" == "argv" ]]; then
|
||||
commands=$(re_quote_pytest_markers "$commands")
|
||||
fi
|
||||
|
||||
if is_multi_node "$commands"; then
|
||||
echo "Native CI does not support multi-node jobs yet."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! initialize_native_environment; then
|
||||
echo "Failed to initialize the native test environment"
|
||||
exit 1
|
||||
fi
|
||||
if ! prepare_native_workspace; then
|
||||
echo "Failed to prepare native test workspace"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
|
||||
echo "Native test commands: $commands"
|
||||
run_native_preflight || exit 1
|
||||
# Keep AMD CI orchestration variables out of vLLM's runtime environment.
|
||||
clear_ci_orchestration_env
|
||||
/bin/bash -o pipefail -c "${commands}"
|
||||
handle_pytest_exit "$?"
|
||||
fi
|
||||
|
||||
# --- GPU initialization for legacy Docker execution ---
|
||||
echo "--- ROCm info"
|
||||
rocminfo
|
||||
|
||||
@@ -452,25 +797,27 @@ fi
|
||||
|
||||
echo "Final commands: $commands"
|
||||
|
||||
# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack).
|
||||
# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for
|
||||
# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container.
|
||||
vllm_standalone_merge_base=""
|
||||
checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
|
||||
if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
|
||||
checkout="."
|
||||
standalone_merge_base_env=()
|
||||
if [[ "$commands" == *python_only_compile.sh* ]]; then
|
||||
# The ROCm test image often ships /vllm-workspace without .git. Resolve the
|
||||
# wheels.vllm.ai commit from the agent checkout for this test only.
|
||||
vllm_standalone_merge_base=""
|
||||
checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
|
||||
if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
|
||||
checkout="."
|
||||
fi
|
||||
# Pass safe.directory per-command because Buildkite uses mixed user IDs.
|
||||
if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
vllm_standalone_merge_base="$(
|
||||
git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
|
||||
)"
|
||||
fi
|
||||
if [[ -z "${vllm_standalone_merge_base}" ]]; then
|
||||
vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
|
||||
fi
|
||||
echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
|
||||
standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}")
|
||||
fi
|
||||
# Pass safe.directory per-command (-c) because buildkite runs will always fail
|
||||
# the next check on git 2.35.2+ due to mixed uses of root and buildkite-agent/uids.
|
||||
if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
vllm_standalone_merge_base="$(
|
||||
git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
|
||||
)"
|
||||
fi
|
||||
if [[ -z "${vllm_standalone_merge_base}" ]]; then
|
||||
vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
|
||||
fi
|
||||
echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
|
||||
|
||||
MYPYTHONPATH="/vllm-workspace"
|
||||
|
||||
@@ -501,6 +848,7 @@ else
|
||||
fi
|
||||
|
||||
# --- Route: multi-node vs single-node ---
|
||||
clear_ci_orchestration_env
|
||||
if is_multi_node "$commands"; then
|
||||
echo "--- Multi-node job detected"
|
||||
export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
|
||||
@@ -589,7 +937,9 @@ else
|
||||
-e FORCE_COLOR \
|
||||
-e CLICOLOR_FORCE \
|
||||
-e PY_COLORS \
|
||||
-e PYTHONFAULTHANDLER \
|
||||
-e PYTEST_ADDOPTS \
|
||||
-e PYTEST_TIMEOUT \
|
||||
-v "${HF_CACHE}:${HF_MOUNT}" \
|
||||
-e "HF_HOME=${HF_MOUNT}" \
|
||||
-e "PYTHONPATH=${MYPYTHONPATH}" \
|
||||
@@ -599,7 +949,7 @@ else
|
||||
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
|
||||
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
|
||||
-e "PYTORCH_ROCM_ARCH=" \
|
||||
-e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \
|
||||
"${standalone_merge_base_env[@]}" \
|
||||
--name "${container_name}" \
|
||||
"${image_name}" \
|
||||
/bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
|
||||
|
||||
Reference in New Issue
Block a user