diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index eb8b53c6cc..86e4b34869 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -261,6 +261,8 @@ validate_native_workspace() { } prepare_native_workspace() { + local test_commands="${1:-}" + if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1" return 1 @@ -281,6 +283,8 @@ prepare_native_workspace() { local recorded_base="" local recorded_commit="" local recorded_wheel="" + local checkout="" + local checkout_commit="" local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" local wheel_dir="" local attempt=0 @@ -400,6 +404,53 @@ prepare_native_workspace() { return 1 fi + # The ROCm artifact intentionally contains only the installed wheel and the + # test workspace. The Python-only compilation job also needs setup.py and the + # vllm source tree, so overlay the matching Buildkite checkout for that job. + if [[ "${test_commands}" == *python_only_compile.sh* ]]; then + checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" + if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then + echo "Python-only native CI requires BUILDKITE_BUILD_CHECKOUT_PATH" >&2 + return 1 + fi + if ! git -c "safe.directory=${checkout}" -C "${checkout}" \ + rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "Buildkite checkout is not a Git worktree: ${checkout}" >&2 + return 1 + fi + checkout_commit=$( + git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse HEAD + ) || return 1 + if [[ "${checkout_commit}" != "${recorded_commit}" ]]; then + echo "Buildkite checkout ${checkout_commit} does not match ROCm artifact ${recorded_commit}" >&2 + return 1 + fi + + # setup.py normally derives this from .git via setuptools-scm. The native + # source overlay deliberately excludes Git metadata, so preserve the exact + # version from the already installed, artifact-matched wheel. + VLLM_VERSION_OVERRIDE=$( + python3 -c 'import importlib.metadata as m; print(m.version("vllm"))' + ) || return 1 + export VLLM_VERSION_OVERRIDE + VLLM_PRECOMPILED_WHEEL_LOCATION="${wheels[0]}" + export VLLM_PRECOMPILED_WHEEL_LOCATION + echo "INFO: native Python-only wheel=${VLLM_PRECOMPILED_WHEEL_LOCATION}" + + echo "--- Overlaying full source checkout for Python-only compilation" + # Archive the verified commit instead of copying the worktree so dirty or + # untracked agent files cannot contaminate the artifact-matched workspace. + git -c "safe.directory=${checkout}" -C "${checkout}" \ + archive --format=tar "${recorded_commit}" \ + | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1 + for required_source in setup.py pyproject.toml vllm; do + if [[ ! -e "${workspace_dir}/${required_source}" ]]; then + echo "Full source checkout is missing ${required_source}" >&2 + return 1 + fi + done + fi + return 0 } @@ -954,7 +1005,13 @@ if is_native_runtime; then echo "Failed to initialize the native test environment" exit 1 fi - if ! prepare_native_workspace; then + if [[ "${commands}" == *python_only_compile.sh* ]]; then + # This no-GPU job validates the ROCm precompiled/editable install path, + # rather than CPU runtime platform selection. + VLLM_TARGET_DEVICE=rocm + export VLLM_TARGET_DEVICE + fi + if ! prepare_native_workspace "${commands}"; then echo "Failed to prepare native test workspace" exit 1 fi diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index c1658e7a83..b477b298bb 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -411,8 +411,9 @@ steps: - label: Python-only Installation # TBD timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - agent_pool: mi250_1 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 126bc65704..8f168cbd36 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -275,7 +275,8 @@ steps: - bash standalone_tests/python_only_compile.sh mirror: amd: - device: mi250_1 + dind: false + device: mi300_1 timeout_in_minutes: 55 soft_fail: true depends_on: diff --git a/tests/standalone_tests/python_only_compile.sh b/tests/standalone_tests/python_only_compile.sh index 8e3d08d456..7036dd4314 100644 --- a/tests/standalone_tests/python_only_compile.sh +++ b/tests/standalone_tests/python_only_compile.sh @@ -4,37 +4,70 @@ set -e -# ROCm CI runs this script inside `run-amd-test.sh` where /vllm-workspace often has no .git -# (wheel artifact layout). The wrapper passes CI_STANDALONE_MERGE_BASE from the agent checkout. merge_base_commit="" -if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then - merge_base_commit="${CI_STANDALONE_MERGE_BASE}" -elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then - : -elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then - : -else - echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2 - exit 1 -fi - -echo "INFO: current merge base commit with main: $merge_base_commit" -if git show --oneline -s "$merge_base_commit" 2>/dev/null; then - : -else - echo "INFO: git show unavailable in this environment; using SHA above for precompiled metadata." -fi - -# test whether the metadata.json url is valid, retry each 3 minutes up to 5 times -# this avoids cumbersome error messages & manual retries in case the precompiled wheel -# for the given commit is still being built in the release pipeline +rocm_wheel="" +is_rocm=0 _vllm_target_lower="$(printf '%s' "${VLLM_TARGET_DEVICE:-}" | tr '[:upper:]' '[:lower:]')" -if [[ "${_vllm_target_lower}" == "rocm" ]] || [[ -d /opt/rocm ]] || command -v rocminfo >/dev/null 2>&1; then - _rocm_env_variant="$(python3 - <<'PY' +if [[ "${_vllm_target_lower}" == "rocm" || -n "${ROCM_PATH:-}" || -d /opt/rocm ]] \ + || command -v rocminfo >/dev/null 2>&1; then + is_rocm=1 +fi +unset -v _vllm_target_lower + +if [[ "${is_rocm}" == "1" ]]; then + # Native CI passes the verified wheel artifact explicitly. Legacy ROCm + # images carry the same-build wheel in /opt/vllm-wheels. + if [[ -n "${VLLM_PRECOMPILED_WHEEL_LOCATION:-}" ]]; then + rocm_wheel="${VLLM_PRECOMPILED_WHEEL_LOCATION}" + if [[ ! -f "${rocm_wheel}" || "$(basename "${rocm_wheel}")" != vllm-*.whl ]]; then + echo "ERROR: invalid ROCm wheel location: ${rocm_wheel}" >&2 + exit 1 + fi + rocm_wheel="$(realpath -- "${rocm_wheel}")" + elif [[ -d /opt/vllm-wheels ]]; then + shopt -s nullglob + rocm_wheels=(/opt/vllm-wheels/vllm-*.whl) + shopt -u nullglob + if [[ "${#rocm_wheels[@]}" -ne 1 ]]; then + echo "ERROR: expected exactly one vLLM wheel in /opt/vllm-wheels, found ${#rocm_wheels[@]}." >&2 + exit 1 + fi + rocm_wheel="${rocm_wheels[0]}" + fi +fi + +if [[ -n "${rocm_wheel}" ]]; then + echo "INFO: using same-build ROCm wheel: ${rocm_wheel}" +else + # Some CI images do not include .git under /vllm-workspace. Their wrapper + # passes CI_STANDALONE_MERGE_BASE from the agent checkout. + if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then + merge_base_commit="${CI_STANDALONE_MERGE_BASE}" + elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then + : + elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then + : + else + echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2 + exit 1 + fi + + echo "INFO: current merge base commit with main: $merge_base_commit" + if git show --oneline -s "$merge_base_commit" 2>/dev/null; then + : + else + echo "INFO: git show unavailable in this environment; using SHA above for precompiled metadata." + fi + + # Test whether the metadata.json URL is valid, retry each 5 minutes up to 5 times. + # This avoids manual retries while a new main-branch wheel is still publishing. + if [[ "${is_rocm}" == "1" ]]; then + _rocm_env_variant="$(python3 - <<'PY' import ctypes import os from pathlib import Path + def get_rocm_version() -> str | None: rocm_home = os.environ.get("ROCM_HOME") or os.environ.get("ROCM_PATH") or "/opt/rocm" try: @@ -54,76 +87,77 @@ def get_rocm_version() -> str | None: return None return None + version = get_rocm_version() if version: print(f"rocm{version.replace('.', '')}", end="") PY )" - _available_variants="$(curl -sf "https://wheels.vllm.ai/rocm/${merge_base_commit}/" \ - | grep -oP 'rocm\d+' | sort -u | tr '\n' ' ' || true)" - if [[ -n "${VLLM_PRECOMPILED_WHEEL_VARIANT:-}" ]]; then - _rocm_variant="${VLLM_PRECOMPILED_WHEEL_VARIANT}" - if [[ -n "${_rocm_env_variant}" && "${_rocm_variant}" != "${_rocm_env_variant}" ]]; then - echo "ERROR: VLLM_PRECOMPILED_WHEEL_VARIANT=${_rocm_variant} does not match detected environment ROCm variant ${_rocm_env_variant}" >&2 - exit 1 - fi - else - _rocm_variant="${_rocm_env_variant}" - fi - if [[ -z "${_rocm_variant}" ]]; then - echo "ERROR: Could not detect ROCm variant from the environment for commit ${merge_base_commit}" >&2 - exit 1 - fi - if [[ -z "${_available_variants}" ]] \ - || [[ " ${_available_variants} " != *" ${_rocm_variant} "* ]]; then - echo "ERROR: Environment ROCm variant '${_rocm_variant}' is not published for commit ${merge_base_commit} (available:${_available_variants:-none})" >&2 - exit 1 - fi - meta_json_url="https://wheels.vllm.ai/rocm/${merge_base_commit}/${_rocm_variant}/vllm/metadata.json" - unset -v _rocm_env_variant _available_variants _rocm_variant -else - meta_json_url="https://wheels.vllm.ai/${merge_base_commit}/vllm/metadata.json" -fi -unset -v _vllm_target_lower -echo "INFO: will use metadata.json from ${meta_json_url}" - -for i in {1..5}; do - echo "Checking metadata.json URL (attempt $i)..." - if curl --fail "$meta_json_url" > metadata.json; then - echo "INFO: metadata.json URL is valid." - # check whether it is valid json by python (printed to stdout) - if python3 -m json.tool metadata.json; then - echo "INFO: metadata.json is valid JSON. Proceeding with the check." - # check whether there is an object in the json matching: - # "package_name": "vllm", and "platform_tag" matches the current architecture - # see `determine_wheel_url` in setup.py for more details - if python3 -c "import platform as p,json as j,sys as s; d = j.load(open('metadata.json')); \ - s.exit(int(not any(o.get('package_name') == 'vllm' and p.machine() in o.get('platform_tag') \ - for o in d)))" 2>/dev/null; then - echo "INFO: metadata.json contains a pre-compiled wheel for the current architecture." - break - else - echo "WARN: metadata.json does not have a pre-compiled wheel for the current architecture." + _available_variants="$(curl -sf "https://wheels.vllm.ai/rocm/${merge_base_commit}/" \ + | grep -oP 'rocm\d+' | sort -u | tr '\n' ' ' || true)" + if [[ -n "${VLLM_PRECOMPILED_WHEEL_VARIANT:-}" ]]; then + _rocm_variant="${VLLM_PRECOMPILED_WHEEL_VARIANT}" + if [[ -n "${_rocm_env_variant}" && "${_rocm_variant}" != "${_rocm_env_variant}" ]]; then + echo "ERROR: VLLM_PRECOMPILED_WHEEL_VARIANT=${_rocm_variant} does not match detected environment ROCm variant ${_rocm_env_variant}" >&2 + exit 1 fi else - echo "CRITICAL: metadata.json exists but is not valid JSON, please do report in #sig-ci channel!" - echo "INFO: metadata.json content:" - cat metadata.json + _rocm_variant="${_rocm_env_variant}" + fi + if [[ -z "${_rocm_variant}" ]]; then + echo "ERROR: Could not detect ROCm variant from the environment for commit ${merge_base_commit}" >&2 exit 1 fi - fi - # failure handling & retry logic - if [ "$i" -eq 5 ]; then - echo "ERROR: metadata is still not available after 5 attempts." - echo "ERROR: Please check whether the precompiled wheel for commit $merge_base_commit is available." - echo " NOTE: If $merge_base_commit is a new commit on main, maybe try again after its release pipeline finishes." - echo " NOTE: If it fails, please report in #sig-ci channel." - exit 1 + if [[ -z "${_available_variants}" ]] \ + || [[ " ${_available_variants} " != *" ${_rocm_variant} "* ]]; then + echo "ERROR: Environment ROCm variant '${_rocm_variant}' is not published for commit ${merge_base_commit} (available:${_available_variants:-none})" >&2 + exit 1 + fi + meta_json_url="https://wheels.vllm.ai/rocm/${merge_base_commit}/${_rocm_variant}/vllm/metadata.json" + unset -v _rocm_env_variant _available_variants _rocm_variant else - echo "WARNING: metadata is not available. Retrying after 5 minutes..." - sleep 300 + meta_json_url="https://wheels.vllm.ai/${merge_base_commit}/vllm/metadata.json" fi -done + echo "INFO: will use metadata.json from ${meta_json_url}" + + for i in {1..5}; do + echo "Checking metadata.json URL (attempt $i)..." + if curl --fail "$meta_json_url" > metadata.json; then + echo "INFO: metadata.json URL is valid." + # check whether it is valid json by python (printed to stdout) + if python3 -m json.tool metadata.json; then + echo "INFO: metadata.json is valid JSON. Proceeding with the check." + # check whether there is an object in the json matching: + # "package_name": "vllm", and "platform_tag" matches the current architecture + # see `determine_wheel_url` in setup.py for more details + if python3 -c "import platform as p,json as j,sys as s; d = j.load(open('metadata.json')); \ + s.exit(int(not any(o.get('package_name') == 'vllm' and p.machine() in o.get('platform_tag') \ + for o in d)))" 2>/dev/null; then + echo "INFO: metadata.json contains a pre-compiled wheel for the current architecture." + break + else + echo "WARN: metadata.json does not have a pre-compiled wheel for the current architecture." + fi + else + echo "CRITICAL: metadata.json exists but is not valid JSON, please do report in #sig-ci channel!" + echo "INFO: metadata.json content:" + cat metadata.json + exit 1 + fi + fi + # failure handling & retry logic + if [ "$i" -eq 5 ]; then + echo "ERROR: metadata is still not available after 5 attempts." + echo "ERROR: Please check whether the precompiled wheel for commit $merge_base_commit is available." + echo " NOTE: If $merge_base_commit is a new commit on main, maybe try again after its release pipeline finishes." + echo " NOTE: If it fails, please report in #sig-ci channel." + exit 1 + else + echo "WARNING: metadata is not available. Retrying after 5 minutes..." + sleep 300 + fi + done +fi set -x @@ -143,16 +177,17 @@ fi apt remove --purge build-essential -y apt autoremove -y +rm -f /tmp/changed.file echo 'import os; os.system("touch /tmp/changed.file")' >> vllm/__init__.py # ROCm CI uses setuptools develop for editable installs (see Dockerfile.rocm and run-amd-test.sh). -_vllm_target_lower="$(printf '%s' "${VLLM_TARGET_DEVICE:-}" | tr '[:upper:]' '[:lower:]')" -if [[ "${_vllm_target_lower}" == "rocm" ]]; then - VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 python3 setup.py develop +if [[ -n "${rocm_wheel}" ]]; then + VLLM_PRECOMPILED_WHEEL_LOCATION="${rocm_wheel}" VLLM_USE_PRECOMPILED=1 python3 setup.py develop --no-deps +elif [[ "${is_rocm}" == "1" ]]; then + VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 python3 setup.py develop --no-deps else - VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 pip3 install -vvv -e . + VLLM_PRECOMPILED_WHEEL_COMMIT=$merge_base_commit VLLM_USE_PRECOMPILED=1 pip3 install -vvv -e . fi -unset -v _vllm_target_lower # Run the script python3 -c 'import vllm'