diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 8ba73b536a..45e45cf33a 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -302,6 +302,8 @@ validate_ci_build_context_source() { local mode="" local stage="" local path="" + local worktree_diff_status=0 + local staged_diff_status=0 local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes" local info_attributes="" @@ -319,15 +321,46 @@ validate_ci_build_context_source() { return 1 fi ROCM_BUILD_CONTEXT_COMMIT="${context_commit}" - if ! git -C "${source_root}" diff \ - --quiet --no-ext-diff --ignore-submodules=none --; then + # Shared AMD workspaces may present tracked files with inflated executable + # bits. Context modes come from the pinned Git tree, so ignore only that + # filesystem drift while continuing to reject content changes. + git -C "${source_root}" -c core.fileMode=false diff \ + --quiet --no-ext-diff --ignore-submodules=none -- \ + || worktree_diff_status=$? + if (( worktree_diff_status != 0 )); then + if (( worktree_diff_status > 1 )); then + printf 'Failed to inspect tracked CI worktree changes (git diff exited %s)\n' \ + "${worktree_diff_status}" >&2 + return 1 + fi echo "Tracked worktree changes cannot be omitted from the CI Docker context" \ >&2 + echo "Tracked worktree diff (first 50 entries):" >&2 + git -C "${source_root}" -c core.fileMode=false diff \ + --no-ext-diff --ignore-submodules=none --name-status -- \ + | sed -n '1,50p' >&2 || true + git -C "${source_root}" -c core.fileMode=false diff \ + --no-ext-diff --ignore-submodules=none --summary -- \ + | sed -n '1,50p' >&2 || true return 1 fi - if ! git -C "${source_root}" diff --cached \ - --quiet --no-ext-diff --ignore-submodules=none HEAD --; then + git -C "${source_root}" diff --cached \ + --quiet --no-ext-diff --ignore-submodules=none HEAD -- \ + || staged_diff_status=$? + if (( staged_diff_status != 0 )); then + if (( staged_diff_status > 1 )); then + printf 'Failed to inspect staged CI changes (git diff exited %s)\n' \ + "${staged_diff_status}" >&2 + return 1 + fi echo "Staged changes cannot be omitted from the CI Docker context" >&2 + echo "Staged diff (first 50 entries):" >&2 + git -C "${source_root}" diff --cached \ + --no-ext-diff --ignore-submodules=none --name-status HEAD -- \ + | sed -n '1,50p' >&2 || true + git -C "${source_root}" diff --cached \ + --no-ext-diff --ignore-submodules=none --summary HEAD -- \ + | sed -n '1,50p' >&2 || true return 1 fi # Untracked and ignored worker outputs are intentionally absent: the diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 635b6f1de5..42fe3f4546 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -429,11 +429,15 @@ initialize_native_environment() { # datasets uses POSIX locks that are unsupported by the shared HF NFS cache. # Keep processed datasets job-local while retaining the persistent Hub cache. HF_DATASETS_CACHE="${native_root}/cache/huggingface/datasets" + # openai-harmony downloads its tiktoken vocab on first use and caches it under + # $TMPDIR by default, which is job-local; keep it with the persistent Hub cache. + TIKTOKEN_RS_CACHE_DIR="${HF_HOME}/tiktoken-rs-cache" : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" : "${HF_HUB_ETAG_TIMEOUT:=60}" export TMPDIR VLLM_RPC_BASE_PATH export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT + export TIKTOKEN_RS_CACHE_DIR export PYTORCH_ROCM_ARCH="" mkdir -p "${TMPDIR}" \ @@ -442,6 +446,7 @@ initialize_native_environment() { "${VLLM_CACHE_ROOT}" \ "${XDG_CACHE_HOME}" \ "${HF_HOME}" \ + "${TIKTOKEN_RS_CACHE_DIR}" \ "${HF_DATASETS_CACHE}" || return 1 echo "Native compile caches: VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT} TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR}" @@ -1092,7 +1097,7 @@ container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}" container_job_id_short="${container_job_id:0:8}" CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}" CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache" -CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\"" +CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TIKTOKEN_RS_CACHE_DIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\"" # Verify GPU access render_gid=$(getent group render | cut -d: -f3) @@ -1210,6 +1215,7 @@ else -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ -e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \ + -e "TIKTOKEN_RS_CACHE_DIR=${HF_MOUNT}/tiktoken-rs-cache" \ -e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \ -e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \ -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \