[CI][AMD] Persist the openai-harmony tiktoken vocab cache across jobs (#51666)

Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
stefankoncarevic
2026-08-11 06:11:06 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 608c12473f
commit 75903e5067
2 changed files with 44 additions and 5 deletions
+37 -4
View File
@@ -302,6 +302,8 @@ validate_ci_build_context_source() {
local mode=""
local stage=""
local path=""
local worktree_diff_status=0
local staged_diff_status=0
local index_modes_file="${SCRIPT_TMP_DIR}/git-index-modes"
local info_attributes=""
@@ -319,15 +321,46 @@ validate_ci_build_context_source() {
return 1
fi
ROCM_BUILD_CONTEXT_COMMIT="${context_commit}"
if ! git -C "${source_root}" diff \
--quiet --no-ext-diff --ignore-submodules=none --; then
# Shared AMD workspaces may present tracked files with inflated executable
# bits. Context modes come from the pinned Git tree, so ignore only that
# filesystem drift while continuing to reject content changes.
git -C "${source_root}" -c core.fileMode=false diff \
--quiet --no-ext-diff --ignore-submodules=none -- \
|| worktree_diff_status=$?
if (( worktree_diff_status != 0 )); then
if (( worktree_diff_status > 1 )); then
printf 'Failed to inspect tracked CI worktree changes (git diff exited %s)\n' \
"${worktree_diff_status}" >&2
return 1
fi
echo "Tracked worktree changes cannot be omitted from the CI Docker context" \
>&2
echo "Tracked worktree diff (first 50 entries):" >&2
git -C "${source_root}" -c core.fileMode=false diff \
--no-ext-diff --ignore-submodules=none --name-status -- \
| sed -n '1,50p' >&2 || true
git -C "${source_root}" -c core.fileMode=false diff \
--no-ext-diff --ignore-submodules=none --summary -- \
| sed -n '1,50p' >&2 || true
return 1
fi
if ! git -C "${source_root}" diff --cached \
--quiet --no-ext-diff --ignore-submodules=none HEAD --; then
git -C "${source_root}" diff --cached \
--quiet --no-ext-diff --ignore-submodules=none HEAD -- \
|| staged_diff_status=$?
if (( staged_diff_status != 0 )); then
if (( staged_diff_status > 1 )); then
printf 'Failed to inspect staged CI changes (git diff exited %s)\n' \
"${staged_diff_status}" >&2
return 1
fi
echo "Staged changes cannot be omitted from the CI Docker context" >&2
echo "Staged diff (first 50 entries):" >&2
git -C "${source_root}" diff --cached \
--no-ext-diff --ignore-submodules=none --name-status HEAD -- \
| sed -n '1,50p' >&2 || true
git -C "${source_root}" diff --cached \
--no-ext-diff --ignore-submodules=none --summary HEAD -- \
| sed -n '1,50p' >&2 || true
return 1
fi
# Untracked and ignored worker outputs are intentionally absent: the
@@ -429,11 +429,15 @@ initialize_native_environment() {
# datasets uses POSIX locks that are unsupported by the shared HF NFS cache.
# Keep processed datasets job-local while retaining the persistent Hub cache.
HF_DATASETS_CACHE="${native_root}/cache/huggingface/datasets"
# openai-harmony downloads its tiktoken vocab on first use and caches it under
# $TMPDIR by default, which is job-local; keep it with the persistent Hub cache.
TIKTOKEN_RS_CACHE_DIR="${HF_HOME}/tiktoken-rs-cache"
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
: "${HF_HUB_ETAG_TIMEOUT:=60}"
export TMPDIR VLLM_RPC_BASE_PATH
export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
export TIKTOKEN_RS_CACHE_DIR
export PYTORCH_ROCM_ARCH=""
mkdir -p "${TMPDIR}" \
@@ -442,6 +446,7 @@ initialize_native_environment() {
"${VLLM_CACHE_ROOT}" \
"${XDG_CACHE_HOME}" \
"${HF_HOME}" \
"${TIKTOKEN_RS_CACHE_DIR}" \
"${HF_DATASETS_CACHE}" || return 1
echo "Native compile caches: VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT} TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR}"
@@ -1092,7 +1097,7 @@ container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}"
container_job_id_short="${container_job_id:0:8}"
CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}"
CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache"
CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TIKTOKEN_RS_CACHE_DIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
# Verify GPU access
render_gid=$(getent group render | cut -d: -f3)
@@ -1210,6 +1215,7 @@ else
-e "HF_HOME=${HF_MOUNT}" \
-e "PYTHONPATH=${MYPYTHONPATH}" \
-e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \
-e "TIKTOKEN_RS_CACHE_DIR=${HF_MOUNT}/tiktoken-rs-cache" \
-e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \
-e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \