[ROCm][CI] Cache Rust builds by source inputs (#46527)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
co-authored by
Matthew Wong
parent
36484e464a
commit
b7b58d1eba
@@ -15,9 +15,9 @@ set -euo pipefail
|
|||||||
|
|
||||||
DEFAULT_REPO_SLUG="vllm-project/vllm"
|
DEFAULT_REPO_SLUG="vllm-project/vllm"
|
||||||
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
|
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
|
||||||
DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
|
DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
|
||||||
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
|
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
|
||||||
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
|
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_rixl build_rocshmem build_deepep mori_base ci_base"
|
||||||
DEFAULT_CI_BASE_METADATA_VERSION="1"
|
DEFAULT_CI_BASE_METADATA_VERSION="1"
|
||||||
IMAGE_EXISTED_BEFORE_BUILD=0
|
IMAGE_EXISTED_BEFORE_BUILD=0
|
||||||
|
|
||||||
@@ -764,7 +764,7 @@ configure_ci_base_image_refs() {
|
|||||||
fi
|
fi
|
||||||
set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}"
|
set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}"
|
||||||
set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}"
|
set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}"
|
||||||
set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT:-}"
|
set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}"
|
||||||
set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
|
set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
@@ -1211,12 +1211,24 @@ uses_rocm_csrc_cache() {
|
|||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
uses_rocm_rust_cache() {
|
||||||
|
case "${TARGET}" in
|
||||||
|
rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
compute_rocm_csrc_content_hash() {
|
compute_rocm_csrc_content_hash() {
|
||||||
local bake_dir=""
|
local bake_dir=""
|
||||||
local dockerfile_rocm=""
|
local dockerfile_rocm=""
|
||||||
local -a content_paths=(
|
local -a content_paths=(
|
||||||
"requirements/common.txt"
|
"requirements/common.txt"
|
||||||
"requirements/rocm.txt"
|
"requirements/rocm.txt"
|
||||||
|
"pyproject.toml"
|
||||||
"setup.py"
|
"setup.py"
|
||||||
"CMakeLists.txt"
|
"CMakeLists.txt"
|
||||||
"cmake"
|
"cmake"
|
||||||
@@ -1260,6 +1272,56 @@ compute_rocm_csrc_content_hash_if_needed() {
|
|||||||
echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
|
echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
compute_rocm_rust_content_hash() {
|
||||||
|
local bake_dir=""
|
||||||
|
local dockerfile_rocm=""
|
||||||
|
local -a content_paths=(
|
||||||
|
"requirements/build/rust.txt"
|
||||||
|
"rust/Cargo.lock"
|
||||||
|
"rust/Cargo.toml"
|
||||||
|
"rust/proto"
|
||||||
|
"rust/src"
|
||||||
|
"rust-toolchain.toml"
|
||||||
|
"tools/build_rust.py"
|
||||||
|
"tools/install_protoc.sh"
|
||||||
|
"build_rust.sh"
|
||||||
|
)
|
||||||
|
local -a content_args=()
|
||||||
|
|
||||||
|
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||||
|
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||||
|
mapfile -t content_args < <(
|
||||||
|
get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
|
||||||
|
)
|
||||||
|
|
||||||
|
{
|
||||||
|
printf 'rust-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
|
||||||
|
printf 'dockerfile:%s\n' "${dockerfile_rocm}"
|
||||||
|
printf 'resolved-build-args:\n'
|
||||||
|
hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
|
||||||
|
printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n'
|
||||||
|
if [[ -f "${dockerfile_rocm}" ]]; then
|
||||||
|
hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build"
|
||||||
|
else
|
||||||
|
printf 'missing:%s\n' "${dockerfile_rocm}"
|
||||||
|
fi
|
||||||
|
} | sha256sum | cut -d' ' -f1
|
||||||
|
}
|
||||||
|
|
||||||
|
compute_rocm_rust_content_hash_if_needed() {
|
||||||
|
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
|
||||||
|
|
||||||
|
if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash)
|
||||||
|
ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
|
||||||
|
export ROCM_RUST_CONTENT_HASH
|
||||||
|
export ROCM_RUST_CONTENT_CACHE_REF
|
||||||
|
echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}"
|
||||||
|
}
|
||||||
|
|
||||||
write_hcl_string_list_entries() {
|
write_hcl_string_list_entries() {
|
||||||
local indent="$1"
|
local indent="$1"
|
||||||
local value=""
|
local value=""
|
||||||
@@ -1317,6 +1379,7 @@ write_rocm_build_arg_override() {
|
|||||||
"${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
|
"${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
|
||||||
"${CI_BASE_CONTENT_ARGS:-}"
|
"${CI_BASE_CONTENT_ARGS:-}"
|
||||||
get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
|
get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
|
||||||
|
get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
|
||||||
} | awk 'NF && !seen[$0]++'
|
} | awk 'NF && !seen[$0]++'
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1365,46 +1428,133 @@ validate_cache_export_mode() {
|
|||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validate_content_cache_export_mode() {
|
||||||
|
local mode="$1"
|
||||||
|
local env_name="$2"
|
||||||
|
|
||||||
|
case "${mode}" in
|
||||||
|
missing|always|never)
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Error: ${env_name} must be one of: missing, always, never"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
should_export_content_cache_ref() {
|
||||||
|
local cache_ref="$1"
|
||||||
|
local cache_name="$2"
|
||||||
|
local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
|
||||||
|
|
||||||
|
case "${mode}" in
|
||||||
|
always)
|
||||||
|
echo "${cache_name} content cache export mode is always; exporting ${cache_ref}"
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
never)
|
||||||
|
echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}"
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
missing|"")
|
||||||
|
if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then
|
||||||
|
echo "${cache_name} content cache exists; not re-exporting ${cache_ref}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "${cache_name} content cache missing; will export ${cache_ref}"
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "Error: ROCM_CONTENT_CACHE_EXPORT_MODE must be one of: missing, always, never"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
write_rocm_cache_override() {
|
write_rocm_cache_override() {
|
||||||
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
|
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
|
||||||
|
local content_cache_export_mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
|
||||||
local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}"
|
local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}"
|
||||||
|
local rust_cache_to_mode="${ROCM_RUST_CACHE_TO_MODE:-max}"
|
||||||
local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}"
|
local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}"
|
||||||
local -a content_cache_from=()
|
local -a csrc_content_cache_from=()
|
||||||
|
local -a rust_content_cache_from=()
|
||||||
|
local -a combined_content_cache_from=()
|
||||||
local -a csrc_cache_to=()
|
local -a csrc_cache_to=()
|
||||||
|
local -a rust_cache_to=()
|
||||||
local -a rocm_cache_to=()
|
local -a rocm_cache_to=()
|
||||||
local -a export_wheel_cache_to=()
|
local -a export_wheel_cache_to=()
|
||||||
|
local export_csrc_cache=1
|
||||||
|
local export_rust_cache=1
|
||||||
|
|
||||||
if ! uses_rocm_csrc_cache; then
|
if ! uses_rocm_csrc_cache && ! uses_rocm_rust_cache; then
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
validate_content_cache_export_mode \
|
||||||
|
"${content_cache_export_mode}" \
|
||||||
|
"ROCM_CONTENT_CACHE_EXPORT_MODE"
|
||||||
validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE"
|
validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE"
|
||||||
|
validate_cache_export_mode "${rust_cache_to_mode}" "ROCM_RUST_CACHE_TO_MODE"
|
||||||
validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
|
validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
|
||||||
|
echo "ROCm content cache export mode: ${content_cache_export_mode}"
|
||||||
echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
|
echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
|
||||||
|
echo "ROCm Rust cache export mode: ${rust_cache_to_mode}"
|
||||||
echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
|
echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
|
||||||
|
|
||||||
if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
|
if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
|
||||||
content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
|
csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
|
||||||
csrc_cache_to+=(
|
if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then
|
||||||
"type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
|
csrc_cache_to+=(
|
||||||
)
|
"type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
export_csrc_cache=0
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then
|
||||||
|
rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}")
|
||||||
|
if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then
|
||||||
|
rust_cache_to+=(
|
||||||
|
"type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
export_rust_cache=0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}")
|
||||||
|
|
||||||
# Docker Hub cache exports are best-effort. A cache-only target failure can
|
# Docker Hub cache exports are best-effort. A cache-only target failure can
|
||||||
# otherwise cancel the sibling image target before its manifest is pushed.
|
# otherwise cancel the sibling image target before its manifest is pushed.
|
||||||
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
|
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
|
||||||
csrc_cache_to+=(
|
if [[ ${export_csrc_cache} -eq 1 ]]; then
|
||||||
"type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
|
csrc_cache_to+=(
|
||||||
)
|
"type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
if [[ ${export_rust_cache} -eq 1 ]]; then
|
||||||
|
rust_cache_to+=(
|
||||||
|
"type=registry,ref=${cache_repo}:rust-rocm-${BUILDKITE_COMMIT},mode=${rust_cache_to_mode},ignore-error=true"
|
||||||
|
)
|
||||||
|
fi
|
||||||
rocm_cache_to+=(
|
rocm_cache_to+=(
|
||||||
"type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true"
|
"type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true"
|
||||||
)
|
)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
|
if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
|
||||||
csrc_cache_to+=(
|
if [[ ${export_csrc_cache} -eq 1 ]]; then
|
||||||
"type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
|
csrc_cache_to+=(
|
||||||
)
|
"type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
if [[ ${export_rust_cache} -eq 1 ]]; then
|
||||||
|
rust_cache_to+=(
|
||||||
|
"type=registry,ref=${cache_repo}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rust_cache_to_mode},ignore-error=true"
|
||||||
|
)
|
||||||
|
fi
|
||||||
rocm_cache_to+=(
|
rocm_cache_to+=(
|
||||||
"type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true"
|
"type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true"
|
||||||
)
|
)
|
||||||
@@ -1422,7 +1572,7 @@ target "csrc-rocm-ci" {
|
|||||||
cache-from = concat(
|
cache-from = concat(
|
||||||
get_cache_from_rocm_csrc(),
|
get_cache_from_rocm_csrc(),
|
||||||
EOF
|
EOF
|
||||||
write_hcl_string_list " " "${content_cache_from[@]}"
|
write_hcl_string_list " " "${csrc_content_cache_from[@]}"
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
)
|
)
|
||||||
EOF
|
EOF
|
||||||
@@ -1430,11 +1580,23 @@ EOF
|
|||||||
cat <<EOF
|
cat <<EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
target "rust-rocm-ci" {
|
||||||
|
cache-from = concat(
|
||||||
|
get_cache_from_rocm_rust(),
|
||||||
|
EOF
|
||||||
|
write_hcl_string_list " " "${rust_content_cache_from[@]}"
|
||||||
|
cat <<EOF
|
||||||
|
)
|
||||||
|
EOF
|
||||||
|
write_hcl_string_list_attr " " "cache-to" "${rust_cache_to[@]}"
|
||||||
|
cat <<EOF
|
||||||
|
}
|
||||||
|
|
||||||
target "test-rocm-ci" {
|
target "test-rocm-ci" {
|
||||||
cache-from = concat(
|
cache-from = concat(
|
||||||
get_cache_from_rocm(),
|
get_cache_from_rocm(),
|
||||||
EOF
|
EOF
|
||||||
write_hcl_string_list " " "${content_cache_from[@]}"
|
write_hcl_string_list " " "${combined_content_cache_from[@]}"
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
)
|
)
|
||||||
EOF
|
EOF
|
||||||
@@ -1446,7 +1608,7 @@ target "export-wheel-rocm" {
|
|||||||
cache-from = concat(
|
cache-from = concat(
|
||||||
get_cache_from_rocm(),
|
get_cache_from_rocm(),
|
||||||
EOF
|
EOF
|
||||||
write_hcl_string_list " " "${content_cache_from[@]}"
|
write_hcl_string_list " " "${combined_content_cache_from[@]}"
|
||||||
cat <<EOF
|
cat <<EOF
|
||||||
)
|
)
|
||||||
EOF
|
EOF
|
||||||
@@ -1920,6 +2082,7 @@ main() {
|
|||||||
compute_dependency_cache_keys
|
compute_dependency_cache_keys
|
||||||
write_ci_base_label_override
|
write_ci_base_label_override
|
||||||
compute_rocm_csrc_content_hash_if_needed
|
compute_rocm_csrc_content_hash_if_needed
|
||||||
|
compute_rocm_rust_content_hash_if_needed
|
||||||
write_rocm_cache_override
|
write_rocm_cache_override
|
||||||
resolve_ci_base_dependency_targets
|
resolve_ci_base_dependency_targets
|
||||||
print_bake_config
|
print_bake_config
|
||||||
|
|||||||
+111
-17
@@ -1,5 +1,7 @@
|
|||||||
# default base image
|
# default base image
|
||||||
ARG REMOTE_VLLM="0"
|
ARG REMOTE_VLLM="0"
|
||||||
|
ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
|
||||||
|
ARG VLLM_BRANCH="main"
|
||||||
ARG COMMON_WORKDIR=/app
|
ARG COMMON_WORKDIR=/app
|
||||||
ARG BASE_IMAGE=rocm/vllm-dev:base
|
ARG BASE_IMAGE=rocm/vllm-dev:base
|
||||||
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
|
ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
|
||||||
@@ -110,10 +112,8 @@ WORKDIR ${COMMON_WORKDIR}
|
|||||||
FROM base AS fetch_vllm_0
|
FROM base AS fetch_vllm_0
|
||||||
ONBUILD COPY ./ vllm/
|
ONBUILD COPY ./ vllm/
|
||||||
FROM base AS fetch_vllm_1
|
FROM base AS fetch_vllm_1
|
||||||
ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
|
ARG VLLM_REPO
|
||||||
ARG VLLM_BRANCH="main"
|
ARG VLLM_BRANCH
|
||||||
ENV VLLM_REPO=${VLLM_REPO}
|
|
||||||
ENV VLLM_BRANCH=${VLLM_BRANCH}
|
|
||||||
ONBUILD RUN git clone ${VLLM_REPO} \
|
ONBUILD RUN git clone ${VLLM_REPO} \
|
||||||
&& cd vllm \
|
&& cd vllm \
|
||||||
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||||
@@ -124,34 +124,118 @@ ONBUILD RUN git clone ${VLLM_REPO} \
|
|||||||
FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
|
FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
|
||||||
|
|
||||||
# -----------------------
|
# -----------------------
|
||||||
# Rust build stage
|
# Rust/protoc toolchain inputs
|
||||||
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
|
#
|
||||||
# don't need the rust toolchain or protoc.
|
# Keep these separate from Rust source inputs so Rust source-only changes do not
|
||||||
FROM fetch_vllm AS rust-build
|
# invalidate the long-lived ci_base image that carries the toolchain.
|
||||||
ARG COMMON_WORKDIR
|
FROM scratch AS rust_toolchain_input_0
|
||||||
ARG USE_SCCACHE
|
COPY rust-toolchain.toml /rust-toolchain-input/vllm/rust-toolchain.toml
|
||||||
|
COPY tools/install_protoc.sh /rust-toolchain-input/vllm/tools/install_protoc.sh
|
||||||
|
|
||||||
|
FROM base AS rust_toolchain_input_1
|
||||||
|
ARG VLLM_REPO
|
||||||
|
ARG VLLM_BRANCH
|
||||||
|
RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-toolchain-input/vllm \
|
||||||
|
&& cd /rust-toolchain-input/vllm \
|
||||||
|
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||||
|
&& git sparse-checkout init --no-cone \
|
||||||
|
&& git sparse-checkout set \
|
||||||
|
rust-toolchain.toml \
|
||||||
|
tools/install_protoc.sh \
|
||||||
|
&& git checkout FETCH_HEAD
|
||||||
|
|
||||||
|
FROM rust_toolchain_input_${REMOTE_VLLM} AS rust-toolchain-input
|
||||||
|
|
||||||
|
# -----------------------
|
||||||
|
# Rust build inputs
|
||||||
|
#
|
||||||
|
# REMOTE_VLLM=0: use only the local files that can affect Rust artifacts.
|
||||||
|
# REMOTE_VLLM=1: fetch only those same files from the requested remote ref.
|
||||||
|
FROM scratch AS rust_input_0
|
||||||
|
COPY requirements/build/rust.txt /rust-input/vllm/requirements/build/rust.txt
|
||||||
|
COPY rust/Cargo.lock rust/Cargo.toml /rust-input/vllm/rust/
|
||||||
|
COPY rust/proto /rust-input/vllm/rust/proto
|
||||||
|
COPY rust/src /rust-input/vllm/rust/src
|
||||||
|
COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml
|
||||||
|
COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py
|
||||||
|
COPY build_rust.sh /rust-input/vllm/build_rust.sh
|
||||||
|
|
||||||
|
FROM base AS rust_input_1
|
||||||
|
ARG VLLM_REPO
|
||||||
|
ARG VLLM_BRANCH
|
||||||
|
RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \
|
||||||
|
&& cd /rust-input/vllm \
|
||||||
|
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||||
|
&& git sparse-checkout init --no-cone \
|
||||||
|
&& git sparse-checkout set \
|
||||||
|
requirements/build/rust.txt \
|
||||||
|
rust/Cargo.lock \
|
||||||
|
rust/Cargo.toml \
|
||||||
|
rust/proto/** \
|
||||||
|
rust/src/** \
|
||||||
|
rust-toolchain.toml \
|
||||||
|
tools/build_rust.py \
|
||||||
|
build_rust.sh \
|
||||||
|
&& git checkout FETCH_HEAD
|
||||||
|
|
||||||
|
FROM rust_input_${REMOTE_VLLM} AS rust-input
|
||||||
|
|
||||||
|
# -----------------------
|
||||||
|
# Rust/protoc toolchain
|
||||||
|
FROM base AS rust-toolchain
|
||||||
|
|
||||||
|
ENV CARGO_HOME=/root/.cargo
|
||||||
|
ENV RUSTUP_HOME=/root/.rustup
|
||||||
|
ENV PATH=${CARGO_HOME}/bin:${PATH}
|
||||||
|
|
||||||
# protoc is used by tonic-build/prost-build.
|
# protoc is used by tonic-build/prost-build.
|
||||||
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
|
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
|
||||||
ca-certificates curl unzip \
|
ca-certificates curl unzip \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY tools/install_protoc.sh /tmp/install_protoc.sh
|
COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/tools/install_protoc.sh /tmp/install_protoc.sh
|
||||||
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
|
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
|
||||||
|
|
||||||
|
COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/rust-toolchain.toml /tmp/rust-toolchain.toml
|
||||||
|
RUN TOOLCHAIN="$(grep '^channel' /tmp/rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/')" \
|
||||||
|
&& if ! command -v rustup >/dev/null 2>&1; then \
|
||||||
|
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
||||||
|
| sh -s -- -y --default-toolchain none; \
|
||||||
|
fi \
|
||||||
|
&& rustup toolchain install "${TOOLCHAIN}" \
|
||||||
|
&& rustup default "${TOOLCHAIN}" \
|
||||||
|
&& rustc --version \
|
||||||
|
&& cargo --version
|
||||||
|
|
||||||
|
# -----------------------
|
||||||
|
# Rust build stage
|
||||||
|
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
|
||||||
|
# don't need the rust toolchain or protoc.
|
||||||
|
FROM rust-toolchain AS rust-build
|
||||||
|
ARG COMMON_WORKDIR
|
||||||
|
ARG USE_SCCACHE
|
||||||
|
|
||||||
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
|
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
|
||||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||||
ENV CARGO_BUILD_JOBS=4
|
ENV CARGO_BUILD_JOBS=4
|
||||||
ENV CARGO_NET_RETRY=10
|
ENV CARGO_NET_RETRY=10
|
||||||
ENV RUSTUP_MAX_RETRIES=10
|
ENV RUSTUP_MAX_RETRIES=10
|
||||||
|
|
||||||
|
WORKDIR ${COMMON_WORKDIR}/vllm
|
||||||
|
|
||||||
|
COPY --from=rust-input /rust-input/vllm/requirements/build/rust.txt requirements/build/rust.txt
|
||||||
|
|
||||||
# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on
|
# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on
|
||||||
# a separate local sccache daemon while sharing the same remote cache backend.
|
# a separate local sccache daemon while sharing the same remote cache backend.
|
||||||
ENV SCCACHE_SERVER_PORT=4227
|
ENV SCCACHE_SERVER_PORT=4227
|
||||||
|
|
||||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||||
cd ${COMMON_WORKDIR}/vllm \
|
uv pip install --system -r requirements/build/rust.txt
|
||||||
&& uv pip install --system -r requirements/build/rust.txt
|
|
||||||
|
COPY --from=rust-input /rust-input/vllm/rust rust
|
||||||
|
COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml
|
||||||
|
COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py
|
||||||
|
COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh
|
||||||
|
|
||||||
# Build the release binary. Cargo's registry/git caches can be written by
|
# Build the release binary. Cargo's registry/git caches can be written by
|
||||||
# concurrent BuildKit jobs on shared workers, so lock those cache mounts while
|
# concurrent BuildKit jobs on shared workers, so lock those cache mounts while
|
||||||
@@ -159,8 +243,7 @@ RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
|||||||
# can outlive source updates across BuildKit cache reuse.
|
# can outlive source updates across BuildKit cache reuse.
|
||||||
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
|
RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \
|
||||||
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
|
--mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \
|
||||||
cd ${COMMON_WORKDIR}/vllm \
|
if [ "$USE_SCCACHE" = "1" ]; then \
|
||||||
&& if [ "$USE_SCCACHE" = "1" ]; then \
|
|
||||||
export RUSTC_WRAPPER=sccache \
|
export RUSTC_WRAPPER=sccache \
|
||||||
&& sccache --show-stats; \
|
&& sccache --show-stats; \
|
||||||
fi \
|
fi \
|
||||||
@@ -185,8 +268,8 @@ COPY requirements/common.txt requirements/common.txt
|
|||||||
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
|
||||||
uv pip install --system -r requirements/rocm.txt
|
uv pip install --system -r requirements/rocm.txt
|
||||||
|
|
||||||
# pyproject.toml is bind-mounted in the RUN step so metadata-only changes do
|
# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image,
|
||||||
# not invalidate the expensive native build layer.
|
# but it still participates in the BuildKit cache key for this RUN step.
|
||||||
COPY setup.py CMakeLists.txt ./
|
COPY setup.py CMakeLists.txt ./
|
||||||
COPY tools/build_rust.py tools/build_rust.py
|
COPY tools/build_rust.py tools/build_rust.py
|
||||||
COPY cmake cmake/
|
COPY cmake cmake/
|
||||||
@@ -556,6 +639,17 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1
|
|||||||
FROM mori_base AS ci_base
|
FROM mori_base AS ci_base
|
||||||
ARG COMMON_WORKDIR
|
ARG COMMON_WORKDIR
|
||||||
|
|
||||||
|
ENV CARGO_HOME=/root/.cargo
|
||||||
|
ENV RUSTUP_HOME=/root/.rustup
|
||||||
|
ENV PATH=${CARGO_HOME}/bin:${PATH}
|
||||||
|
|
||||||
|
# Bake the Rust/protoc toolchain into ci_base so Rust build/test steps do not
|
||||||
|
# need to fetch it on each per-commit image build.
|
||||||
|
COPY --from=rust-toolchain /root/.cargo /root/.cargo
|
||||||
|
COPY --from=rust-toolchain /root/.rustup /root/.rustup
|
||||||
|
COPY --from=rust-toolchain /usr/local/bin/protoc /usr/local/bin/protoc
|
||||||
|
COPY --from=rust-toolchain /usr/local/include/google /usr/local/include/google
|
||||||
|
|
||||||
# Update rdma-core to support latest rocshmem.
|
# Update rdma-core to support latest rocshmem.
|
||||||
ARG DEEPEP_NIC
|
ARG DEEPEP_NIC
|
||||||
RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \
|
RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \
|
||||||
|
|||||||
+45
-2
@@ -141,6 +141,10 @@ variable "ROCM_CSRC_CACHE_TO_MODE" {
|
|||||||
default = "max"
|
default = "max"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
variable "ROCM_RUST_CACHE_TO_MODE" {
|
||||||
|
default = "max"
|
||||||
|
}
|
||||||
|
|
||||||
variable "ROCM_FINAL_CACHE_TO_MODE" {
|
variable "ROCM_FINAL_CACHE_TO_MODE" {
|
||||||
default = "min"
|
default = "min"
|
||||||
}
|
}
|
||||||
@@ -164,6 +168,13 @@ function "get_cache_from_rocm" {
|
|||||||
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
||||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||||
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:csrc-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||||
|
# Import the source-scoped Rust frontend cache so non-Rust changes do not
|
||||||
|
# force a fresh cargo release build.
|
||||||
|
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
|
||||||
|
PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
|
||||||
|
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
||||||
|
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||||
|
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||||
# Branch-scoped full image cache - fallback when parent-commit cache is evicted
|
# Branch-scoped full image cache - fallback when parent-commit cache is evicted
|
||||||
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||||
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||||
@@ -204,6 +215,27 @@ function "get_cache_to_rocm_csrc" {
|
|||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function "get_cache_from_rocm_rust" {
|
||||||
|
params = []
|
||||||
|
result = compact([
|
||||||
|
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT}" : "",
|
||||||
|
PARENT_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${PARENT_COMMIT}" : "",
|
||||||
|
VLLM_MERGE_BASE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${VLLM_MERGE_BASE_COMMIT}" : "",
|
||||||
|
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG}" : "",
|
||||||
|
ROCM_CACHE_UPSTREAM_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_UPSTREAM_BRANCH_TAG}" : "",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
function "get_cache_to_rocm_rust" {
|
||||||
|
params = []
|
||||||
|
result = compact([
|
||||||
|
# Export exact-commit and branch-scoped Rust caches. A content-addressed
|
||||||
|
# cache ref is appended by ci-bake-rocm.sh when that wrapper is used.
|
||||||
|
BUILDKITE_COMMIT != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-${BUILDKITE_COMMIT},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
|
||||||
|
ROCM_CACHE_BRANCH_TAG != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${ROCM_RUST_CACHE_TO_MODE}" : "",
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
# Cache functions for upstream dependency stages (RIXL/UCX, ROCShmem, DeepEP).
|
# Cache functions for upstream dependency stages (RIXL/UCX, ROCShmem, DeepEP).
|
||||||
# These stages are pinned to specific upstream commit hashes, so cache keys use
|
# These stages are pinned to specific upstream commit hashes, so cache keys use
|
||||||
# those hashes rather than the Buildkite commit. This means the cache persists
|
# those hashes rather than the Buildkite commit. This means the cache persists
|
||||||
@@ -276,6 +308,17 @@ target "csrc-rocm-ci" {
|
|||||||
output = ["type=cacheonly"]
|
output = ["type=cacheonly"]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Cache-only target for the Rust frontend build stage. Final-image cache
|
||||||
|
# exports use mode=min and do not reliably persist intermediate cargo layers,
|
||||||
|
# so Rust gets its own source-scoped cache target.
|
||||||
|
target "rust-rocm-ci" {
|
||||||
|
inherits = ["_common-rocm", "_ci-rocm"]
|
||||||
|
target = "rust-build"
|
||||||
|
cache-from = get_cache_from_rocm_rust()
|
||||||
|
cache-to = get_cache_to_rocm_rust()
|
||||||
|
output = ["type=cacheonly"]
|
||||||
|
}
|
||||||
|
|
||||||
# Keep wheel export on the same CI graph as the test image build so the
|
# Keep wheel export on the same CI graph as the test image build so the
|
||||||
# shared build_vllm/export_vllm stages resolve identically within one bake
|
# shared build_vllm/export_vllm stages resolve identically within one bake
|
||||||
# invocation. Without this, export-wheel-rocm uses the plain local target
|
# invocation. Without this, export-wheel-rocm uses the plain local target
|
||||||
@@ -292,13 +335,13 @@ target "export-wheel-rocm" {
|
|||||||
# Artifact-only vLLM build. GPU test jobs consume this artifact on top of
|
# Artifact-only vLLM build. GPU test jobs consume this artifact on top of
|
||||||
# ci_base, avoiding a per-commit multi-GB image push/pull.
|
# ci_base, avoiding a per-commit multi-GB image push/pull.
|
||||||
group "test-rocm-ci-with-artifacts" {
|
group "test-rocm-ci-with-artifacts" {
|
||||||
targets = ["csrc-rocm-ci", "export-wheel-rocm"]
|
targets = ["rust-rocm-ci", "csrc-rocm-ci", "export-wheel-rocm"]
|
||||||
}
|
}
|
||||||
|
|
||||||
# Full test image + wheel export. Kept for fallback/debugging when a pushed
|
# Full test image + wheel export. Kept for fallback/debugging when a pushed
|
||||||
# per-commit image is useful.
|
# per-commit image is useful.
|
||||||
group "test-rocm-ci-with-wheel" {
|
group "test-rocm-ci-with-wheel" {
|
||||||
targets = ["csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
|
||||||
}
|
}
|
||||||
|
|
||||||
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG
|
# Image tags for the ci_base build. ci-bake-rocm.sh rewrites CI_BASE_IMAGE_TAG
|
||||||
|
|||||||
Reference in New Issue
Block a user