diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 2891291105..3b9a9a1d71 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -285,7 +285,7 @@ get_content_arg_names() { fi | awk 'NF && !seen[$0]++' } -compute_ci_base_content_hash() { +compute_ci_base_content_hash_once() { local -a content_paths=() local -a content_args=() local dockerfile="${CI_BASE_DOCKERFILE:-}" @@ -301,7 +301,8 @@ compute_ci_base_content_hash() { if [[ -n "${dockerfile}" ]]; then printf 'dockerfile:%s\n' "${dockerfile}" printf 'resolved-build-args:\n' - hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}" + hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}" \ + || return 1 if [[ -n "${stages}" ]]; then printf 'dockerfile-stages:%s\n' "${stages}" if [[ -f "${dockerfile}" ]]; then @@ -314,6 +315,53 @@ compute_ci_base_content_hash() { } | sha256sum | cut -d' ' -f1 } +compute_ci_base_content_hash() { + local attempts="${CI_BASE_HASH_ATTEMPTS:-3}" + local delay_secs="${CI_BASE_HASH_RETRY_DELAY:-5}" + local attempt=0 + local hash="" + local failed=0 + local -a hashes=() + + if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then + echo "Invalid CI_BASE_HASH_ATTEMPTS: ${attempts}" >&2 + return 1 + fi + if [[ ! "${delay_secs}" =~ ^[0-9]+$ ]]; then + echo "Invalid CI_BASE_HASH_RETRY_DELAY: ${delay_secs}" >&2 + return 1 + fi + + for ((attempt = 1; attempt <= attempts; attempt++)); do + if ! hash=$(compute_ci_base_content_hash_once); then + echo "ci_base content hash calculation ${attempt}/${attempts} failed" >&2 + failed=1 + else + hashes+=("${hash}") + echo "ci_base content hash calculation ${attempt}/${attempts}: ${hash}" >&2 + fi + + if ((attempt < attempts)); then + sleep "${delay_secs}" + fi + done + + if ((failed)) || ((${#hashes[@]} != attempts)); then + echo "Could not calculate a reliable ci_base content hash" >&2 + return 1 + fi + + for hash in "${hashes[@]:1}"; do + if [[ "${hash}" != "${hashes[0]}" ]]; then + echo "ci_base content hash changed between calculations" >&2 + printf ' observed: %s\n' "${hashes[@]}" >&2 + return 1 + fi + done + + printf '%s\n' "${hashes[0]}" +} + extract_dockerfile_arg_default() { local dockerfile="$1" local arg_name="$2" @@ -366,7 +414,11 @@ hash_dockerfile_arg_values() { printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-}" if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then digest=$(resolve_image_digest "${arg_value}") - printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest:-unknown}" + if [[ -z "${digest}" ]]; then + echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2 + return 1 + fi + printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}" fi done } diff --git a/.buildkite/scripts/rocm/build-test-image.sh b/.buildkite/scripts/rocm/build-test-image.sh index 9803e20d02..9feb6c0924 100755 --- a/.buildkite/scripts/rocm/build-test-image.sh +++ b/.buildkite/scripts/rocm/build-test-image.sh @@ -13,6 +13,18 @@ metadata_get() { fi } +use_ci_base_if_present() { + local ci_base_image="" + + ci_base_image="$(metadata_get rocm-ci-base-image)" + if [[ -z "${ci_base_image}" ]]; then + return 1 + fi + + export CI_BASE_IMAGE="${ci_base_image}" + echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}" +} + use_refreshed_base_if_present() { local base_refreshed="" @@ -22,15 +34,12 @@ use_refreshed_base_if_present() { fi export BASE_IMAGE - export CI_BASE_IMAGE export IMAGE_TAG_LATEST BASE_IMAGE="$(metadata_get rocm-base-image)" - CI_BASE_IMAGE="$(metadata_get rocm-ci-base-image)" IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)" echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}" - echo "Using refreshed ROCm ci_base image for test image: ${CI_BASE_IMAGE}" if [[ -n "${IMAGE_TAG_LATEST}" ]]; then echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}" fi @@ -41,6 +50,8 @@ use_refreshed_base_if_present() { main() { local base_refreshed=0 + use_ci_base_if_present || true + if use_refreshed_base_if_present; then base_refreshed=1 fi