[CI/Build][CPU] Fix flaky rust downloads, broken prune flag, and triton-cpu cache coupling (#56247)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -13,17 +13,16 @@ TIMEOUT_VAL=$1
|
||||
TEST_COMMAND=$2
|
||||
|
||||
# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
|
||||
# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
|
||||
# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
|
||||
# DISK_USAGE_THRESHOLD percent, and only prune images/cache unused for at least
|
||||
# CACHE_MAX_AGE so recently-built layers survive for reuse.
|
||||
DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-80}
|
||||
BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-200GB}
|
||||
CACHE_MAX_AGE=${CACHE_MAX_AGE:-24h}
|
||||
|
||||
# Reclaim disk only when the host is under pressure. We trim (not purge) the
|
||||
# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
|
||||
# touch dangling images; other agents' uniquely tagged images are left alone.
|
||||
# Cache mounts (exec.cachemount: uv/cargo/apt) are excluded so they survive
|
||||
# pruning -- they're what let installs reuse packages instead of hitting the
|
||||
# network, and are otherwise reclaimable like any other build cache record.
|
||||
# Reclaim disk only when the host is under pressure, aging out anything unused
|
||||
# for less than CACHE_MAX_AGE so hot layers survive -- same `--filter
|
||||
# until=<N>h` pattern the TPU CI scripts already rely on. `docker buildx
|
||||
# prune --max-used-space` is a no-op on this host's `docker` driver (BuildKit
|
||||
# embedded in dockerd never enforces the size cap), so we don't use it.
|
||||
prune_if_disk_pressure() {
|
||||
local docker_root disk_usage
|
||||
docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
|
||||
@@ -33,8 +32,7 @@ prune_if_disk_pressure() {
|
||||
disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
|
||||
echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
|
||||
docker image prune -f || true
|
||||
docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" --filter type!=exec.cachemount || true
|
||||
docker system prune --force --all --filter "until=${CACHE_MAX_AGE}" || true
|
||||
else
|
||||
echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
|
||||
fi
|
||||
@@ -61,18 +59,18 @@ docker pull ubuntu:25.04 || true
|
||||
|
||||
# building the docker image
|
||||
echo "--- :docker: Building Docker image"
|
||||
BUILD_RETRY_PATTERN='dial tcp|i/o timeout|failed to authorize|TLS handshake timeout|connection reset|Could not resolve host|Temporary failure in name resolution|dns error: failed to lookup address information|client error \(Connect\)|error sending request for url|Failed to fetch:'
|
||||
BUILD_RETRY_PATTERN='dial tcp|i/o timeout|failed to authorize|TLS handshake timeout|connection reset|PROTOCOL_ERROR|Could not resolve host|Temporary failure in name resolution|dns error: failed to lookup address information|client error \(Connect\)|error sending request for url|Failed to fetch:'
|
||||
BUILD_MAX_ATTEMPTS=4 # 1 initial + 3 retries
|
||||
BUILD_RETRY_WAITS=(10 20 40) # seconds to wait before retry 1/2/3
|
||||
build_log="$(mktemp)"
|
||||
attempt=1
|
||||
while true; do
|
||||
if docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test \
|
||||
--build-arg USE_SCCACHE=1 --build-arg SCCACHE_LOCAL_ONLY=1 \
|
||||
--build-arg USE_SCCACHE=1 --build-arg SCCACHE_LOCAL_ONLY=1 --build-arg max_jobs=16 \
|
||||
-f docker/Dockerfile.cpu . 2>&1 | tee "$build_log"; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -ge "$BUILD_MAX_ATTEMPTS" ] || ! grep -qE "$BUILD_RETRY_PATTERN" "$build_log"; then
|
||||
if [ "$attempt" -ge "$BUILD_MAX_ATTEMPTS" ] || ! grep -qiE "$BUILD_RETRY_PATTERN" "$build_log"; then
|
||||
rm -f "$build_log"
|
||||
exit 1
|
||||
fi
|
||||
@@ -83,6 +81,24 @@ while true; do
|
||||
done
|
||||
rm -f "$build_log"
|
||||
|
||||
# Run the image, setting --shm-size=4g for tensor parallel.
|
||||
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
|
||||
# Run the image, setting --shm-size=4g for tensor parallel. Default to
|
||||
# HF_HUB_OFFLINE so a warm ~/.cache/huggingface doesn't hit the network;
|
||||
# retry once online if the cache is missing something.
|
||||
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)'
|
||||
run_test() {
|
||||
local hf_offline=$1
|
||||
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
|
||||
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
|
||||
}
|
||||
|
||||
test_log="$(mktemp)"
|
||||
if run_test 1 2>&1 | tee "$test_log"; then
|
||||
rm -f "$test_log"
|
||||
elif grep -qE "$OFFLINE_RETRY_PATTERN" "$test_log"; then
|
||||
rm -f "$test_log"
|
||||
echo "--- :warning: HF_HUB_OFFLINE caused a cache miss, retrying with online fallback"
|
||||
run_test 0
|
||||
else
|
||||
rm -f "$test_log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user