[Bugfix][CPU] Add per-tensor FP8 W8A16 kernel to fix Ministral crash on CPU (#56985)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-16 15:49:17 +08:00
committed by GitHub
parent f12fe10c5c
commit b68408043d
6 changed files with 233 additions and 7 deletions
+15 -6
View File
@@ -83,14 +83,23 @@ rm -f "$build_log"
# Run the image, setting --shm-size=4g for tensor parallel. Default to
# HF_HUB_OFFLINE so a warm ~/.cache/huggingface doesn't hit the network;
# retry once online if the cache is missing something. vllm's get_config()
# wraps the raw huggingface_hub offline-mode errors in a generic ValueError
# (see transformers_utils/config.py), so match that message too or the
# fallback never triggers for a config-lookup cache miss.
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified'
# retry once online if the cache is missing something. vllm wraps offline
# cache-miss errors in ways that don't preserve the raw huggingface_hub
# exception name: get_config() re-raises as a generic ValueError
# (transformers_utils/config.py), and default_loader.py raises its own
# RuntimeError when a snapshot dir has a config but no weight files. Match
# those messages too or the fallback never triggers for those cache misses.
#
# Also mount ~/.cache/vllm: multimodal test fixtures (e.g. VideoAsset, via
# vllm/assets/video.py) download into VLLM_ASSETS_CACHE (~/.cache/vllm/assets
# by default), a directory distinct from the HF hub cache above. Without a
# persistent mount there, those assets never survive past one container's
# lifetime, so every run re-fetches them from the network on the "offline"
# attempt and unconditionally falls back to the online retry.
OFFLINE_RETRY_PATTERN='huggingface_hub\.errors\.(LocalEntryNotFoundError|OfflineModeIsEnabled)|Invalid repository ID or local directory specified|Cannot find any model weights with'
run_test() {
local hf_offline=$1
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface -v ~/.cache/vllm:/root/.cache/vllm --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 -e HF_HUB_OFFLINE="$hf_offline" -e HF_DATASETS_OFFLINE="$hf_offline" --shm-size=4g "$IMAGE_NAME" \
timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
}