From c6005000d326fe2726aed62553f5837d42c5e04c Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 2 Sep 2026 03:28:39 -0500 Subject: [PATCH] [CI][ROCm] Prefetch safetensors weights in AMD CI (#54898) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .../Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml | 2 +- tests/weight_loading/models-amd.txt | 3 ++- tests/weight_loading/models-large-amd.txt | 7 ++++--- tests/weight_loading/run_model_weight_loading_test.sh | 4 ++++ tests/weight_loading/test_weight_loading.py | 4 ++++ 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml index a9c9857fc3..6631896ad5 100644 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml +++ b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml @@ -1,7 +1,7 @@ # For hf script, without -t option (tensor parallel size). # bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5 model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" -rocm_safetensors_load_strategy: lazy +rocm_safetensors_load_strategy: prefetch required_gpu_arch: - gfx942 - gfx950 diff --git a/tests/weight_loading/models-amd.txt b/tests/weight_loading/models-amd.txt index 816d136839..a6a85736cb 100644 --- a/tests/weight_loading/models-amd.txt +++ b/tests/weight_loading/models-amd.txt @@ -1,6 +1,7 @@ +# quantization, model, revision[, minimum capability[, safetensors load strategy]] fp8, amd/Meta-Llama-3.1-8B-Instruct-FP8-KV, main None, amd/Llama-3.2-1B-Instruct-FP8-KV, main -fp8, amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV, main +fp8, amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV, main, 80, prefetch gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main gptq, TheBloke/Llama-2-7B-GPTQ, main gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True diff --git a/tests/weight_loading/models-large-amd.txt b/tests/weight_loading/models-large-amd.txt index b6f5b4b16b..41e8ee5200 100644 --- a/tests/weight_loading/models-large-amd.txt +++ b/tests/weight_loading/models-large-amd.txt @@ -1,3 +1,4 @@ -fp8, amd/Meta-Llama-3.1-70B-Instruct-FP8-KV, main -None, microsoft/phi-4, main -fp8, amd/Mixtral-8x22B-Instruct-v0.1-FP8-KV, main +# quantization, model, revision[, minimum capability[, safetensors load strategy]] +fp8, amd/Meta-Llama-3.1-70B-Instruct-FP8-KV, main, 80, prefetch +None, microsoft/phi-4, main, 80, prefetch +fp8, amd/Mixtral-8x22B-Instruct-v0.1-FP8-KV, main, 80, prefetch diff --git a/tests/weight_loading/run_model_weight_loading_test.sh b/tests/weight_loading/run_model_weight_loading_test.sh index 8a899bc154..598f1620c7 100755 --- a/tests/weight_loading/run_model_weight_loading_test.sh +++ b/tests/weight_loading/run_model_weight_loading_test.sh @@ -28,6 +28,7 @@ do echo "=== RUNNING MODEL: $MODEL_CONFIG ===" + unset WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY export QUANTIZATION=${array[0]} export MODEL_NAME=${array[1]} export REVISION=${array[2]} @@ -35,6 +36,9 @@ do if [ ${#array[@]} -gt 3 ]; then export MIN_CAPABILITY=${array[3]} fi + if [ ${#array[@]} -gt 4 ]; then + export WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY=${array[4]} + fi pytest -s weight_loading/test_weight_loading.py || LOCAL_SUCCESS=$? if [[ $LOCAL_SUCCESS == 0 ]]; then diff --git a/tests/weight_loading/test_weight_loading.py b/tests/weight_loading/test_weight_loading.py index 72c0a23367..ce1835e254 100644 --- a/tests/weight_loading/test_weight_loading.py +++ b/tests/weight_loading/test_weight_loading.py @@ -16,6 +16,9 @@ MODEL_NAME = os.environ.get( REVISION = os.environ.get("REVISION", "main") QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin") MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "80") +SAFETENSORS_LOAD_STRATEGY = os.environ.get( + "WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY" +) @pytest.mark.skipif( @@ -42,6 +45,7 @@ def test_weight_loading(vllm_runner): quantization=None if QUANTIZATION == "None" else QUANTIZATION, max_model_len=MAX_MODEL_LEN, tensor_parallel_size=2, + safetensors_load_strategy=SAFETENSORS_LOAD_STRATEGY, # Generating 20 tokens needs a tiny KV cache, while sizing and # allocating a full-size one dominates this test on large devices. kv_cache_memory_bytes=2 * GiB_bytes,