[CI][ROCm] Prefetch safetensors weights in AMD CI (#54898)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
e52407ef4c
commit
c6005000d3
@@ -16,6 +16,9 @@ MODEL_NAME = os.environ.get(
|
||||
REVISION = os.environ.get("REVISION", "main")
|
||||
QUANTIZATION = os.environ.get("QUANTIZATION", "gptq_marlin")
|
||||
MIN_CAPABILITY = os.environ.get("MIN_CAPABILITY", "80")
|
||||
SAFETENSORS_LOAD_STRATEGY = os.environ.get(
|
||||
"WEIGHT_LOADING_TEST_SAFETENSORS_LOAD_STRATEGY"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
@@ -42,6 +45,7 @@ def test_weight_loading(vllm_runner):
|
||||
quantization=None if QUANTIZATION == "None" else QUANTIZATION,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=2,
|
||||
safetensors_load_strategy=SAFETENSORS_LOAD_STRATEGY,
|
||||
# Generating 20 tokens needs a tiny KV cache, while sizing and
|
||||
# allocating a full-size one dominates this test on large devices.
|
||||
kv_cache_memory_bytes=2 * GiB_bytes,
|
||||
|
||||
Reference in New Issue
Block a user