[XPU][Test] Support MultiConnector accuracy testing on XPU (#51160)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
This commit is contained in:
@@ -18,6 +18,9 @@
|
|||||||
# Environment variables:
|
# Environment variables:
|
||||||
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
|
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
|
||||||
# GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6)
|
# GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6)
|
||||||
|
# MAX_MODEL_LEN - maximum model context length
|
||||||
|
# KV_BUFFER_DEVICE - NIXL KV buffer device (default: cuda; use xpu for XPU)
|
||||||
|
# NUM_CONCURRENT - lm-eval request concurrency (default: 100)
|
||||||
# ATTENTION_BACKEND - optional attention backend for vllm serve
|
# ATTENTION_BACKEND - optional attention backend for vllm serve
|
||||||
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
|
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
|
||||||
# SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test
|
# SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test
|
||||||
@@ -35,6 +38,8 @@ fi
|
|||||||
|
|
||||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6}
|
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6}
|
||||||
BLOCK_SIZE=${BLOCK_SIZE:-128}
|
BLOCK_SIZE=${BLOCK_SIZE:-128}
|
||||||
|
MAX_MODEL_LEN=${MAX_MODEL_LEN:-8192}
|
||||||
|
KV_BUFFER_DEVICE=${KV_BUFFER_DEVICE:-cuda}
|
||||||
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
|
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
|
||||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||||
|
|
||||||
@@ -42,6 +47,12 @@ SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
|||||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||||
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
|
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
|
||||||
|
|
||||||
|
if [[ "$KV_BUFFER_DEVICE" == "xpu" ]]; then
|
||||||
|
DEVICE_VISIBILITY_ENV=ZE_AFFINITY_MASK
|
||||||
|
else
|
||||||
|
DEVICE_VISIBILITY_ENV=CUDA_VISIBLE_DEVICES
|
||||||
|
fi
|
||||||
|
|
||||||
# ── KV transfer configs ─────────────────────────────────────────────────
|
# ── KV transfer configs ─────────────────────────────────────────────────
|
||||||
|
|
||||||
# Normal layout: OffloadingConnector prefers cross-layer but NixlConnector
|
# Normal layout: OffloadingConnector prefers cross-layer but NixlConnector
|
||||||
@@ -51,7 +62,8 @@ KV_CONFIG_NORMAL='{
|
|||||||
"kv_role":"kv_both",
|
"kv_role":"kv_both",
|
||||||
"kv_connector_extra_config":{
|
"kv_connector_extra_config":{
|
||||||
"connectors":[
|
"connectors":[
|
||||||
{"kv_connector":"NixlConnector","kv_role":"kv_both"},
|
{"kv_connector":"NixlConnector","kv_role":"kv_both",
|
||||||
|
"kv_buffer_device":"'$KV_BUFFER_DEVICE'"},
|
||||||
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
|
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
|
||||||
"kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}}
|
"kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}}
|
||||||
]
|
]
|
||||||
@@ -67,6 +79,7 @@ KV_CONFIG_CROSS_LAYERS='{
|
|||||||
"kv_connector_extra_config":{
|
"kv_connector_extra_config":{
|
||||||
"connectors":[
|
"connectors":[
|
||||||
{"kv_connector":"NixlConnector","kv_role":"kv_both",
|
{"kv_connector":"NixlConnector","kv_role":"kv_both",
|
||||||
|
"kv_buffer_device":"'$KV_BUFFER_DEVICE'",
|
||||||
"kv_connector_extra_config":{"enable_cross_layers_blocks":"True"}},
|
"kv_connector_extra_config":{"enable_cross_layers_blocks":"True"}},
|
||||||
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
|
{"kv_connector":"OffloadingConnector","kv_role":"kv_both",
|
||||||
"kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}}
|
"kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}}
|
||||||
@@ -99,7 +112,7 @@ get_num_gpus() {
|
|||||||
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
|
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
|
||||||
$SMI_BIN -l | grep -c GPU
|
$SMI_BIN -l | grep -c GPU
|
||||||
else
|
else
|
||||||
echo "1"
|
echo "2"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,7 +137,7 @@ run_tests_for_model() {
|
|||||||
|
|
||||||
# ── Start prefill instance ──
|
# ── Start prefill instance ──
|
||||||
echo "Starting prefill instance on GPU $PREFILL_GPU, port $PREFILL_PORT"
|
echo "Starting prefill instance on GPU $PREFILL_GPU, port $PREFILL_PORT"
|
||||||
BASE_CMD="CUDA_VISIBLE_DEVICES=$PREFILL_GPU \
|
BASE_CMD="$DEVICE_VISIBILITY_ENV=$PREFILL_GPU \
|
||||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||||
UCX_NET_DEVICES=all \
|
UCX_NET_DEVICES=all \
|
||||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$PREFILL_SIDE_CHANNEL_PORT \
|
VLLM_NIXL_SIDE_CHANNEL_PORT=$PREFILL_SIDE_CHANNEL_PORT \
|
||||||
@@ -132,6 +145,7 @@ run_tests_for_model() {
|
|||||||
--port $PREFILL_PORT \
|
--port $PREFILL_PORT \
|
||||||
--enforce-eager \
|
--enforce-eager \
|
||||||
--block-size ${BLOCK_SIZE} \
|
--block-size ${BLOCK_SIZE} \
|
||||||
|
--max-model-len ${MAX_MODEL_LEN} \
|
||||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||||
--tensor-parallel-size 1 \
|
--tensor-parallel-size 1 \
|
||||||
--kv-transfer-config '$kv_config'"
|
--kv-transfer-config '$kv_config'"
|
||||||
@@ -149,7 +163,7 @@ run_tests_for_model() {
|
|||||||
|
|
||||||
# ── Start decode instance ──
|
# ── Start decode instance ──
|
||||||
echo "Starting decode instance on GPU $DECODE_GPU, port $DECODE_PORT"
|
echo "Starting decode instance on GPU $DECODE_GPU, port $DECODE_PORT"
|
||||||
BASE_CMD="CUDA_VISIBLE_DEVICES=$DECODE_GPU \
|
BASE_CMD="$DEVICE_VISIBILITY_ENV=$DECODE_GPU \
|
||||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||||
UCX_NET_DEVICES=all \
|
UCX_NET_DEVICES=all \
|
||||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$DECODE_SIDE_CHANNEL_PORT \
|
VLLM_NIXL_SIDE_CHANNEL_PORT=$DECODE_SIDE_CHANNEL_PORT \
|
||||||
@@ -157,6 +171,7 @@ run_tests_for_model() {
|
|||||||
--port $DECODE_PORT \
|
--port $DECODE_PORT \
|
||||||
--enforce-eager \
|
--enforce-eager \
|
||||||
--block-size ${BLOCK_SIZE} \
|
--block-size ${BLOCK_SIZE} \
|
||||||
|
--max-model-len ${MAX_MODEL_LEN} \
|
||||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||||
--tensor-parallel-size 1 \
|
--tensor-parallel-size 1 \
|
||||||
--kv-transfer-config '$kv_config'"
|
--kv-transfer-config '$kv_config'"
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import lm_eval
|
|||||||
import openai
|
import openai
|
||||||
|
|
||||||
BASE_URL = "http://localhost:8192/v1"
|
BASE_URL = "http://localhost:8192/v1"
|
||||||
NUM_CONCURRENT = 100
|
NUM_CONCURRENT = int(os.getenv("NUM_CONCURRENT", "100"))
|
||||||
TASK = "gsm8k"
|
TASK = "gsm8k"
|
||||||
FILTER = "exact_match,strict-match"
|
FILTER = "exact_match,strict-match"
|
||||||
# TODO(#43186): Widened from 0.03 to absorb chunk_scan/SSU numeric jitter
|
# TODO(#43186): Widened from 0.03 to absorb chunk_scan/SSU numeric jitter
|
||||||
|
|||||||
Reference in New Issue
Block a user