[ROCm][CI] Adding nixl multiconn (#47481)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -2439,6 +2439,59 @@ steps:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||
agent_pool: mi300_2
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/v1/core/sched/
|
||||
- vllm/v1/core/kv_cache_coordinator.py
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
|
||||
|
||||
- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||
agent_pool: mi300_2
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
|
||||
|
||||
- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||
agent_pool: mi300_2
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
|
||||
|
||||
- label: V1 e2e (4 GPUs) # TBD
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||
|
||||
@@ -9,8 +9,10 @@ PREFILL_GPU_ID=${PREFILL_GPU_ID:-0}
|
||||
DECODE_GPU_ID=${DECODE_GPU_ID:-1}
|
||||
MODEL=${MODEL:-"ibm-granite/granite-4.0-h-tiny"}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8}
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
ATTENTION_BACKEND=${ATTENTION_BACKEND:-FLASHINFER}
|
||||
|
||||
echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL)"
|
||||
echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL, backend=$ATTENTION_BACKEND)"
|
||||
|
||||
KV_CONFIG='{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
|
||||
|
||||
@@ -36,6 +38,14 @@ cleanup_instances() {
|
||||
|
||||
cleanup_instances
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a EXTRA_ARGS <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
fi
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
EXTRA_ARGS+=(--attention-backend "$ATTENTION_BACKEND")
|
||||
fi
|
||||
|
||||
# Start prefill instance
|
||||
PREFILL_PORT=8001
|
||||
CUDA_VISIBLE_DEVICES=$PREFILL_GPU_ID \
|
||||
@@ -51,8 +61,8 @@ vllm serve $MODEL \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode all \
|
||||
--attention-backend FLASHINFER \
|
||||
--kv-transfer-config "$KV_CONFIG" &
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
# Start decode instance
|
||||
DECODE_PORT=8002
|
||||
@@ -69,8 +79,8 @@ vllm serve $MODEL \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode all \
|
||||
--attention-backend FLASHINFER \
|
||||
--kv-transfer-config "$KV_CONFIG" &
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
echo "Waiting for prefill instance on port $PREFILL_PORT..."
|
||||
wait_for_server "$PREFILL_PORT"
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
# Environment variables:
|
||||
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
|
||||
# GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6)
|
||||
# ATTENTION_BACKEND - optional attention backend for vllm serve
|
||||
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
|
||||
# SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test
|
||||
# SKIP_NORMAL_LAYOUT - set to 1 to skip the normal layout test
|
||||
@@ -34,9 +35,11 @@ fi
|
||||
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-128}
|
||||
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
|
||||
GIT_ROOT=$(git rev-parse --show-toplevel)
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
|
||||
|
||||
# ── KV transfer configs ─────────────────────────────────────────────────
|
||||
@@ -139,6 +142,9 @@ run_tests_for_model() {
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
|
||||
fi
|
||||
eval "$BASE_CMD &"
|
||||
|
||||
# ── Start decode instance ──
|
||||
@@ -161,6 +167,9 @@ run_tests_for_model() {
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
|
||||
fi
|
||||
eval "$BASE_CMD &"
|
||||
|
||||
# ── Wait for servers ──
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
|
||||
# KV_CACHE_MEMORY_BYTES - GPU KV cache size in bytes (default: 268435456 = 256 MiB)
|
||||
# BLOCK_SIZE - KV cache block size (default: 128)
|
||||
# ATTENTION_BACKEND - optional attention backend for vllm serve
|
||||
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
|
||||
set -xe
|
||||
|
||||
@@ -34,9 +35,11 @@ fi
|
||||
KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-268435456} # 256 MiB
|
||||
MAX_MODEL_LEN=${MAX_MODEL_LEN:-2048}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-128}
|
||||
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
|
||||
GIT_ROOT=$(git rev-parse --show-toplevel)
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||
|
||||
# ── KV transfer config ──────────────────────────────────────────────────
|
||||
|
||||
@@ -110,6 +113,9 @@ run_tests_for_model() {
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
|
||||
fi
|
||||
eval "$BASE_CMD &"
|
||||
|
||||
# ── Start decode instance ──
|
||||
@@ -133,6 +139,9 @@ run_tests_for_model() {
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
if [[ -n "$ATTENTION_BACKEND" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
|
||||
fi
|
||||
eval "$BASE_CMD &"
|
||||
|
||||
# ── Wait for servers ──
|
||||
|
||||
Reference in New Issue
Block a user