[ROCm][CI] Adding nixl multiconn (#47481)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-07-06 15:04:58 +08:00
committed by GitHub
parent 98ba9b9583
commit 8f0e75e16b
4 changed files with 88 additions and 7 deletions
+53
View File
@@ -2439,6 +2439,59 @@ steps:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- vllm/v1/core/kv_cache_coordinator.py
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
- label: V1 e2e (4 GPUs) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -9,8 +9,10 @@ PREFILL_GPU_ID=${PREFILL_GPU_ID:-0}
DECODE_GPU_ID=${DECODE_GPU_ID:-1}
MODEL=${MODEL:-"ibm-granite/granite-4.0-h-tiny"}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8}
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
ATTENTION_BACKEND=${ATTENTION_BACKEND:-FLASHINFER}
echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL)"
echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL, backend=$ATTENTION_BACKEND)"
KV_CONFIG='{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
@@ -36,6 +38,14 @@ cleanup_instances() {
cleanup_instances
EXTRA_ARGS=()
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a EXTRA_ARGS <<< "$VLLM_SERVE_EXTRA_ARGS"
fi
if [[ -n "$ATTENTION_BACKEND" ]]; then
EXTRA_ARGS+=(--attention-backend "$ATTENTION_BACKEND")
fi
# Start prefill instance
PREFILL_PORT=8001
CUDA_VISIBLE_DEVICES=$PREFILL_GPU_ID \
@@ -51,8 +61,8 @@ vllm serve $MODEL \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--attention-backend FLASHINFER \
--kv-transfer-config "$KV_CONFIG" &
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
# Start decode instance
DECODE_PORT=8002
@@ -69,8 +79,8 @@ vllm serve $MODEL \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode all \
--attention-backend FLASHINFER \
--kv-transfer-config "$KV_CONFIG" &
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
echo "Waiting for prefill instance on port $PREFILL_PORT..."
wait_for_server "$PREFILL_PORT"
@@ -18,6 +18,7 @@
# Environment variables:
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
# GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6)
# ATTENTION_BACKEND - optional attention backend for vllm serve
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
# SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test
# SKIP_NORMAL_LAYOUT - set to 1 to skip the normal layout test
@@ -34,9 +35,11 @@ fi
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6}
BLOCK_SIZE=${BLOCK_SIZE:-128}
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
GIT_ROOT=$(git rev-parse --show-toplevel)
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
# ── KV transfer configs ─────────────────────────────────────────────────
@@ -139,6 +142,9 @@ run_tests_for_model() {
BASE_CMD="${BASE_CMD} $arg"
done
fi
if [[ -n "$ATTENTION_BACKEND" ]]; then
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
fi
eval "$BASE_CMD &"
# ── Start decode instance ──
@@ -161,6 +167,9 @@ run_tests_for_model() {
BASE_CMD="${BASE_CMD} $arg"
done
fi
if [[ -n "$ATTENTION_BACKEND" ]]; then
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
fi
eval "$BASE_CMD &"
# ── Wait for servers ──
@@ -19,6 +19,7 @@
# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B)
# KV_CACHE_MEMORY_BYTES - GPU KV cache size in bytes (default: 268435456 = 256 MiB)
# BLOCK_SIZE - KV cache block size (default: 128)
# ATTENTION_BACKEND - optional attention backend for vllm serve
# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve
set -xe
@@ -34,9 +35,11 @@ fi
KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-268435456} # 256 MiB
MAX_MODEL_LEN=${MAX_MODEL_LEN:-2048}
BLOCK_SIZE=${BLOCK_SIZE:-128}
ATTENTION_BACKEND=${ATTENTION_BACKEND:-}
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
GIT_ROOT=$(git rev-parse --show-toplevel)
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
# ── KV transfer config ──────────────────────────────────────────────────
@@ -110,6 +113,9 @@ run_tests_for_model() {
BASE_CMD="${BASE_CMD} $arg"
done
fi
if [[ -n "$ATTENTION_BACKEND" ]]; then
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
fi
eval "$BASE_CMD &"
# ── Start decode instance ──
@@ -133,6 +139,9 @@ run_tests_for_model() {
BASE_CMD="${BASE_CMD} $arg"
done
fi
if [[ -n "$ATTENTION_BACKEND" ]]; then
BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND"
fi
eval "$BASE_CMD &"
# ── Wait for servers ──