diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 57c976030d..b045428e66 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2439,6 +2439,59 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/core/sched/ + - vllm/v1/core/kv_cache_coordinator.py + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh + +- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh + +- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/rocm.py + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt + - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh + - label: V1 e2e (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] diff --git a/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh b/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh index a34f07edc9..d3fee8d6ba 100755 --- a/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh @@ -9,8 +9,10 @@ PREFILL_GPU_ID=${PREFILL_GPU_ID:-0} DECODE_GPU_ID=${DECODE_GPU_ID:-1} MODEL=${MODEL:-"ibm-granite/granite-4.0-h-tiny"} GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.8} +VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} +ATTENTION_BACKEND=${ATTENTION_BACKEND:-FLASHINFER} -echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL)" +echo "Running Mamba prefix cache test (GPUs: P=$PREFILL_GPU_ID, D=$DECODE_GPU_ID, model=$MODEL, backend=$ATTENTION_BACKEND)" KV_CONFIG='{"kv_connector":"NixlConnector","kv_role":"kv_both"}' @@ -36,6 +38,14 @@ cleanup_instances() { cleanup_instances +EXTRA_ARGS=() +if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a EXTRA_ARGS <<< "$VLLM_SERVE_EXTRA_ARGS" +fi +if [[ -n "$ATTENTION_BACKEND" ]]; then + EXTRA_ARGS+=(--attention-backend "$ATTENTION_BACKEND") +fi + # Start prefill instance PREFILL_PORT=8001 CUDA_VISIBLE_DEVICES=$PREFILL_GPU_ID \ @@ -51,8 +61,8 @@ vllm serve $MODEL \ --trust-remote-code \ --enable-prefix-caching \ --mamba-cache-mode all \ - --attention-backend FLASHINFER \ - --kv-transfer-config "$KV_CONFIG" & + --kv-transfer-config "$KV_CONFIG" \ + "${EXTRA_ARGS[@]}" & # Start decode instance DECODE_PORT=8002 @@ -69,8 +79,8 @@ vllm serve $MODEL \ --trust-remote-code \ --enable-prefix-caching \ --mamba-cache-mode all \ - --attention-backend FLASHINFER \ - --kv-transfer-config "$KV_CONFIG" & + --kv-transfer-config "$KV_CONFIG" \ + "${EXTRA_ARGS[@]}" & echo "Waiting for prefill instance on port $PREFILL_PORT..." wait_for_server "$PREFILL_PORT" diff --git a/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh index 2e71858983..dae632dfce 100755 --- a/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh @@ -18,6 +18,7 @@ # Environment variables: # MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B) # GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6) +# ATTENTION_BACKEND - optional attention backend for vllm serve # VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve # SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test # SKIP_NORMAL_LAYOUT - set to 1 to skip the normal layout test @@ -34,9 +35,11 @@ fi GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6} BLOCK_SIZE=${BLOCK_SIZE:-128} +ATTENTION_BACKEND=${ATTENTION_BACKEND:-} VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} -GIT_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "") # ── KV transfer configs ───────────────────────────────────────────────── @@ -139,6 +142,9 @@ run_tests_for_model() { BASE_CMD="${BASE_CMD} $arg" done fi + if [[ -n "$ATTENTION_BACKEND" ]]; then + BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND" + fi eval "$BASE_CMD &" # ── Start decode instance ── @@ -161,6 +167,9 @@ run_tests_for_model() { BASE_CMD="${BASE_CMD} $arg" done fi + if [[ -n "$ATTENTION_BACKEND" ]]; then + BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND" + fi eval "$BASE_CMD &" # ── Wait for servers ── diff --git a/tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh b/tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh index a80950b341..de6c9abcc6 100755 --- a/tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh @@ -19,6 +19,7 @@ # MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B) # KV_CACHE_MEMORY_BYTES - GPU KV cache size in bytes (default: 268435456 = 256 MiB) # BLOCK_SIZE - KV cache block size (default: 128) +# ATTENTION_BACKEND - optional attention backend for vllm serve # VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve set -xe @@ -34,9 +35,11 @@ fi KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-268435456} # 256 MiB MAX_MODEL_LEN=${MAX_MODEL_LEN:-2048} BLOCK_SIZE=${BLOCK_SIZE:-128} +ATTENTION_BACKEND=${ATTENTION_BACKEND:-} VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} -GIT_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" # ── KV transfer config ────────────────────────────────────────────────── @@ -110,6 +113,9 @@ run_tests_for_model() { BASE_CMD="${BASE_CMD} $arg" done fi + if [[ -n "$ATTENTION_BACKEND" ]]; then + BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND" + fi eval "$BASE_CMD &" # ── Start decode instance ── @@ -133,6 +139,9 @@ run_tests_for_model() { BASE_CMD="${BASE_CMD} $arg" done fi + if [[ -n "$ATTENTION_BACKEND" ]]; then + BASE_CMD="${BASE_CMD} --attention-backend $ATTENTION_BACKEND" + fi eval "$BASE_CMD &" # ── Wait for servers ──