[ROCm][CI] Fix NIXL spec-decode acceptance startup and diagnostics (#41313)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-10 14:50:16 +08:00
committed by GitHub
parent 3f5bd482f5
commit f2840120f6
5 changed files with 137 additions and 27 deletions
+2 -2
View File
@@ -880,7 +880,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
- ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
- label: V1 e2e (2 GPUs) # TBD
timeout_in_minutes: 180
@@ -3318,7 +3318,7 @@ steps:
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
- ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
timeout_in_minutes: 180
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Probe a NIXL side-channel socket for handshake metadata readiness."""
import argparse
import ipaddress
import msgspec
import zmq
GET_META_MSG = b"get_meta_msg"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--host", required=True)
parser.add_argument("--port", required=True, type=int)
parser.add_argument("--rank", default=0, type=int)
parser.add_argument("--timeout-ms", default=1000, type=int)
return parser.parse_args()
def make_zmq_path(host: str, port: int) -> str:
try:
if isinstance(ipaddress.ip_address(host), ipaddress.IPv6Address):
return f"tcp://[{host}]:{port}"
except ValueError:
pass
return f"tcp://{host}:{port}"
def main() -> None:
args = parse_args()
ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.setsockopt(zmq.LINGER, 0)
sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms)
try:
sock.connect(make_zmq_path(args.host, args.port))
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank)))
sock.recv()
finally:
sock.close()
ctx.term()
if __name__ == "__main__":
main()
@@ -51,6 +51,8 @@ PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7}
BLOCK_SIZE=${BLOCK_SIZE:-16}
SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}"
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -96,18 +98,57 @@ trap 'echo " Interrupted."; exit 130' INT TERM
wait_for_server() {
local port=$1
local deadline=600
local server_pid=$2
local server_name=$3
local endpoint=${4:-/v1/completions}
local deadline=${5:-600}
local elapsed=0
echo "Waiting for server on port ${port}..."
echo "Waiting for ${server_name} on port ${port}..."
while [ $elapsed -lt $deadline ]; do
if curl -s "localhost:${port}/v1/completions" > /dev/null 2>&1; then
echo "Server on port ${port} ready"
if ! ps -p "$server_pid" > /dev/null 2>&1; then
local status=0
wait "$server_pid" || status=$?
echo "FAIL: ${server_name} process ${server_pid} exited with status ${status} before port ${port} became ready"
exit 1
fi
if curl -s "http://${SERVER_HOST}:${port}${endpoint}" > /dev/null 2>&1; then
echo "${server_name} on port ${port} ready"
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "FAIL: Server on port ${port} did not start within ${deadline}s"
echo "FAIL: ${server_name} on port ${port} did not start within ${deadline}s"
exit 1
}
wait_for_nixl_side_channel() {
local host=$1
local port=$2
local server_pid=$3
local server_name=$4
local deadline=120
local elapsed=0
echo "Waiting for ${server_name} NIXL side channel on ${host}:${port}..."
while [ $elapsed -lt $deadline ]; do
if ! ps -p "$server_pid" > /dev/null 2>&1; then
local status=0
wait "$server_pid" || status=$?
echo "FAIL: ${server_name} server process ${server_pid} exited with status ${status} before NIXL side channel ${host}:${port} became ready"
exit 1
fi
if python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py" \
--host "$host" \
--port "$port" \
--timeout-ms 1000 > /dev/null 2>&1
then
echo "${server_name} NIXL side channel on ${host}:${port} ready"
return 0
fi
sleep 2
elapsed=$((elapsed + 2))
done
echo "FAIL: ${server_name} NIXL side channel ${host}:${port} did not start within ${deadline}s"
exit 1
}
@@ -158,6 +199,8 @@ run_test_for_device() {
echo "KV buffer device: ${kv_device}"
echo "Attention backend: ${ATTENTION_BACKEND}"
echo "GPU platform: ${GPU_PLATFORM}"
echo "Server host: ${SERVER_HOST}"
echo "NIXL side channel: ${NIXL_SIDE_CHANNEL_HOST}"
echo "GPUs available: ${ALL_GPUS[*]}"
echo "================================================================"
@@ -167,7 +210,8 @@ run_test_for_device() {
local DECODE_PORTS=()
local GPU_IDX=0
# Start prefill instances
# Start prefill instances and wait for each one before allocating the next
# server. This keeps failures from leaving extra model servers spinning.
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
local GPU_ID="${ALL_GPUS[$GPU_IDX]}"
GPU_IDX=$((GPU_IDX + 1))
@@ -184,6 +228,7 @@ run_test_for_device() {
${GPU_DEVICE_VAR}=$GPU_ID \
VLLM_KV_CACHE_LAYOUT='HND' \
UCX_NET_DEVICES=all \
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
vllm serve $MODEL_NAME \
--port $PORT \
@@ -195,12 +240,15 @@ run_test_for_device() {
--kv-transfer-config "$kv_config" \
--speculative-config "$PREFILL_SPEC_CONFIG" \
--attention-backend $ATTENTION_BACKEND &
local SERVER_PID=$!
PREFILL_HOSTS+=("localhost")
PREFILL_HOSTS+=("$SERVER_HOST")
PREFILL_PORTS+=("$PORT")
wait_for_server "$PORT" "$SERVER_PID" "prefill"
wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "prefill"
done
# Start decode instances
# Start decode instances after prefill is ready.
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
local GPU_ID="${ALL_GPUS[$GPU_IDX]}"
GPU_IDX=$((GPU_IDX + 1))
@@ -217,6 +265,7 @@ run_test_for_device() {
${GPU_DEVICE_VAR}=$GPU_ID \
VLLM_KV_CACHE_LAYOUT='HND' \
UCX_NET_DEVICES=all \
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
vllm serve $MODEL_NAME \
--port $PORT \
@@ -228,17 +277,12 @@ run_test_for_device() {
--kv-transfer-config "$kv_config" \
--speculative-config "$DECODE_SPEC_CONFIG" \
--attention-backend $ATTENTION_BACKEND &
local SERVER_PID=$!
DECODE_HOSTS+=("localhost")
DECODE_HOSTS+=("$SERVER_HOST")
DECODE_PORTS+=("$PORT")
done
# Wait for servers
for PORT in "${PREFILL_PORTS[@]}"; do
wait_for_server "$PORT"
done
for PORT in "${DECODE_PORTS[@]}"; do
wait_for_server "$PORT"
wait_for_server "$PORT" "$SERVER_PID" "decode"
wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "decode"
done
# Start proxy
@@ -250,12 +294,14 @@ run_test_for_device() {
--prefiller-ports ${PREFILL_PORTS[*]} \
--decoder-hosts ${DECODE_HOSTS[*]} \
--decoder-ports ${DECODE_PORTS[*]} &
local PROXY_PID=$!
sleep 5
wait_for_server "$PROXY_PORT" "$PROXY_PID" "proxy" "/healthcheck" 60
# Run test
echo "Running spec decode acceptance test (kv_buffer_device=${kv_device}, backend=${ATTENTION_BACKEND})..."
DECODE_PORT=${DECODE_PORTS[0]} \
SERVER_HOST=$SERVER_HOST \
TEST_MODEL=$MODEL_NAME \
python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py"
@@ -27,7 +27,8 @@ from transformers import AutoTokenizer
from vllm.benchmarks.datasets import get_samples
PROXY_BASE_URL = "http://localhost:8192/v1"
SERVER_HOST = os.environ.get("SERVER_HOST", "127.0.0.1")
PROXY_BASE_URL = f"http://{SERVER_HOST}:8192/v1"
DECODE_PORT = os.environ.get("DECODE_PORT", "8200")
MODEL_NAME = os.environ.get("TEST_MODEL", "meta-llama/Llama-3.1-8B-Instruct")
@@ -101,7 +102,7 @@ def _get_mt_bench_prompts() -> list[str]:
def _fetch_metric(metric_name: str) -> float:
"""Fetch a single counter metric from the decode server's /metrics."""
url = f"http://localhost:{DECODE_PORT}/metrics"
url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics"
body = urlopen(url).read().decode()
for line in body.split("\n"):
if line.startswith(metric_name + "{") or line.startswith(metric_name + " "):
@@ -111,7 +112,7 @@ def _fetch_metric(metric_name: str) -> float:
def _fetch_per_position_acceptance() -> dict[int, float]:
"""Fetch per-position acceptance counts from decode /metrics."""
url = f"http://localhost:{DECODE_PORT}/metrics"
url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics"
body = urlopen(url).read().decode()
counts: dict[int, float] = {}
for line in body.split("\n"):
@@ -413,7 +413,10 @@ class NixlConnectorWorker:
self.kv_cache_layout = get_kv_cache_layout()
self.host_buffer_kv_cache_layout = self.kv_cache_layout
logger.info("Detected attention backend %s", self.backend_name)
logger.info(
"Detected attention backend(s) %s",
[backend.get_name() for backend in self.attn_backends],
)
logger.info("Detected kv cache layout %s", self.kv_cache_layout)
# lazy initialized in register_kv_caches
@@ -870,9 +873,21 @@ class NixlConnectorWorker:
else:
self.block_len_per_layer.append(physical_page_size)
assert cache.shape[0] == num_blocks, (
"All kv cache tensors must have the same number of blocks"
)
if cache.shape[0] != num_blocks:
raise AssertionError(
"All kv cache tensors must have the same number of "
f"blocks; layer={layer_name}, "
f"expected_num_blocks={num_blocks}, "
f"cache_shape={tuple(cache.shape)}, "
f"cache_stride={tuple(cache.stride())}, "
f"layer_spec={type(layer_spec).__name__}, "
f"backend={self.backend_name}, "
"all_backends="
f"{[backend.get_name() for backend in self.attn_backends]}, "
f"kv_cache_layout={self.kv_cache_layout}, "
"blocks_first="
f"{self.transfer_topo.is_kv_layout_blocks_first}"
)
if not self.use_mla:
# Different kv cache shape is not supported by HeteroTP.