diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 9fb3503c8d..1f52149e4d 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -880,7 +880,7 @@ steps: - vllm/platforms/rocm.py commands: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh + - ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh - label: V1 e2e (2 GPUs) # TBD timeout_in_minutes: 180 @@ -3318,7 +3318,7 @@ steps: - vllm/platforms/rocm.py commands: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh + - ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh - label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD timeout_in_minutes: 180 diff --git a/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py b/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py new file mode 100644 index 0000000000..24ecbd795e --- /dev/null +++ b/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Probe a NIXL side-channel socket for handshake metadata readiness.""" + +import argparse +import ipaddress + +import msgspec +import zmq + +GET_META_MSG = b"get_meta_msg" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--host", required=True) + parser.add_argument("--port", required=True, type=int) + parser.add_argument("--rank", default=0, type=int) + parser.add_argument("--timeout-ms", default=1000, type=int) + return parser.parse_args() + + +def make_zmq_path(host: str, port: int) -> str: + try: + if isinstance(ipaddress.ip_address(host), ipaddress.IPv6Address): + return f"tcp://[{host}]:{port}" + except ValueError: + pass + return f"tcp://{host}:{port}" + + +def main() -> None: + args = parse_args() + ctx = zmq.Context() + sock = ctx.socket(zmq.REQ) + sock.setsockopt(zmq.LINGER, 0) + sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms) + try: + sock.connect(make_zmq_path(args.host, args.port)) + sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank))) + sock.recv() + finally: + sock.close() + ctx.term() + + +if __name__ == "__main__": + main() diff --git a/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh b/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh index a82dae2d51..fdec5f40cb 100755 --- a/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh +++ b/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh @@ -51,6 +51,8 @@ PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7} BLOCK_SIZE=${BLOCK_SIZE:-16} +SERVER_HOST="${SERVER_HOST:-127.0.0.1}" +NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}" # Resolve the repository root from the script location instead of `.git`. SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" @@ -96,18 +98,57 @@ trap 'echo " Interrupted."; exit 130' INT TERM wait_for_server() { local port=$1 - local deadline=600 + local server_pid=$2 + local server_name=$3 + local endpoint=${4:-/v1/completions} + local deadline=${5:-600} local elapsed=0 - echo "Waiting for server on port ${port}..." + echo "Waiting for ${server_name} on port ${port}..." while [ $elapsed -lt $deadline ]; do - if curl -s "localhost:${port}/v1/completions" > /dev/null 2>&1; then - echo "Server on port ${port} ready" + if ! ps -p "$server_pid" > /dev/null 2>&1; then + local status=0 + wait "$server_pid" || status=$? + echo "FAIL: ${server_name} process ${server_pid} exited with status ${status} before port ${port} became ready" + exit 1 + fi + if curl -s "http://${SERVER_HOST}:${port}${endpoint}" > /dev/null 2>&1; then + echo "${server_name} on port ${port} ready" return 0 fi sleep 2 elapsed=$((elapsed + 2)) done - echo "FAIL: Server on port ${port} did not start within ${deadline}s" + echo "FAIL: ${server_name} on port ${port} did not start within ${deadline}s" + exit 1 +} + +wait_for_nixl_side_channel() { + local host=$1 + local port=$2 + local server_pid=$3 + local server_name=$4 + local deadline=120 + local elapsed=0 + echo "Waiting for ${server_name} NIXL side channel on ${host}:${port}..." + while [ $elapsed -lt $deadline ]; do + if ! ps -p "$server_pid" > /dev/null 2>&1; then + local status=0 + wait "$server_pid" || status=$? + echo "FAIL: ${server_name} server process ${server_pid} exited with status ${status} before NIXL side channel ${host}:${port} became ready" + exit 1 + fi + if python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py" \ + --host "$host" \ + --port "$port" \ + --timeout-ms 1000 > /dev/null 2>&1 + then + echo "${server_name} NIXL side channel on ${host}:${port} ready" + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "FAIL: ${server_name} NIXL side channel ${host}:${port} did not start within ${deadline}s" exit 1 } @@ -158,6 +199,8 @@ run_test_for_device() { echo "KV buffer device: ${kv_device}" echo "Attention backend: ${ATTENTION_BACKEND}" echo "GPU platform: ${GPU_PLATFORM}" + echo "Server host: ${SERVER_HOST}" + echo "NIXL side channel: ${NIXL_SIDE_CHANNEL_HOST}" echo "GPUs available: ${ALL_GPUS[*]}" echo "================================================================" @@ -167,7 +210,8 @@ run_test_for_device() { local DECODE_PORTS=() local GPU_IDX=0 - # Start prefill instances + # Start prefill instances and wait for each one before allocating the next + # server. This keeps failures from leaving extra model servers spinning. for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do local GPU_ID="${ALL_GPUS[$GPU_IDX]}" GPU_IDX=$((GPU_IDX + 1)) @@ -184,6 +228,7 @@ run_test_for_device() { ${GPU_DEVICE_VAR}=$GPU_ID \ VLLM_KV_CACHE_LAYOUT='HND' \ UCX_NET_DEVICES=all \ + VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \ VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \ vllm serve $MODEL_NAME \ --port $PORT \ @@ -195,12 +240,15 @@ run_test_for_device() { --kv-transfer-config "$kv_config" \ --speculative-config "$PREFILL_SPEC_CONFIG" \ --attention-backend $ATTENTION_BACKEND & + local SERVER_PID=$! - PREFILL_HOSTS+=("localhost") + PREFILL_HOSTS+=("$SERVER_HOST") PREFILL_PORTS+=("$PORT") + wait_for_server "$PORT" "$SERVER_PID" "prefill" + wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "prefill" done - # Start decode instances + # Start decode instances after prefill is ready. for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do local GPU_ID="${ALL_GPUS[$GPU_IDX]}" GPU_IDX=$((GPU_IDX + 1)) @@ -217,6 +265,7 @@ run_test_for_device() { ${GPU_DEVICE_VAR}=$GPU_ID \ VLLM_KV_CACHE_LAYOUT='HND' \ UCX_NET_DEVICES=all \ + VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \ VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \ vllm serve $MODEL_NAME \ --port $PORT \ @@ -228,17 +277,12 @@ run_test_for_device() { --kv-transfer-config "$kv_config" \ --speculative-config "$DECODE_SPEC_CONFIG" \ --attention-backend $ATTENTION_BACKEND & + local SERVER_PID=$! - DECODE_HOSTS+=("localhost") + DECODE_HOSTS+=("$SERVER_HOST") DECODE_PORTS+=("$PORT") - done - - # Wait for servers - for PORT in "${PREFILL_PORTS[@]}"; do - wait_for_server "$PORT" - done - for PORT in "${DECODE_PORTS[@]}"; do - wait_for_server "$PORT" + wait_for_server "$PORT" "$SERVER_PID" "decode" + wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "decode" done # Start proxy @@ -250,12 +294,14 @@ run_test_for_device() { --prefiller-ports ${PREFILL_PORTS[*]} \ --decoder-hosts ${DECODE_HOSTS[*]} \ --decoder-ports ${DECODE_PORTS[*]} & + local PROXY_PID=$! - sleep 5 + wait_for_server "$PROXY_PORT" "$PROXY_PID" "proxy" "/healthcheck" 60 # Run test echo "Running spec decode acceptance test (kv_buffer_device=${kv_device}, backend=${ATTENTION_BACKEND})..." DECODE_PORT=${DECODE_PORTS[0]} \ + SERVER_HOST=$SERVER_HOST \ TEST_MODEL=$MODEL_NAME \ python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py" diff --git a/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py b/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py index b747f953a2..67cb753a98 100644 --- a/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py +++ b/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py @@ -27,7 +27,8 @@ from transformers import AutoTokenizer from vllm.benchmarks.datasets import get_samples -PROXY_BASE_URL = "http://localhost:8192/v1" +SERVER_HOST = os.environ.get("SERVER_HOST", "127.0.0.1") +PROXY_BASE_URL = f"http://{SERVER_HOST}:8192/v1" DECODE_PORT = os.environ.get("DECODE_PORT", "8200") MODEL_NAME = os.environ.get("TEST_MODEL", "meta-llama/Llama-3.1-8B-Instruct") @@ -101,7 +102,7 @@ def _get_mt_bench_prompts() -> list[str]: def _fetch_metric(metric_name: str) -> float: """Fetch a single counter metric from the decode server's /metrics.""" - url = f"http://localhost:{DECODE_PORT}/metrics" + url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics" body = urlopen(url).read().decode() for line in body.split("\n"): if line.startswith(metric_name + "{") or line.startswith(metric_name + " "): @@ -111,7 +112,7 @@ def _fetch_metric(metric_name: str) -> float: def _fetch_per_position_acceptance() -> dict[int, float]: """Fetch per-position acceptance counts from decode /metrics.""" - url = f"http://localhost:{DECODE_PORT}/metrics" + url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics" body = urlopen(url).read().decode() counts: dict[int, float] = {} for line in body.split("\n"): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index 37a810691f..18144f2ef5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -413,7 +413,10 @@ class NixlConnectorWorker: self.kv_cache_layout = get_kv_cache_layout() self.host_buffer_kv_cache_layout = self.kv_cache_layout - logger.info("Detected attention backend %s", self.backend_name) + logger.info( + "Detected attention backend(s) %s", + [backend.get_name() for backend in self.attn_backends], + ) logger.info("Detected kv cache layout %s", self.kv_cache_layout) # lazy initialized in register_kv_caches @@ -870,9 +873,21 @@ class NixlConnectorWorker: else: self.block_len_per_layer.append(physical_page_size) - assert cache.shape[0] == num_blocks, ( - "All kv cache tensors must have the same number of blocks" - ) + if cache.shape[0] != num_blocks: + raise AssertionError( + "All kv cache tensors must have the same number of " + f"blocks; layer={layer_name}, " + f"expected_num_blocks={num_blocks}, " + f"cache_shape={tuple(cache.shape)}, " + f"cache_stride={tuple(cache.stride())}, " + f"layer_spec={type(layer_spec).__name__}, " + f"backend={self.backend_name}, " + "all_backends=" + f"{[backend.get_name() for backend in self.attn_backends]}, " + f"kv_cache_layout={self.kv_cache_layout}, " + "blocks_first=" + f"{self.transfer_topo.is_kv_layout_blocks_first}" + ) if not self.use_mla: # Different kv cache shape is not supported by HeteroTP.