[ROCm][CI] Fix NIXL spec-decode acceptance startup and diagnostics (#41313)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -880,7 +880,7 @@ steps:
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
|
||||
- ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
|
||||
|
||||
- label: V1 e2e (2 GPUs) # TBD
|
||||
timeout_in_minutes: 180
|
||||
@@ -3318,7 +3318,7 @@ steps:
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ROCM_ATTN=1 bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
|
||||
- ATTENTION_BACKEND=ROCM_ATTN bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh
|
||||
|
||||
- label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD
|
||||
timeout_in_minutes: 180
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Probe a NIXL side-channel socket for handshake metadata readiness."""
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
|
||||
GET_META_MSG = b"get_meta_msg"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", required=True)
|
||||
parser.add_argument("--port", required=True, type=int)
|
||||
parser.add_argument("--rank", default=0, type=int)
|
||||
parser.add_argument("--timeout-ms", default=1000, type=int)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def make_zmq_path(host: str, port: int) -> str:
|
||||
try:
|
||||
if isinstance(ipaddress.ip_address(host), ipaddress.IPv6Address):
|
||||
return f"tcp://[{host}]:{port}"
|
||||
except ValueError:
|
||||
pass
|
||||
return f"tcp://{host}:{port}"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
ctx = zmq.Context()
|
||||
sock = ctx.socket(zmq.REQ)
|
||||
sock.setsockopt(zmq.LINGER, 0)
|
||||
sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms)
|
||||
try:
|
||||
sock.connect(make_zmq_path(args.host, args.port))
|
||||
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank)))
|
||||
sock.recv()
|
||||
finally:
|
||||
sock.close()
|
||||
ctx.term()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -51,6 +51,8 @@ PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
|
||||
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-16}
|
||||
SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
|
||||
NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}"
|
||||
|
||||
# Resolve the repository root from the script location instead of `.git`.
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
@@ -96,18 +98,57 @@ trap 'echo " Interrupted."; exit 130' INT TERM
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local deadline=600
|
||||
local server_pid=$2
|
||||
local server_name=$3
|
||||
local endpoint=${4:-/v1/completions}
|
||||
local deadline=${5:-600}
|
||||
local elapsed=0
|
||||
echo "Waiting for server on port ${port}..."
|
||||
echo "Waiting for ${server_name} on port ${port}..."
|
||||
while [ $elapsed -lt $deadline ]; do
|
||||
if curl -s "localhost:${port}/v1/completions" > /dev/null 2>&1; then
|
||||
echo "Server on port ${port} ready"
|
||||
if ! ps -p "$server_pid" > /dev/null 2>&1; then
|
||||
local status=0
|
||||
wait "$server_pid" || status=$?
|
||||
echo "FAIL: ${server_name} process ${server_pid} exited with status ${status} before port ${port} became ready"
|
||||
exit 1
|
||||
fi
|
||||
if curl -s "http://${SERVER_HOST}:${port}${endpoint}" > /dev/null 2>&1; then
|
||||
echo "${server_name} on port ${port} ready"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
echo "FAIL: Server on port ${port} did not start within ${deadline}s"
|
||||
echo "FAIL: ${server_name} on port ${port} did not start within ${deadline}s"
|
||||
exit 1
|
||||
}
|
||||
|
||||
wait_for_nixl_side_channel() {
|
||||
local host=$1
|
||||
local port=$2
|
||||
local server_pid=$3
|
||||
local server_name=$4
|
||||
local deadline=120
|
||||
local elapsed=0
|
||||
echo "Waiting for ${server_name} NIXL side channel on ${host}:${port}..."
|
||||
while [ $elapsed -lt $deadline ]; do
|
||||
if ! ps -p "$server_pid" > /dev/null 2>&1; then
|
||||
local status=0
|
||||
wait "$server_pid" || status=$?
|
||||
echo "FAIL: ${server_name} server process ${server_pid} exited with status ${status} before NIXL side channel ${host}:${port} became ready"
|
||||
exit 1
|
||||
fi
|
||||
if python3 "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py" \
|
||||
--host "$host" \
|
||||
--port "$port" \
|
||||
--timeout-ms 1000 > /dev/null 2>&1
|
||||
then
|
||||
echo "${server_name} NIXL side channel on ${host}:${port} ready"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
echo "FAIL: ${server_name} NIXL side channel ${host}:${port} did not start within ${deadline}s"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -158,6 +199,8 @@ run_test_for_device() {
|
||||
echo "KV buffer device: ${kv_device}"
|
||||
echo "Attention backend: ${ATTENTION_BACKEND}"
|
||||
echo "GPU platform: ${GPU_PLATFORM}"
|
||||
echo "Server host: ${SERVER_HOST}"
|
||||
echo "NIXL side channel: ${NIXL_SIDE_CHANNEL_HOST}"
|
||||
echo "GPUs available: ${ALL_GPUS[*]}"
|
||||
echo "================================================================"
|
||||
|
||||
@@ -167,7 +210,8 @@ run_test_for_device() {
|
||||
local DECODE_PORTS=()
|
||||
local GPU_IDX=0
|
||||
|
||||
# Start prefill instances
|
||||
# Start prefill instances and wait for each one before allocating the next
|
||||
# server. This keeps failures from leaving extra model servers spinning.
|
||||
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
|
||||
local GPU_ID="${ALL_GPUS[$GPU_IDX]}"
|
||||
GPU_IDX=$((GPU_IDX + 1))
|
||||
@@ -184,6 +228,7 @@ run_test_for_device() {
|
||||
${GPU_DEVICE_VAR}=$GPU_ID \
|
||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||
UCX_NET_DEVICES=all \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $MODEL_NAME \
|
||||
--port $PORT \
|
||||
@@ -195,12 +240,15 @@ run_test_for_device() {
|
||||
--kv-transfer-config "$kv_config" \
|
||||
--speculative-config "$PREFILL_SPEC_CONFIG" \
|
||||
--attention-backend $ATTENTION_BACKEND &
|
||||
local SERVER_PID=$!
|
||||
|
||||
PREFILL_HOSTS+=("localhost")
|
||||
PREFILL_HOSTS+=("$SERVER_HOST")
|
||||
PREFILL_PORTS+=("$PORT")
|
||||
wait_for_server "$PORT" "$SERVER_PID" "prefill"
|
||||
wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "prefill"
|
||||
done
|
||||
|
||||
# Start decode instances
|
||||
# Start decode instances after prefill is ready.
|
||||
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
|
||||
local GPU_ID="${ALL_GPUS[$GPU_IDX]}"
|
||||
GPU_IDX=$((GPU_IDX + 1))
|
||||
@@ -217,6 +265,7 @@ run_test_for_device() {
|
||||
${GPU_DEVICE_VAR}=$GPU_ID \
|
||||
VLLM_KV_CACHE_LAYOUT='HND' \
|
||||
UCX_NET_DEVICES=all \
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST=$NIXL_SIDE_CHANNEL_HOST \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $MODEL_NAME \
|
||||
--port $PORT \
|
||||
@@ -228,17 +277,12 @@ run_test_for_device() {
|
||||
--kv-transfer-config "$kv_config" \
|
||||
--speculative-config "$DECODE_SPEC_CONFIG" \
|
||||
--attention-backend $ATTENTION_BACKEND &
|
||||
local SERVER_PID=$!
|
||||
|
||||
DECODE_HOSTS+=("localhost")
|
||||
DECODE_HOSTS+=("$SERVER_HOST")
|
||||
DECODE_PORTS+=("$PORT")
|
||||
done
|
||||
|
||||
# Wait for servers
|
||||
for PORT in "${PREFILL_PORTS[@]}"; do
|
||||
wait_for_server "$PORT"
|
||||
done
|
||||
for PORT in "${DECODE_PORTS[@]}"; do
|
||||
wait_for_server "$PORT"
|
||||
wait_for_server "$PORT" "$SERVER_PID" "decode"
|
||||
wait_for_nixl_side_channel "$NIXL_SIDE_CHANNEL_HOST" "$SIDE_CHANNEL_PORT" "$SERVER_PID" "decode"
|
||||
done
|
||||
|
||||
# Start proxy
|
||||
@@ -250,12 +294,14 @@ run_test_for_device() {
|
||||
--prefiller-ports ${PREFILL_PORTS[*]} \
|
||||
--decoder-hosts ${DECODE_HOSTS[*]} \
|
||||
--decoder-ports ${DECODE_PORTS[*]} &
|
||||
local PROXY_PID=$!
|
||||
|
||||
sleep 5
|
||||
wait_for_server "$PROXY_PORT" "$PROXY_PID" "proxy" "/healthcheck" 60
|
||||
|
||||
# Run test
|
||||
echo "Running spec decode acceptance test (kv_buffer_device=${kv_device}, backend=${ATTENTION_BACKEND})..."
|
||||
DECODE_PORT=${DECODE_PORTS[0]} \
|
||||
SERVER_HOST=$SERVER_HOST \
|
||||
TEST_MODEL=$MODEL_NAME \
|
||||
python3 -m pytest -s -x "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py"
|
||||
|
||||
|
||||
@@ -27,7 +27,8 @@ from transformers import AutoTokenizer
|
||||
|
||||
from vllm.benchmarks.datasets import get_samples
|
||||
|
||||
PROXY_BASE_URL = "http://localhost:8192/v1"
|
||||
SERVER_HOST = os.environ.get("SERVER_HOST", "127.0.0.1")
|
||||
PROXY_BASE_URL = f"http://{SERVER_HOST}:8192/v1"
|
||||
DECODE_PORT = os.environ.get("DECODE_PORT", "8200")
|
||||
MODEL_NAME = os.environ.get("TEST_MODEL", "meta-llama/Llama-3.1-8B-Instruct")
|
||||
|
||||
@@ -101,7 +102,7 @@ def _get_mt_bench_prompts() -> list[str]:
|
||||
|
||||
def _fetch_metric(metric_name: str) -> float:
|
||||
"""Fetch a single counter metric from the decode server's /metrics."""
|
||||
url = f"http://localhost:{DECODE_PORT}/metrics"
|
||||
url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics"
|
||||
body = urlopen(url).read().decode()
|
||||
for line in body.split("\n"):
|
||||
if line.startswith(metric_name + "{") or line.startswith(metric_name + " "):
|
||||
@@ -111,7 +112,7 @@ def _fetch_metric(metric_name: str) -> float:
|
||||
|
||||
def _fetch_per_position_acceptance() -> dict[int, float]:
|
||||
"""Fetch per-position acceptance counts from decode /metrics."""
|
||||
url = f"http://localhost:{DECODE_PORT}/metrics"
|
||||
url = f"http://{SERVER_HOST}:{DECODE_PORT}/metrics"
|
||||
body = urlopen(url).read().decode()
|
||||
counts: dict[int, float] = {}
|
||||
for line in body.split("\n"):
|
||||
|
||||
@@ -413,7 +413,10 @@ class NixlConnectorWorker:
|
||||
|
||||
self.kv_cache_layout = get_kv_cache_layout()
|
||||
self.host_buffer_kv_cache_layout = self.kv_cache_layout
|
||||
logger.info("Detected attention backend %s", self.backend_name)
|
||||
logger.info(
|
||||
"Detected attention backend(s) %s",
|
||||
[backend.get_name() for backend in self.attn_backends],
|
||||
)
|
||||
logger.info("Detected kv cache layout %s", self.kv_cache_layout)
|
||||
|
||||
# lazy initialized in register_kv_caches
|
||||
@@ -870,9 +873,21 @@ class NixlConnectorWorker:
|
||||
else:
|
||||
self.block_len_per_layer.append(physical_page_size)
|
||||
|
||||
assert cache.shape[0] == num_blocks, (
|
||||
"All kv cache tensors must have the same number of blocks"
|
||||
)
|
||||
if cache.shape[0] != num_blocks:
|
||||
raise AssertionError(
|
||||
"All kv cache tensors must have the same number of "
|
||||
f"blocks; layer={layer_name}, "
|
||||
f"expected_num_blocks={num_blocks}, "
|
||||
f"cache_shape={tuple(cache.shape)}, "
|
||||
f"cache_stride={tuple(cache.stride())}, "
|
||||
f"layer_spec={type(layer_spec).__name__}, "
|
||||
f"backend={self.backend_name}, "
|
||||
"all_backends="
|
||||
f"{[backend.get_name() for backend in self.attn_backends]}, "
|
||||
f"kv_cache_layout={self.kv_cache_layout}, "
|
||||
"blocks_first="
|
||||
f"{self.transfer_topo.is_kv_layout_blocks_first}"
|
||||
)
|
||||
|
||||
if not self.use_mla:
|
||||
# Different kv cache shape is not supported by HeteroTP.
|
||||
|
||||
Reference in New Issue
Block a user