[KVConnector][NIXL] Support pipeline-parallel prefill in push mode (#45880)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -39,6 +39,19 @@ steps:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: Push NixlConnector PP prefill PD accuracy (4 GPUs)
|
||||
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- tests/v1/kv_connector/nixl_push_integration/
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs)
|
||||
key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
|
||||
timeout_in_minutes: 30
|
||||
|
||||
@@ -9,9 +9,9 @@ the same; the difference is in how P and D coordinate the KV transfer:
|
||||
* Pull mode: proxy forwards P's ``kv_transfer_params`` (including
|
||||
``remote_block_ids``) to D, and D pulls KV from P via NIXL READ.
|
||||
* Push mode: proxy hands D **only** P's coordinates
|
||||
(``remote_engine_id``, ``remote_host``, ``remote_port``, ``tp_size``)
|
||||
and the shared ``remote_request_id``. D registers its locally allocated
|
||||
blocks with P over a NIXL notification; P then pushes the KV to D via
|
||||
(``remote_engine_id``, ``remote_host``, ``remote_port``, ``tp_size``,
|
||||
``pp_size``) and the shared ``remote_request_id``. D registers its locally
|
||||
allocated blocks with P over a NIXL notification; P then pushes the KV to D via
|
||||
NIXL WRITE.
|
||||
|
||||
Launch multiple vLLM instances configured with ``NixlPushConnector`` and
|
||||
@@ -26,6 +26,7 @@ disagg_proxy_pushconnector_demo.py \
|
||||
--prefill-kv-host 10.0.0.1 \
|
||||
--prefill-side-channel-port 5600 \
|
||||
--prefill-tp-size 1 \
|
||||
--prefill-pp-size 1 \
|
||||
--port 8000
|
||||
"""
|
||||
|
||||
@@ -89,6 +90,7 @@ class PushProxy:
|
||||
prefill_kv_host: str,
|
||||
prefill_side_channel_port: int,
|
||||
prefill_tp_size: int,
|
||||
prefill_pp_size: int,
|
||||
custom_create_completion: Callable[[Request], StreamingResponse] | None = None,
|
||||
custom_create_chat_completion: Callable[[Request], StreamingResponse]
|
||||
| None = None,
|
||||
@@ -110,6 +112,7 @@ class PushProxy:
|
||||
"remote_host": prefill_kv_host,
|
||||
"remote_port": prefill_side_channel_port,
|
||||
"tp_size": prefill_tp_size,
|
||||
"pp_size": prefill_pp_size,
|
||||
}
|
||||
|
||||
self.custom_create_completion = custom_create_completion
|
||||
@@ -198,6 +201,7 @@ class PushProxy:
|
||||
"prefill_kv_host": self.push_metadata["remote_host"],
|
||||
"prefill_side_channel_port": self.push_metadata["remote_port"],
|
||||
"prefill_tp_size": self.push_metadata["tp_size"],
|
||||
"prefill_pp_size": self.push_metadata["pp_size"],
|
||||
}
|
||||
|
||||
# ── push-mode request handling ──────────────────────────────────── #
|
||||
@@ -265,7 +269,7 @@ class PushProxy:
|
||||
generator = self.forward_request(
|
||||
f"http://{decode_instance}{path}", decode_request, headers
|
||||
)
|
||||
return StreamingResponse(generator)
|
||||
return StreamingResponse(generator, media_type="application/json")
|
||||
|
||||
async def create_completion(self, raw_request: Request):
|
||||
try:
|
||||
@@ -316,6 +320,7 @@ class PushProxyServer:
|
||||
prefill_kv_host=args.prefill_kv_host,
|
||||
prefill_side_channel_port=args.prefill_side_channel_port,
|
||||
prefill_tp_size=args.prefill_tp_size,
|
||||
prefill_pp_size=args.prefill_pp_size,
|
||||
custom_create_completion=create_completion,
|
||||
custom_create_chat_completion=create_chat_completion,
|
||||
)
|
||||
@@ -420,6 +425,12 @@ def parse_args():
|
||||
default=1,
|
||||
help="Tensor parallel size of the prefill vLLM instance",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prefill-pp-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Pipeline parallel size of the prefill vLLM instance",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -40,7 +40,6 @@ sw_attn_configs=(
|
||||
# Gemma4: SW + cross-layer KV sharing
|
||||
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-4-E2B-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
|
||||
# Select config array based on DP_EP env var
|
||||
if [[ -n "${DP_EP:-}" ]]; then
|
||||
configs=("${dp_ep_configs[@]}")
|
||||
|
||||
@@ -15,6 +15,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", required=True)
|
||||
parser.add_argument("--port", required=True, type=int)
|
||||
parser.add_argument("--pp-rank", default=0, type=int)
|
||||
parser.add_argument("--rank", default=0, type=int)
|
||||
parser.add_argument("--timeout-ms", default=1000, type=int)
|
||||
return parser.parse_args()
|
||||
@@ -37,7 +38,7 @@ def main() -> None:
|
||||
sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms)
|
||||
try:
|
||||
sock.connect(make_zmq_path(args.host, args.port))
|
||||
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank)))
|
||||
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.pp_rank, args.rank)))
|
||||
sock.recv()
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
@@ -49,13 +49,16 @@ else
|
||||
KV_EXTRA_CONFIG=''
|
||||
fi
|
||||
|
||||
# Connector: default pull NixlConnector; NixlPushConnector enables PP prefill.
|
||||
KV_CONNECTOR=${KV_CONNECTOR:-NixlConnector}
|
||||
|
||||
# Build the kv-transfer-config for P and D
|
||||
if [[ "$KV_BUFFER_DEVICE" == "cuda" ]]; then
|
||||
KV_CONFIG_P='{"kv_connector":"NixlConnector","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
KV_CONFIG_D='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
KV_CONFIG_P='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
KV_CONFIG_D='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
|
||||
else
|
||||
KV_CONFIG_P="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
KV_CONFIG_D="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
KV_CONFIG_P="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
KV_CONFIG_D="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
|
||||
fi
|
||||
|
||||
# Models to run
|
||||
@@ -72,6 +75,7 @@ fi
|
||||
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1
|
||||
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1
|
||||
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
|
||||
PREFILLER_PP_SIZE=${PREFILLER_PP_SIZE:-1} # >1 requires NixlPushConnector
|
||||
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
|
||||
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
|
||||
@@ -138,8 +142,9 @@ run_tests_for_model() {
|
||||
# Calculate GPU ID - we'll distribute across available GPUs
|
||||
GPU_ID=$((i % $(get_num_gpus)))
|
||||
NEXT_GPU=${GPU_ID}
|
||||
# If PREFILLER_TP_SIZE is more than 1
|
||||
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
|
||||
# Reserve TP*PP GPUs for the prefiller (TP shards across PP stages).
|
||||
PREFILLER_WORLD_SIZE=$((PREFILLER_TP_SIZE * PREFILLER_PP_SIZE))
|
||||
for (( j=1; j < PREFILLER_WORLD_SIZE; j++ )); do
|
||||
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
|
||||
GPU_ID="${GPU_ID},${NEXT_GPU}"
|
||||
done
|
||||
@@ -161,6 +166,7 @@ run_tests_for_model() {
|
||||
--block-size ${PREFILL_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--pipeline-parallel-size $PREFILLER_PP_SIZE \
|
||||
--kv-transfer-config '$KV_CONFIG_P'"
|
||||
if [[ "$ENFORCE_EAGER" == "1" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --enforce-eager"
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# NixlPushConnector (push-mode PD) integration tests. Kept separate from the
|
||||
# pull-mode sweep in ../nixl_integration so push coverage can grow on its own;
|
||||
# the per-config runner is shared.
|
||||
SCRIPT="v1/kv_connector/nixl_integration/run_accuracy_test.sh"
|
||||
IMPORT_CANARY="v1/kv_connector/nixl_integration/test_nixl_imports.py"
|
||||
|
||||
echo "=== Running NIXL import canary ==="
|
||||
python3 -m pytest -s -x "${IMPORT_CANARY}"
|
||||
|
||||
# PP=2 prefill -> TP=2 decode: each PP stage writes its layer slice into the
|
||||
# decoder's matching region range (non-MLA and MLA contiguous-slice paths).
|
||||
configs=(
|
||||
"KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.6 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2" # non-MLA, Qwen3-0.6B
|
||||
"KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.8 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA
|
||||
)
|
||||
|
||||
run_tests() {
|
||||
local label=$1
|
||||
local extra_args=$2
|
||||
|
||||
echo "=== Running push tests (${label}) ==="
|
||||
for cfg in "${configs[@]}"; do
|
||||
local -a cfg_parts extra_args_parts
|
||||
read -r -a cfg_parts <<< "$cfg"
|
||||
read -r -a extra_args_parts <<< "$extra_args"
|
||||
|
||||
echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
|
||||
# Use 'env' to safely set variables without eval; keep argv splitting
|
||||
# safe and SC2086-clean via arrays.
|
||||
if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
|
||||
echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "✅ All ${label} push tests passed!"
|
||||
}
|
||||
|
||||
run_tests "default backend" ""
|
||||
@@ -408,11 +408,10 @@ def test_kv_transfer_handshake(dist_init):
|
||||
decoder = msgspec.msgpack.Decoder(NixlAgentMetadata)
|
||||
expected_agent_metadata = decoder.decode(metadata.agent_metadata_bytes)
|
||||
|
||||
# The scheduler connector expects metadata to be in
|
||||
# dict[int, KVConnectorHandshakeMetadata], where the first key is
|
||||
# the dp_rank, the second key is the tp_rank.
|
||||
# The scheduler connector expects metadata keyed by
|
||||
# (pp_rank, tp_rank).
|
||||
scheduler_connector = scheduler.get_kv_connector()
|
||||
scheduler_connector.set_xfer_handshake_metadata({0: metadata})
|
||||
scheduler_connector.set_xfer_handshake_metadata_pp_aware({(0, 0): metadata})
|
||||
|
||||
# Simulate a request that finishes prefill, which returns
|
||||
# corresponding NixlConnectorMetadata for decode instance.
|
||||
@@ -502,8 +501,14 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
)
|
||||
|
||||
def _nixl_handshake(
|
||||
self, host: str, port: int, remote_tp_size: int, expected_engine_id: str
|
||||
) -> dict[int, str]:
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
remote_tp_size: int,
|
||||
expected_engine_id: str,
|
||||
remote_pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> dict[tuple[int, int], str]:
|
||||
# Mimic slow _nixl_handshake, as well as bypass zmq communication.
|
||||
time.sleep(self._hand_shake_latency)
|
||||
# These should've been done in register_kv_caches(), called by
|
||||
@@ -533,7 +538,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
# When remote tp_size > local tp_size, handshake with multiple
|
||||
# remote ranks.
|
||||
num_handshakes = 1 if tp_ratio > 0 else -tp_ratio
|
||||
remote_agents: dict[int, str] = {}
|
||||
remote_agents: dict[tuple[int, int], str] = {}
|
||||
for remote_tp_rank in range(num_handshakes):
|
||||
remote_agent_name = self.add_remote_agent(
|
||||
NixlAgentMetadata(
|
||||
@@ -554,7 +559,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
remote_tp_rank=remote_tp_rank,
|
||||
remote_tp_size=remote_tp_size,
|
||||
)
|
||||
remote_agents[remote_tp_rank] = remote_agent_name
|
||||
remote_agents[(0, remote_tp_rank)] = remote_agent_name
|
||||
return remote_agents
|
||||
|
||||
|
||||
@@ -749,7 +754,7 @@ class TestNixlHandshake:
|
||||
|
||||
def check_handshake(remote_tp_size: int):
|
||||
tp_ratio = remote_tp_size // local_tp_size
|
||||
assert set(remote_agents.keys()) == set(range(tp_ratio))
|
||||
assert set(remote_agents.keys()) == {(0, r) for r in range(tp_ratio)}
|
||||
|
||||
remote_engine_id = worker.REMOTE_ENGINE_ID
|
||||
remote_info = worker.transfer_topo.get_engine_info(remote_engine_id)
|
||||
@@ -2044,7 +2049,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
|
||||
# P TP = 2 * D TP case, we should register 2 local handles
|
||||
worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]}
|
||||
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
|
||||
worker._remote_agents = {"engine1": {0: "agent1"}}
|
||||
worker._remote_agents = {"engine1": {(0, 0): "agent1"}}
|
||||
# _cleanup_remote_engine (called by shutdown) also clears these:
|
||||
worker.kv_caches_base_addr["engine1"] = {0: [0xABC]}
|
||||
worker.dst_num_blocks["engine1"] = 50
|
||||
@@ -2097,7 +2102,7 @@ def _setup_worker_with_remote_engine(
|
||||
)
|
||||
|
||||
engine_id = "remote-engine-1"
|
||||
worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"}
|
||||
worker._remote_agents[engine_id] = {(0, 0): "agent_0", (0, 1): "agent_1"}
|
||||
worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200}
|
||||
worker.kv_caches_base_addr[engine_id] = {0: [0xABC]}
|
||||
worker.dst_num_blocks[engine_id] = 50
|
||||
@@ -2993,7 +2998,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
||||
local_block_len=worker.block_size * 4096,
|
||||
)
|
||||
worker._remote_agents[remote_engine_id] = {
|
||||
rank: f"agent_p{rank}" for rank in range(prefill_tp_size)
|
||||
(0, rank): f"agent_p{rank}" for rank in range(prefill_tp_size)
|
||||
}
|
||||
worker.dst_xfer_side_handles = {
|
||||
remote_engine_id: {rank: 100 + rank for rank in range(prefill_tp_size)}
|
||||
@@ -3044,7 +3049,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
|
||||
|
||||
# Broadcast goes to ranks {1, 2, 3} only, never to the read target.
|
||||
expected_recipients = {
|
||||
worker._remote_agents[remote_engine_id][r]
|
||||
worker._remote_agents[remote_engine_id][(0, r)]
|
||||
for r in range(1, prefill_tp_size)
|
||||
}
|
||||
assert {agent for agent, _ in send_notif_calls} == expected_recipients
|
||||
|
||||
@@ -29,9 +29,11 @@ from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
PUSH_REG_NOTIF_PREFIX,
|
||||
NixlAgentMetadata,
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
@@ -512,7 +514,7 @@ class TestPushWriterNotifs:
|
||||
# Pretend the writer thread already forwarded a completion notif
|
||||
# for a request whose KV is being received.
|
||||
request_id = "req-recv-1"
|
||||
w._recving_metadata[request_id] = MagicMock()
|
||||
w._recving_metadata[request_id] = MagicMock(pp_size=1)
|
||||
# Compose the standard completion notif: req_id:tp_size.
|
||||
notif_msg = f"{request_id}:1".encode()
|
||||
w._pending_completion_notifs.put(notif_msg)
|
||||
@@ -813,3 +815,177 @@ class TestPushWriterNegative:
|
||||
assert w._reqs_to_send[rid] >= now
|
||||
# Unknown request must not be inserted by the heartbeat path.
|
||||
assert "req-unknown" not in w._reqs_to_send
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Pipeline-parallel producer (push-mode PP-disagg) #
|
||||
# ----------------------------------------------------------------- #
|
||||
|
||||
|
||||
class TestPushPipelineParallel:
|
||||
"""PP-sharded producer: per-stage completion counting, pp_size plumbing,
|
||||
and remote-region slicing."""
|
||||
|
||||
def test_completion_waits_for_one_notif_per_pp_stage(self):
|
||||
"""Each PP stage WRITEs its own layers and sends one notif; D must
|
||||
collect pp_size notifs before reporting the recv done."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
request_id = "req-pp-2"
|
||||
w._recving_metadata[request_id] = MagicMock(pp_size=2)
|
||||
notif = f"{request_id}:1".encode()
|
||||
|
||||
# First stage: counted, not yet done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
assert request_id not in w._recving_transfers
|
||||
assert w.consumer_notification_counts_by_req[request_id] == 1
|
||||
|
||||
# Second (final) stage: now reported done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
assert request_id in w._recving_transfers
|
||||
assert request_id not in w.consumer_notification_counts_by_req
|
||||
|
||||
def test_req_meta_reads_pp_size_from_kv_transfer_params(self):
|
||||
"""D learns the producer's pp_size from kv_transfer_params (forwarded
|
||||
by the proxy) and defaults to 1 when absent."""
|
||||
metadata = NixlConnectorMetadata()
|
||||
params = {
|
||||
"remote_block_ids": ([0],),
|
||||
"remote_engine_id": "p-engine",
|
||||
"remote_request_id": "p-req",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"tp_size": 1,
|
||||
"pp_size": 2,
|
||||
}
|
||||
metadata.add_new_req_to_recv("req", ([0],), params)
|
||||
assert metadata.reqs_to_recv["req"].pp_size == 2
|
||||
|
||||
params.pop("pp_size")
|
||||
metadata.add_new_req_to_recv("req-default", ([0],), params)
|
||||
assert metadata.reqs_to_recv["req-default"].pp_size == 1
|
||||
|
||||
def test_add_remote_agent_slices_remote_regions_to_local_pp_window(self):
|
||||
"""With PP>1 the producer registered regions for the full model, but
|
||||
this worker holds only a contiguous layer slice; add_remote_agent
|
||||
trims the remote region list to [offset : offset + num_local_regions]
|
||||
before building descriptors. We stop right after the slice via a
|
||||
sentinel on the next collaborator call."""
|
||||
block_len = 4096 * 16
|
||||
w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown
|
||||
w.pp_size = 2
|
||||
w._remote_region_offset = 2 # this worker owns layers [2, 4)
|
||||
w.block_len_per_layer = [block_len, block_len] # 2 local layers
|
||||
w.nixl_wrapper = MagicMock()
|
||||
|
||||
class _StopAfterSlice(RuntimeError):
|
||||
pass
|
||||
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.register_remote_engine.side_effect = _StopAfterSlice()
|
||||
|
||||
meta = NixlAgentMetadata(
|
||||
engine_id="p-engine",
|
||||
agent_metadata=b"agent",
|
||||
kv_caches_base_addr=[10, 11, 12, 13], # full 4-layer model
|
||||
device_id=0,
|
||||
num_blocks=4,
|
||||
block_lens=[block_len] * 4,
|
||||
kv_cache_layout="HND",
|
||||
block_size=16,
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name="FLASH_ATTN",
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
|
||||
with pytest.raises(_StopAfterSlice):
|
||||
w.add_remote_agent(meta, remote_tp_rank=0, remote_tp_size=1)
|
||||
|
||||
# Sliced down to this worker's layer window (regions [2:4]).
|
||||
assert meta.kv_caches_base_addr == [12, 13]
|
||||
assert meta.block_lens == [block_len, block_len]
|
||||
|
||||
|
||||
class TestPushWriterMlaReplication:
|
||||
"""MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP
|
||||
(``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank
|
||||
it handshook -- not write one and merely notify the rest, which would
|
||||
leave the un-written ranks decoding against stale KV."""
|
||||
|
||||
@staticmethod
|
||||
def _mla_worker_writing_to(d_ranks):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
)
|
||||
|
||||
engine_id = "decode-engine"
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.use_mla = True
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.transfer_topo = MagicMock()
|
||||
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_tp_size=len(d_ranks),
|
||||
remote_block_size=16,
|
||||
remote_physical_blocks_per_logical=1,
|
||||
)
|
||||
# D_TP > P_TP => negative ratio (one P rank feeds |ratio| D ranks).
|
||||
w.transfer_topo.tp_ratio.return_value = -len(d_ranks)
|
||||
# The tp-mapping collapses MLA to a single source rank (correct for
|
||||
# the pull/read direction); the push path must fan it back out.
|
||||
w.tp_mappings = {
|
||||
engine_id: TPMapping(
|
||||
source_ranks_per_group=((0,),),
|
||||
all_source_ranks=(0,),
|
||||
rank_to_attention_slot={0: 0},
|
||||
rank_offset_factor=0,
|
||||
)
|
||||
}
|
||||
w._logical_to_remote_kernel_block_ids = lambda block_ids, ratio: block_ids
|
||||
w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}}
|
||||
w.src_xfer_handles_by_block_size = {16: 2000}
|
||||
w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}}
|
||||
return w, engine_id
|
||||
|
||||
def test_mla_hetero_tp_writes_every_d_rank(self):
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
)
|
||||
|
||||
w, engine_id = self._mla_worker_writing_to(d_ranks=(0, 1))
|
||||
|
||||
written: list[int] = []
|
||||
|
||||
def _fake_xfer(**kw):
|
||||
rank = kw["read_spec"].remote_rank
|
||||
written.append(rank)
|
||||
return 1000 + rank # in-flight handle, as the real method returns
|
||||
|
||||
w._xfer_blocks = _fake_xfer
|
||||
|
||||
meta = ReqMeta(
|
||||
local_block_ids=([100, 101],),
|
||||
local_physical_block_ids=([100, 101],),
|
||||
tp_size=1,
|
||||
remote=RemoteMeta(
|
||||
block_ids=([200, 201],),
|
||||
host="",
|
||||
port=0,
|
||||
engine_id=engine_id,
|
||||
request_id="d-req",
|
||||
),
|
||||
)
|
||||
|
||||
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
|
||||
|
||||
# Every handshook D rank must receive a real WRITE of the latent...
|
||||
assert sorted(written) == [0, 1]
|
||||
# ...and no rank may be fobbed off with a bare completion notif.
|
||||
assert w.nixl_wrapper.send_notif.call_count == 0
|
||||
# All of the request's WRITE handles must be tracked together, so the
|
||||
# engine thread never sees a partial set and double-frees the request.
|
||||
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]
|
||||
|
||||
@@ -185,6 +185,7 @@ class NixlBaseConnectorScheduler:
|
||||
host = params.get("remote_host")
|
||||
port = params.get("remote_port")
|
||||
tp_size = params.get("tp_size")
|
||||
pp_size = params.get("pp_size", 1)
|
||||
if (
|
||||
remote_engine_id is None
|
||||
or remote_request_id is None
|
||||
@@ -199,6 +200,7 @@ class NixlBaseConnectorScheduler:
|
||||
host=host,
|
||||
port=port,
|
||||
tp_size=tp_size,
|
||||
pp_size=pp_size,
|
||||
)
|
||||
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
|
||||
self._heartbeat_req_engine[request.request_id] = (
|
||||
@@ -244,7 +246,7 @@ class NixlBaseConnectorScheduler:
|
||||
)
|
||||
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[int, KVConnectorHandshakeMetadata]
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for this connector.
|
||||
@@ -252,19 +254,20 @@ class NixlBaseConnectorScheduler:
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
"""
|
||||
encoded_data: dict[int, bytes] = {}
|
||||
encoded_data: dict[tuple[int, int], bytes] = {}
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
for tp_rank, rank_metadata in metadata.items():
|
||||
for (pp_rank, tp_rank), rank_metadata in metadata.items():
|
||||
if not isinstance(rank_metadata, NixlHandshakePayload):
|
||||
raise ValueError(
|
||||
"NixlConnectorScheduler expects NixlHandshakePayload for "
|
||||
"handshake metadata."
|
||||
)
|
||||
encoded_data[tp_rank] = encoder.encode(rank_metadata)
|
||||
encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata)
|
||||
logger.debug(
|
||||
"Tp rank %d: encoded NixlHandshakePayload size: %s bytes",
|
||||
"PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes",
|
||||
pp_rank,
|
||||
tp_rank,
|
||||
str(len(encoded_data[tp_rank])),
|
||||
str(len(encoded_data[(pp_rank, tp_rank)])),
|
||||
)
|
||||
|
||||
# Only start the listener when we have metadata to serve.
|
||||
@@ -287,7 +290,7 @@ class NixlBaseConnectorScheduler:
|
||||
|
||||
@staticmethod
|
||||
def _nixl_handshake_listener(
|
||||
encoded_data: dict[int, Any],
|
||||
encoded_data: dict[tuple[int, int], Any],
|
||||
ready_event: threading.Event,
|
||||
stop_event: threading.Event,
|
||||
host: str,
|
||||
@@ -310,15 +313,18 @@ class NixlBaseConnectorScheduler:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
continue
|
||||
# Decode the message which contains (GET_META_MSG, rank)
|
||||
msg, target_tp_rank = msgspec.msgpack.decode(msg)
|
||||
# Decode (GET_META_MSG, pp_rank, tp_rank).
|
||||
msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg)
|
||||
logger.debug(
|
||||
"Received message for tp rank %s",
|
||||
"Received message for pp rank %s, tp rank %s",
|
||||
target_pp_rank,
|
||||
target_tp_rank,
|
||||
)
|
||||
if msg != GET_META_MSG:
|
||||
logger.warning("Connection listener got unexpected message %s", msg)
|
||||
sock.send_multipart((identity, b"", encoded_data[target_tp_rank]))
|
||||
sock.send_multipart(
|
||||
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)])
|
||||
)
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base worker-side logic for the NIXL connector."""
|
||||
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
@@ -336,8 +337,11 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config)
|
||||
# Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}.
|
||||
self._remote_agents: dict[EngineId, dict[int, str]] = defaultdict(dict)
|
||||
# Map of engine_id -> {(pp_rank, tp_rank): agent_name, ...}.
|
||||
# non-PP remote uses pp_rank 0, i.e. (0, tp_rank).
|
||||
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
|
||||
dict
|
||||
)
|
||||
|
||||
# Metadata.
|
||||
self.engine_id: EngineId = engine_id
|
||||
@@ -411,6 +415,25 @@ class NixlBaseConnectorWorker:
|
||||
# (so 1 per layer for MLA, otherwise 2 per layer)
|
||||
self.num_regions = 0
|
||||
|
||||
# PP>1 (push mode): this worker holds a contiguous layer slice and
|
||||
# transfers into the matching sub-range of a PP=1 remote's regions.
|
||||
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
self._remote_region_offset = 0
|
||||
# PP push slices regions per layer (uniform count); HMA breaks that.
|
||||
if self.pp_size > 1 and self._is_hma_required:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support pipeline_parallel_size > 1 "
|
||||
"with hybrid KV cache layouts (HMA) yet."
|
||||
)
|
||||
# Decode-side PP is unsupported (completions counted per consumer rank).
|
||||
if vllm_config.kv_transfer_config.kv_role == "kv_consumer" and self.pp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector consumer (decode) does not support "
|
||||
"pipeline_parallel_size > 1."
|
||||
)
|
||||
# Keep heartbeat handshakes to a PP-sharded producer notif-only.
|
||||
self._hb_handshake_notif_only = False
|
||||
|
||||
# nixl_prepped_dlist_handle.
|
||||
self.src_xfer_handles_by_block_size: dict[int, int] = {}
|
||||
# Populated dynamically during handshake based on remote configuration.
|
||||
@@ -449,7 +472,7 @@ class NixlBaseConnectorWorker:
|
||||
thread_name_prefix="vllm-nixl-handshake-initiator",
|
||||
)
|
||||
self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]()
|
||||
self._handshake_futures: dict[EngineId, Future[dict[int, str]]] = {}
|
||||
self._handshake_futures: dict[EngineId, Future[dict[tuple[int, int], str]]] = {}
|
||||
# Protects _handshake_futures and _remote_agents.
|
||||
self._handshake_lock = threading.RLock()
|
||||
|
||||
@@ -536,7 +559,9 @@ class NixlBaseConnectorWorker:
|
||||
port: int,
|
||||
remote_tp_size: int,
|
||||
expected_engine_id: str,
|
||||
) -> dict[int, str]:
|
||||
remote_pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> dict[tuple[int, int], str]:
|
||||
"""Do a NIXL handshake with a remote instance."""
|
||||
|
||||
# the first time we connect to a remote agent.
|
||||
@@ -558,20 +583,25 @@ class NixlBaseConnectorWorker:
|
||||
# this happens to be the same single rank_i.
|
||||
assert self.transfer_topo is not None
|
||||
p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size)
|
||||
remote_rank_to_agent_name = {}
|
||||
remote_rank_to_agent_name: dict[tuple[int, int], str] = {}
|
||||
path = make_zmq_path("tcp", host, port)
|
||||
|
||||
with zmq_ctx(zmq.REQ, path) as sock:
|
||||
for remote_rank in p_remote_ranks:
|
||||
for remote_pp_rank, remote_rank in itertools.product(
|
||||
range(remote_pp_size), p_remote_ranks
|
||||
):
|
||||
logger.debug(
|
||||
"Querying metadata on path: %s at remote tp rank %s",
|
||||
"Querying metadata on path: %s at remote pp rank %s, tp rank %s",
|
||||
path,
|
||||
remote_pp_rank,
|
||||
remote_rank,
|
||||
)
|
||||
|
||||
start_time = time.perf_counter()
|
||||
# Send query for the request.
|
||||
msg = msgspec.msgpack.encode((GET_META_MSG, remote_rank))
|
||||
msg = msgspec.msgpack.encode(
|
||||
(GET_META_MSG, remote_pp_rank, remote_rank)
|
||||
)
|
||||
# Set receive timeout to 5 seconds to avoid hanging on dead server
|
||||
sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds
|
||||
sock.send(msg)
|
||||
@@ -638,17 +668,44 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
# Register Remote agent.
|
||||
remote_agent_name = self.add_remote_agent(
|
||||
metadata, remote_rank, remote_tp_size
|
||||
)
|
||||
if notif_agents_only:
|
||||
remote_agent_name = self._add_notif_only_remote_agent(
|
||||
metadata, remote_tp_size
|
||||
)
|
||||
else:
|
||||
remote_agent_name = self.add_remote_agent(
|
||||
metadata, remote_rank, remote_tp_size
|
||||
)
|
||||
setup_agent_time = time.perf_counter()
|
||||
logger.debug(
|
||||
"NIXL handshake: add agent took: %s",
|
||||
setup_agent_time - got_metadata_time,
|
||||
)
|
||||
remote_rank_to_agent_name[remote_rank] = remote_agent_name
|
||||
remote_ranks = (remote_pp_rank, remote_rank)
|
||||
remote_rank_to_agent_name[remote_ranks] = remote_agent_name
|
||||
return remote_rank_to_agent_name
|
||||
|
||||
def _add_notif_only_remote_agent(
|
||||
self, metadata: NixlAgentMetadata, remote_tp_size: int
|
||||
) -> str:
|
||||
"""Load a remote agent for notifs only on the push-mode decode side.
|
||||
|
||||
Skips descriptor setup but records engine info for block accounting.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
self.transfer_topo.register_remote_engine(
|
||||
metadata.engine_id,
|
||||
EngineTransferInfo(
|
||||
remote_tp_size=remote_tp_size,
|
||||
remote_block_size=metadata.block_size,
|
||||
remote_block_len=metadata.block_lens[0],
|
||||
remote_physical_blocks_per_logical=(
|
||||
metadata.physical_blocks_per_logical_kv_block
|
||||
),
|
||||
),
|
||||
)
|
||||
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
|
||||
|
||||
def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""
|
||||
Initialize transfer buffer in CPU mem for accelerators
|
||||
@@ -761,7 +818,9 @@ class NixlBaseConnectorWorker:
|
||||
host: str,
|
||||
port: int,
|
||||
tp_size: int,
|
||||
) -> Future[dict[int, str]] | None:
|
||||
pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> Future[dict[tuple[int, int], str]] | None:
|
||||
"""
|
||||
Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
|
||||
@@ -784,10 +843,12 @@ class NixlBaseConnectorWorker:
|
||||
port,
|
||||
tp_size,
|
||||
engine_id,
|
||||
pp_size,
|
||||
notif_agents_only,
|
||||
)
|
||||
self._handshake_futures[engine_id] = fut
|
||||
|
||||
def done_callback(f: Future[dict[int, str]], eid=engine_id):
|
||||
def done_callback(f: Future[dict[tuple[int, int], str]], eid=engine_id):
|
||||
with self._handshake_lock:
|
||||
del self._handshake_futures[eid]
|
||||
try:
|
||||
@@ -1114,6 +1175,15 @@ class NixlBaseConnectorWorker:
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
|
||||
self.num_regions = len(caches_data)
|
||||
|
||||
if self.pp_size > 1:
|
||||
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
|
||||
self.vllm_config.parallel_config
|
||||
)
|
||||
num_local_layers = end_layer - start_layer
|
||||
assert num_local_layers > 0 and self.num_regions % num_local_layers == 0
|
||||
regions_per_layer = self.num_regions // num_local_layers
|
||||
self._remote_region_offset = regions_per_layer * start_layer
|
||||
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
# NOTE (NickLucche) When FlashInfer is used, memory is registered
|
||||
# with joint KV for each block. This minimizes the overhead in
|
||||
@@ -1464,14 +1534,32 @@ class NixlBaseConnectorWorker:
|
||||
""" # noqa: E501
|
||||
engine_id = nixl_agent_meta.engine_id
|
||||
# TODO re-evaluate refreshing for scaling/recovery
|
||||
if remote_tp_rank in self._remote_agents.get(engine_id, {}):
|
||||
if (0, remote_tp_rank) in self._remote_agents.get(engine_id, {}):
|
||||
logger.debug(
|
||||
"Remote agent with engine_id %s and rank"
|
||||
"%s already exchanged metadata, skip handshake.",
|
||||
engine_id,
|
||||
remote_tp_rank,
|
||||
)
|
||||
return self._remote_agents[engine_id][remote_tp_rank]
|
||||
return self._remote_agents[engine_id][(0, remote_tp_rank)]
|
||||
|
||||
# Compare physical regions, not self.num_regions (doubled by
|
||||
# FlashInfer's virtual K/V split).
|
||||
num_local_regions = len(self.block_len_per_layer)
|
||||
if (
|
||||
self.pp_size > 1
|
||||
and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions
|
||||
):
|
||||
# This worker holds a PP layer-slice; the PP=1 remote registered
|
||||
# the full model. Slice its regions to our layer window so the
|
||||
# logic below sees congruent local/remote lists.
|
||||
start = self._remote_region_offset
|
||||
end = start + num_local_regions
|
||||
assert len(nixl_agent_meta.kv_caches_base_addr) >= end
|
||||
nixl_agent_meta.kv_caches_base_addr = nixl_agent_meta.kv_caches_base_addr[
|
||||
start:end
|
||||
]
|
||||
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
|
||||
|
||||
### Register remote engine in TransferTopology (idempotent).
|
||||
assert self.transfer_topo is not None
|
||||
@@ -2098,7 +2186,12 @@ class NixlBaseConnectorWorker:
|
||||
# the **next** heartbeat for this remote can go through.
|
||||
if (
|
||||
self._ensure_handshake(
|
||||
engine_id, hb_info.host, hb_info.port, hb_info.tp_size
|
||||
engine_id,
|
||||
hb_info.host,
|
||||
hb_info.port,
|
||||
hb_info.tp_size,
|
||||
hb_info.pp_size,
|
||||
self._hb_handshake_notif_only and hb_info.pp_size > 1,
|
||||
)
|
||||
is not None
|
||||
):
|
||||
@@ -2374,19 +2467,23 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
assert engine_id in self._remote_agents
|
||||
|
||||
for handle in self.dst_xfer_side_handles.pop(engine_id).values():
|
||||
# Notif-only engines (push-mode D side) have no descriptor state.
|
||||
for handle in self.dst_xfer_side_handles.pop(engine_id, {}).values():
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
for agent_name in self._remote_agents.pop(engine_id).values():
|
||||
self.nixl_wrapper.remove_remote_agent(agent_name)
|
||||
|
||||
del self.kv_caches_base_addr[engine_id]
|
||||
del self.dst_num_blocks[engine_id]
|
||||
del self.tp_mappings[engine_id]
|
||||
self.kv_caches_base_addr.pop(engine_id, None)
|
||||
self.dst_num_blocks.pop(engine_id, None)
|
||||
self.tp_mappings.pop(engine_id, None)
|
||||
if self.transfer_topo is not None:
|
||||
self.transfer_topo.unregister_remote_engine(engine_id)
|
||||
|
||||
last_active = self._engine_last_active.pop(engine_id)
|
||||
if log_eviction:
|
||||
# Push P-side engines are tracked in _remote_agents but not in
|
||||
# _engine_last_active (they don't participate in stale eviction), so
|
||||
# tolerate a missing entry.
|
||||
last_active = self._engine_last_active.pop(engine_id, None)
|
||||
if log_eviction and last_active is not None:
|
||||
logger.info(
|
||||
"Evicted stale remote engine %s (inactive for %.1fs).",
|
||||
engine_id,
|
||||
|
||||
@@ -207,11 +207,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[int, KVConnectorHandshakeMetadata]
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for this connector.
|
||||
Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
channel can serve every PP stage's agent metadata.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
@@ -147,6 +147,7 @@ class HeartbeatInfo:
|
||||
host: str
|
||||
port: int
|
||||
tp_size: int
|
||||
pp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -167,6 +168,8 @@ class ReqMeta:
|
||||
remote: RemoteMeta | None = None
|
||||
# Remote block size, discovered during NIXL handshake (push mode).
|
||||
remote_block_size: int | None = None
|
||||
# Remote producer pipeline-parallel size (push mode, D side).
|
||||
pp_size: int = 1
|
||||
|
||||
|
||||
class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
@@ -196,6 +199,7 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
# P workers don't need to receive tp_size from proxy here.
|
||||
tp_size=kv_transfer_params.get("tp_size", 1),
|
||||
remote_block_size=kv_transfer_params.get("remote_block_size"),
|
||||
pp_size=kv_transfer_params.get("pp_size", 1),
|
||||
)
|
||||
|
||||
def add_new_req_to_save(
|
||||
|
||||
@@ -183,7 +183,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
notif_id = f"{meta.remote.request_id}:{self.world_size}".encode()
|
||||
remote_agents = self._remote_agents[meta.remote.engine_id]
|
||||
for rank_to_notify, agent in remote_agents.items():
|
||||
if rank_to_notify != read_specs[0].remote_rank:
|
||||
if rank_to_notify != (0, read_specs[0].remote_rank):
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
|
||||
def _read_blocks(
|
||||
@@ -250,7 +250,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
# just notify P worker that we have the blocks we need.
|
||||
if len(local_block_ids) == 0:
|
||||
# A full prefix cache hit is indicated with an empty list.
|
||||
agent_name = self._remote_agents[dst_engine_id][remote_rank]
|
||||
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
|
||||
@@ -186,6 +186,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
"remote_host": params["remote_host"],
|
||||
"remote_port": params["remote_port"],
|
||||
"remote_tp_size": params["tp_size"],
|
||||
"remote_pp_size": params.get("pp_size", 1),
|
||||
}
|
||||
self._push_registration_deadlines[request.request_id] = (
|
||||
time.perf_counter() + self._push_registration_timeout
|
||||
@@ -238,6 +239,8 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
# serving layer via abort_immediately. To keep P from
|
||||
# stranding the prefill blocks, we still register an empty
|
||||
# recv so the worker emits a notif that lets P free them.
|
||||
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, [])
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
@@ -285,6 +288,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
remote_host=self.side_channel_host,
|
||||
remote_port=self.side_channel_port,
|
||||
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
|
||||
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
|
||||
remote_num_tokens=remote_num_tokens,
|
||||
)
|
||||
|
||||
|
||||
@@ -80,6 +80,10 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
# Heartbeat handshakes to a PP-sharded producer must be notif-only,
|
||||
# like the PUSH_REG path.
|
||||
self._hb_handshake_notif_only = True
|
||||
|
||||
# Push-specific state.
|
||||
# P-side: outgoing WRITE handles awaiting completion, keyed by
|
||||
# request_id. Mutated by writer (submit) and main thread
|
||||
@@ -275,18 +279,22 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
and the request is re-queued onto ``_reg_send_inbox`` once it
|
||||
completes (at which point ``_ensure_handshake`` returns ``None`` and we
|
||||
send directly)."""
|
||||
remote_pp_size = reg_data.get("remote_pp_size", 1)
|
||||
fut = self._ensure_handshake(
|
||||
reg_data["remote_engine_id"],
|
||||
reg_data["remote_host"],
|
||||
reg_data["remote_port"],
|
||||
reg_data["remote_tp_size"],
|
||||
pp_size=remote_pp_size,
|
||||
# D never addresses P memory in push mode; just load P's agents.
|
||||
notif_agents_only=remote_pp_size > 1,
|
||||
)
|
||||
if fut is None:
|
||||
self._do_send_reg_notif(req_id, reg_data)
|
||||
return
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[dict[int, str]],
|
||||
f: Future[dict[tuple[int, int], str]],
|
||||
rid: str = req_id,
|
||||
rd: dict[str, Any] = reg_data,
|
||||
) -> None:
|
||||
@@ -500,28 +508,43 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_block_ids = meta.remote.block_ids
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
num_groups = len(local_block_ids)
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=[
|
||||
list(local_block_ids[g])
|
||||
if rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
],
|
||||
remote_block_ids=[
|
||||
list(remote_block_ids[g])
|
||||
if rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
],
|
||||
)
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
|
||||
if self.use_mla and tp_ratio < 0:
|
||||
assert len(read_specs) == 1
|
||||
# MLA latent is replicated across D's TP ranks: the tp-mapping
|
||||
# collapses to one rank (fine for reads), but push must WRITE every
|
||||
# D rank or the rest decode stale KV; only the dst differs per rank.
|
||||
assert len(plan.all_source_ranks) == 1
|
||||
mla_local_ids = [list(ids) for ids in local_block_ids]
|
||||
mla_remote_ids = [list(ids) for ids in remote_block_ids]
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=mla_local_ids,
|
||||
remote_block_ids=mla_remote_ids,
|
||||
)
|
||||
for rank in self.dst_xfer_side_handles[engine_id]
|
||||
]
|
||||
else:
|
||||
read_specs = [
|
||||
ReadSpec(
|
||||
remote_rank=rank,
|
||||
local_block_ids=[
|
||||
list(local_block_ids[g])
|
||||
if rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
],
|
||||
remote_block_ids=[
|
||||
list(remote_block_ids[g])
|
||||
if rank in plan.source_ranks_per_group[g]
|
||||
else []
|
||||
for g in range(num_groups)
|
||||
],
|
||||
)
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
|
||||
handles: list[int] = []
|
||||
for i, spec in enumerate(read_specs):
|
||||
remote_block_size = remote_info.remote_block_size
|
||||
logger.debug(
|
||||
@@ -544,7 +567,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
spec.remote_rank
|
||||
]
|
||||
|
||||
self._xfer_blocks(
|
||||
handle = self._xfer_blocks(
|
||||
read_spec=spec,
|
||||
request_id=req_id,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
@@ -552,13 +575,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_xfer_side_handle=local_xfer_side_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
)
|
||||
if handle is not None:
|
||||
handles.append(handle)
|
||||
|
||||
if self.use_mla and tp_ratio < 0 and read_specs:
|
||||
notif_id = f"{meta.remote.request_id}:{self.world_size}".encode()
|
||||
remote_agents = self._remote_agents[meta.remote.engine_id]
|
||||
for rank_to_notify, agent in remote_agents.items():
|
||||
if rank_to_notify != read_specs[0].remote_rank:
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
# Publish all the request's WRITE handles in one locked update: a
|
||||
# partial set would let ``_pop_done_transfers`` finish the request
|
||||
# early, then double-report it as the remaining writes land.
|
||||
if handles:
|
||||
with self._sending_transfers_lock:
|
||||
self._sending_transfers[req_id].extend(handles)
|
||||
|
||||
def _xfer_blocks(
|
||||
self,
|
||||
@@ -568,8 +593,12 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_request_id: str,
|
||||
local_xfer_side_handle: int,
|
||||
remote_xfer_side_handle: int,
|
||||
):
|
||||
"""Post a WRITE point-to-point xfer request."""
|
||||
) -> int | None:
|
||||
"""Post a WRITE point-to-point xfer request.
|
||||
|
||||
Returns the in-flight transfer handle (so the caller can track all of
|
||||
a request's handles atomically), or ``None`` if nothing was submitted.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
remote_rank = read_spec.remote_rank
|
||||
local_block_ids = read_spec.local_block_ids
|
||||
@@ -597,7 +626,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
if len(local_block_ids) == 0:
|
||||
logger.warning("No blocks to push for request %s", request_id)
|
||||
return
|
||||
return None
|
||||
|
||||
# Align per-group block counts for push.
|
||||
local_block_ids = list(local_block_ids)
|
||||
@@ -637,9 +666,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
notif_msg=notif_id,
|
||||
)
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
# Track push WRITE handles so P can free blocks once done.
|
||||
with self._sending_transfers_lock:
|
||||
self._sending_transfers[request_id].append(handle)
|
||||
# Caller tracks the handle (atomically with the request's other
|
||||
# writes) so P can free blocks once all of them are done.
|
||||
return handle
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_setup_failed",
|
||||
@@ -656,6 +685,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
return None
|
||||
|
||||
# --- Notification handling on engine main thread ------------------ #
|
||||
|
||||
@@ -683,12 +713,20 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
# Not tracked as a P-side send/process for this notif.
|
||||
if req_id not in self._reqs_to_send and req_id not in self._reqs_to_process:
|
||||
if req_id in self._recving_metadata:
|
||||
# D-side: P signalled push completion. The transfer was
|
||||
# driven entirely by P (we don't own a NIXL handle here),
|
||||
# so materialise an empty entry in ``_recving_transfers``
|
||||
# and let ``_pop_done_transfers`` report it done on the
|
||||
# next ``get_finished``.
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None:
|
||||
# Consumer waits for one notif per producer rank writing
|
||||
# here: pp_size stages * producers-per-consumer (>1 when
|
||||
# producer TP > consumer TP; tp_size is the producer TP).
|
||||
producers_per_consumer = max(1, int(tp_size) // self.world_size)
|
||||
expected_notifs = meta.pp_size * producers_per_consumer
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
notifs = self.consumer_notification_counts_by_req[req_id]
|
||||
if notifs < expected_notifs:
|
||||
continue
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
# P drove the transfer (we own no NIXL handle), so
|
||||
# materialise an empty ``_recving_transfers`` entry for
|
||||
# ``_pop_done_transfers`` to report done.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
else:
|
||||
# Not tracked on either side (lease may have expired
|
||||
|
||||
Reference in New Issue
Block a user