[KVConnector][NIXL] Support pipeline-parallel prefill in push mode (#45880)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-07-08 16:49:23 -07:00
committed by GitHub
co-authored by Nick Hill
parent 95d6d6f4bb
commit 7802c20c4e
15 changed files with 506 additions and 104 deletions
@@ -40,7 +40,6 @@ sw_attn_configs=(
# Gemma4: SW + cross-layer KV sharing
"GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-4-E2B-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
)
# Select config array based on DP_EP env var
if [[ -n "${DP_EP:-}" ]]; then
configs=("${dp_ep_configs[@]}")
@@ -15,6 +15,7 @@ def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--host", required=True)
parser.add_argument("--port", required=True, type=int)
parser.add_argument("--pp-rank", default=0, type=int)
parser.add_argument("--rank", default=0, type=int)
parser.add_argument("--timeout-ms", default=1000, type=int)
return parser.parse_args()
@@ -37,7 +38,7 @@ def main() -> None:
sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms)
try:
sock.connect(make_zmq_path(args.host, args.port))
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank)))
sock.send(msgspec.msgpack.encode((GET_META_MSG, args.pp_rank, args.rank)))
sock.recv()
finally:
sock.close()
@@ -49,13 +49,16 @@ else
KV_EXTRA_CONFIG=''
fi
# Connector: default pull NixlConnector; NixlPushConnector enables PP prefill.
KV_CONNECTOR=${KV_CONNECTOR:-NixlConnector}
# Build the kv-transfer-config for P and D
if [[ "$KV_BUFFER_DEVICE" == "cuda" ]]; then
KV_CONFIG_P='{"kv_connector":"NixlConnector","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
KV_CONFIG_D='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
KV_CONFIG_P='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
KV_CONFIG_D='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}'
else
KV_CONFIG_P="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
KV_CONFIG_D="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
KV_CONFIG_P="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
KV_CONFIG_D="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}"
fi
# Models to run
@@ -72,6 +75,7 @@ fi
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
PREFILLER_PP_SIZE=${PREFILLER_PP_SIZE:-1} # >1 requires NixlPushConnector
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
@@ -138,8 +142,9 @@ run_tests_for_model() {
# Calculate GPU ID - we'll distribute across available GPUs
GPU_ID=$((i % $(get_num_gpus)))
NEXT_GPU=${GPU_ID}
# If PREFILLER_TP_SIZE is more than 1
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
# Reserve TP*PP GPUs for the prefiller (TP shards across PP stages).
PREFILLER_WORLD_SIZE=$((PREFILLER_TP_SIZE * PREFILLER_PP_SIZE))
for (( j=1; j < PREFILLER_WORLD_SIZE; j++ )); do
NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus)))
GPU_ID="${GPU_ID},${NEXT_GPU}"
done
@@ -161,6 +166,7 @@ run_tests_for_model() {
--block-size ${PREFILL_BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $PREFILLER_TP_SIZE \
--pipeline-parallel-size $PREFILLER_PP_SIZE \
--kv-transfer-config '$KV_CONFIG_P'"
if [[ "$ENFORCE_EAGER" == "1" ]]; then
BASE_CMD="${BASE_CMD} --enforce-eager"
@@ -0,0 +1,41 @@
#!/usr/bin/env bash
set -euo pipefail
# NixlPushConnector (push-mode PD) integration tests. Kept separate from the
# pull-mode sweep in ../nixl_integration so push coverage can grow on its own;
# the per-config runner is shared.
SCRIPT="v1/kv_connector/nixl_integration/run_accuracy_test.sh"
IMPORT_CANARY="v1/kv_connector/nixl_integration/test_nixl_imports.py"
echo "=== Running NIXL import canary ==="
python3 -m pytest -s -x "${IMPORT_CANARY}"
# PP=2 prefill -> TP=2 decode: each PP stage writes its layer slice into the
# decoder's matching region range (non-MLA and MLA contiguous-slice paths).
configs=(
"KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.6 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2" # non-MLA, Qwen3-0.6B
"KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.8 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA
)
run_tests() {
local label=$1
local extra_args=$2
echo "=== Running push tests (${label}) ==="
for cfg in "${configs[@]}"; do
local -a cfg_parts extra_args_parts
read -r -a cfg_parts <<< "$cfg"
read -r -a extra_args_parts <<< "$extra_args"
echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}"
# Use 'env' to safely set variables without eval; keep argv splitting
# safe and SC2086-clean via arrays.
if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then
echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}"
exit 1
fi
done
echo "✅ All ${label} push tests passed!"
}
run_tests "default backend" ""
@@ -408,11 +408,10 @@ def test_kv_transfer_handshake(dist_init):
decoder = msgspec.msgpack.Decoder(NixlAgentMetadata)
expected_agent_metadata = decoder.decode(metadata.agent_metadata_bytes)
# The scheduler connector expects metadata to be in
# dict[int, KVConnectorHandshakeMetadata], where the first key is
# the dp_rank, the second key is the tp_rank.
# The scheduler connector expects metadata keyed by
# (pp_rank, tp_rank).
scheduler_connector = scheduler.get_kv_connector()
scheduler_connector.set_xfer_handshake_metadata({0: metadata})
scheduler_connector.set_xfer_handshake_metadata_pp_aware({(0, 0): metadata})
# Simulate a request that finishes prefill, which returns
# corresponding NixlConnectorMetadata for decode instance.
@@ -502,8 +501,14 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
)
def _nixl_handshake(
self, host: str, port: int, remote_tp_size: int, expected_engine_id: str
) -> dict[int, str]:
self,
host: str,
port: int,
remote_tp_size: int,
expected_engine_id: str,
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> dict[tuple[int, int], str]:
# Mimic slow _nixl_handshake, as well as bypass zmq communication.
time.sleep(self._hand_shake_latency)
# These should've been done in register_kv_caches(), called by
@@ -533,7 +538,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
# When remote tp_size > local tp_size, handshake with multiple
# remote ranks.
num_handshakes = 1 if tp_ratio > 0 else -tp_ratio
remote_agents: dict[int, str] = {}
remote_agents: dict[tuple[int, int], str] = {}
for remote_tp_rank in range(num_handshakes):
remote_agent_name = self.add_remote_agent(
NixlAgentMetadata(
@@ -554,7 +559,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
remote_tp_rank=remote_tp_rank,
remote_tp_size=remote_tp_size,
)
remote_agents[remote_tp_rank] = remote_agent_name
remote_agents[(0, remote_tp_rank)] = remote_agent_name
return remote_agents
@@ -749,7 +754,7 @@ class TestNixlHandshake:
def check_handshake(remote_tp_size: int):
tp_ratio = remote_tp_size // local_tp_size
assert set(remote_agents.keys()) == set(range(tp_ratio))
assert set(remote_agents.keys()) == {(0, r) for r in range(tp_ratio)}
remote_engine_id = worker.REMOTE_ENGINE_ID
remote_info = worker.transfer_topo.get_engine_info(remote_engine_id)
@@ -2044,7 +2049,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init):
# P TP = 2 * D TP case, we should register 2 local handles
worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]}
worker.dst_xfer_side_handles = {"engine1": {0: 789}}
worker._remote_agents = {"engine1": {0: "agent1"}}
worker._remote_agents = {"engine1": {(0, 0): "agent1"}}
# _cleanup_remote_engine (called by shutdown) also clears these:
worker.kv_caches_base_addr["engine1"] = {0: [0xABC]}
worker.dst_num_blocks["engine1"] = 50
@@ -2097,7 +2102,7 @@ def _setup_worker_with_remote_engine(
)
engine_id = "remote-engine-1"
worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"}
worker._remote_agents[engine_id] = {(0, 0): "agent_0", (0, 1): "agent_1"}
worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200}
worker.kv_caches_base_addr[engine_id] = {0: [0xABC]}
worker.dst_num_blocks[engine_id] = 50
@@ -2993,7 +2998,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
local_block_len=worker.block_size * 4096,
)
worker._remote_agents[remote_engine_id] = {
rank: f"agent_p{rank}" for rank in range(prefill_tp_size)
(0, rank): f"agent_p{rank}" for rank in range(prefill_tp_size)
}
worker.dst_xfer_side_handles = {
remote_engine_id: {rank: 100 + rank for rank in range(prefill_tp_size)}
@@ -3044,7 +3049,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
# Broadcast goes to ranks {1, 2, 3} only, never to the read target.
expected_recipients = {
worker._remote_agents[remote_engine_id][r]
worker._remote_agents[remote_engine_id][(0, r)]
for r in range(1, prefill_tp_size)
}
assert {agent for agent, _ in send_notif_calls} == expected_recipients
@@ -29,9 +29,11 @@ from typing import Any
from unittest.mock import MagicMock, patch
import msgspec
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
PUSH_REG_NOTIF_PREFIX,
NixlAgentMetadata,
NixlConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
@@ -512,7 +514,7 @@ class TestPushWriterNotifs:
# Pretend the writer thread already forwarded a completion notif
# for a request whose KV is being received.
request_id = "req-recv-1"
w._recving_metadata[request_id] = MagicMock()
w._recving_metadata[request_id] = MagicMock(pp_size=1)
# Compose the standard completion notif: req_id:tp_size.
notif_msg = f"{request_id}:1".encode()
w._pending_completion_notifs.put(notif_msg)
@@ -813,3 +815,177 @@ class TestPushWriterNegative:
assert w._reqs_to_send[rid] >= now
# Unknown request must not be inserted by the heartbeat path.
assert "req-unknown" not in w._reqs_to_send
# ----------------------------------------------------------------- #
# Pipeline-parallel producer (push-mode PP-disagg) #
# ----------------------------------------------------------------- #
class TestPushPipelineParallel:
"""PP-sharded producer: per-stage completion counting, pp_size plumbing,
and remote-region slicing."""
def test_completion_waits_for_one_notif_per_pp_stage(self):
"""Each PP stage WRITEs its own layers and sends one notif; D must
collect pp_size notifs before reporting the recv done."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
request_id = "req-pp-2"
w._recving_metadata[request_id] = MagicMock(pp_size=2)
notif = f"{request_id}:1".encode()
# First stage: counted, not yet done.
w._pending_completion_notifs.put(notif)
assert w._get_new_notifs() == set()
assert request_id not in w._recving_transfers
assert w.consumer_notification_counts_by_req[request_id] == 1
# Second (final) stage: now reported done.
w._pending_completion_notifs.put(notif)
assert w._get_new_notifs() == set()
assert request_id in w._recving_transfers
assert request_id not in w.consumer_notification_counts_by_req
def test_req_meta_reads_pp_size_from_kv_transfer_params(self):
"""D learns the producer's pp_size from kv_transfer_params (forwarded
by the proxy) and defaults to 1 when absent."""
metadata = NixlConnectorMetadata()
params = {
"remote_block_ids": ([0],),
"remote_engine_id": "p-engine",
"remote_request_id": "p-req",
"remote_host": "localhost",
"remote_port": 1234,
"tp_size": 1,
"pp_size": 2,
}
metadata.add_new_req_to_recv("req", ([0],), params)
assert metadata.reqs_to_recv["req"].pp_size == 2
params.pop("pp_size")
metadata.add_new_req_to_recv("req-default", ([0],), params)
assert metadata.reqs_to_recv["req-default"].pp_size == 1
def test_add_remote_agent_slices_remote_regions_to_local_pp_window(self):
"""With PP>1 the producer registered regions for the full model, but
this worker holds only a contiguous layer slice; add_remote_agent
trims the remote region list to [offset : offset + num_local_regions]
before building descriptors. We stop right after the slice via a
sentinel on the next collaborator call."""
block_len = 4096 * 16
w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown
w.pp_size = 2
w._remote_region_offset = 2 # this worker owns layers [2, 4)
w.block_len_per_layer = [block_len, block_len] # 2 local layers
w.nixl_wrapper = MagicMock()
class _StopAfterSlice(RuntimeError):
pass
w.transfer_topo = MagicMock()
w.transfer_topo.register_remote_engine.side_effect = _StopAfterSlice()
meta = NixlAgentMetadata(
engine_id="p-engine",
agent_metadata=b"agent",
kv_caches_base_addr=[10, 11, 12, 13], # full 4-layer model
device_id=0,
num_blocks=4,
block_lens=[block_len] * 4,
kv_cache_layout="HND",
block_size=16,
ssm_sizes=(0, 0),
attn_backend_name="FLASH_ATTN",
physical_blocks_per_logical_kv_block=1,
)
with pytest.raises(_StopAfterSlice):
w.add_remote_agent(meta, remote_tp_rank=0, remote_tp_size=1)
# Sliced down to this worker's layer window (regions [2:4]).
assert meta.kv_caches_base_addr == [12, 13]
assert meta.block_lens == [block_len, block_len]
class TestPushWriterMlaReplication:
"""MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP
(``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank
it handshook -- not write one and merely notify the rest, which would
leave the un-written ranks decoding against stale KV."""
@staticmethod
def _mla_worker_writing_to(d_ranks):
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
engine_id = "decode-engine"
w = _StubWriterWorker.fresh()
w.use_mla = True
w.nixl_wrapper = MagicMock()
w.transfer_topo = MagicMock()
w.transfer_topo.get_engine_info.return_value = SimpleNamespace(
remote_tp_size=len(d_ranks),
remote_block_size=16,
remote_physical_blocks_per_logical=1,
)
# D_TP > P_TP => negative ratio (one P rank feeds |ratio| D ranks).
w.transfer_topo.tp_ratio.return_value = -len(d_ranks)
# The tp-mapping collapses MLA to a single source rank (correct for
# the pull/read direction); the push path must fan it back out.
w.tp_mappings = {
engine_id: TPMapping(
source_ranks_per_group=((0,),),
all_source_ranks=(0,),
rank_to_attention_slot={0: 0},
rank_offset_factor=0,
)
}
w._logical_to_remote_kernel_block_ids = lambda block_ids, ratio: block_ids
w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}}
w.src_xfer_handles_by_block_size = {16: 2000}
w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}}
return w, engine_id
def test_mla_hetero_tp_writes_every_d_rank(self):
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
RemoteMeta,
ReqMeta,
)
w, engine_id = self._mla_worker_writing_to(d_ranks=(0, 1))
written: list[int] = []
def _fake_xfer(**kw):
rank = kw["read_spec"].remote_rank
written.append(rank)
return 1000 + rank # in-flight handle, as the real method returns
w._xfer_blocks = _fake_xfer
meta = ReqMeta(
local_block_ids=([100, 101],),
local_physical_block_ids=([100, 101],),
tp_size=1,
remote=RemoteMeta(
block_ids=([200, 201],),
host="",
port=0,
engine_id=engine_id,
request_id="d-req",
),
)
w._xfer_blocks_for_req(req_id="p-req", meta=meta)
# Every handshook D rank must receive a real WRITE of the latent...
assert sorted(written) == [0, 1]
# ...and no rank may be fobbed off with a bare completion notif.
assert w.nixl_wrapper.send_notif.call_count == 0
# All of the request's WRITE handles must be tracked together, so the
# engine thread never sees a partial set and double-frees the request.
assert sorted(w._sending_transfers["p-req"]) == [1000, 1001]