diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 598558939c..4cab698f32 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -39,6 +39,19 @@ steps: - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +- label: Push NixlConnector PP prefill PD accuracy (4 GPUs) + key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - tests/v1/kv_connector/nixl_integration/ + - tests/v1/kv_connector/nixl_push_integration/ + commands: + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh + - label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus timeout_in_minutes: 30 diff --git a/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py index 9f1a0a7f41..672162f8ba 100644 --- a/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py @@ -9,9 +9,9 @@ the same; the difference is in how P and D coordinate the KV transfer: * Pull mode: proxy forwards P's ``kv_transfer_params`` (including ``remote_block_ids``) to D, and D pulls KV from P via NIXL READ. * Push mode: proxy hands D **only** P's coordinates - (``remote_engine_id``, ``remote_host``, ``remote_port``, ``tp_size``) - and the shared ``remote_request_id``. D registers its locally allocated - blocks with P over a NIXL notification; P then pushes the KV to D via + (``remote_engine_id``, ``remote_host``, ``remote_port``, ``tp_size``, + ``pp_size``) and the shared ``remote_request_id``. D registers its locally + allocated blocks with P over a NIXL notification; P then pushes the KV to D via NIXL WRITE. Launch multiple vLLM instances configured with ``NixlPushConnector`` and @@ -26,6 +26,7 @@ disagg_proxy_pushconnector_demo.py \ --prefill-kv-host 10.0.0.1 \ --prefill-side-channel-port 5600 \ --prefill-tp-size 1 \ + --prefill-pp-size 1 \ --port 8000 """ @@ -89,6 +90,7 @@ class PushProxy: prefill_kv_host: str, prefill_side_channel_port: int, prefill_tp_size: int, + prefill_pp_size: int, custom_create_completion: Callable[[Request], StreamingResponse] | None = None, custom_create_chat_completion: Callable[[Request], StreamingResponse] | None = None, @@ -110,6 +112,7 @@ class PushProxy: "remote_host": prefill_kv_host, "remote_port": prefill_side_channel_port, "tp_size": prefill_tp_size, + "pp_size": prefill_pp_size, } self.custom_create_completion = custom_create_completion @@ -198,6 +201,7 @@ class PushProxy: "prefill_kv_host": self.push_metadata["remote_host"], "prefill_side_channel_port": self.push_metadata["remote_port"], "prefill_tp_size": self.push_metadata["tp_size"], + "prefill_pp_size": self.push_metadata["pp_size"], } # ── push-mode request handling ──────────────────────────────────── # @@ -265,7 +269,7 @@ class PushProxy: generator = self.forward_request( f"http://{decode_instance}{path}", decode_request, headers ) - return StreamingResponse(generator) + return StreamingResponse(generator, media_type="application/json") async def create_completion(self, raw_request: Request): try: @@ -316,6 +320,7 @@ class PushProxyServer: prefill_kv_host=args.prefill_kv_host, prefill_side_channel_port=args.prefill_side_channel_port, prefill_tp_size=args.prefill_tp_size, + prefill_pp_size=args.prefill_pp_size, custom_create_completion=create_completion, custom_create_chat_completion=create_chat_completion, ) @@ -420,6 +425,12 @@ def parse_args(): default=1, help="Tensor parallel size of the prefill vLLM instance", ) + parser.add_argument( + "--prefill-pp-size", + type=int, + default=1, + help="Pipeline parallel size of the prefill vLLM instance", + ) return parser.parse_args() diff --git a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh index 9732f46737..acc7ce312f 100755 --- a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh @@ -40,7 +40,6 @@ sw_attn_configs=( # Gemma4: SW + cross-layer KV sharing "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-4-E2B-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" ) - # Select config array based on DP_EP env var if [[ -n "${DP_EP:-}" ]]; then configs=("${dp_ep_configs[@]}") diff --git a/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py b/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py index 24ecbd795e..36b7deef04 100644 --- a/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py +++ b/tests/v1/kv_connector/nixl_integration/nixl_side_channel_probe.py @@ -15,6 +15,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--host", required=True) parser.add_argument("--port", required=True, type=int) + parser.add_argument("--pp-rank", default=0, type=int) parser.add_argument("--rank", default=0, type=int) parser.add_argument("--timeout-ms", default=1000, type=int) return parser.parse_args() @@ -37,7 +38,7 @@ def main() -> None: sock.setsockopt(zmq.RCVTIMEO, args.timeout_ms) try: sock.connect(make_zmq_path(args.host, args.port)) - sock.send(msgspec.msgpack.encode((GET_META_MSG, args.rank))) + sock.send(msgspec.msgpack.encode((GET_META_MSG, args.pp_rank, args.rank))) sock.recv() finally: sock.close() diff --git a/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh index 9ae15ad0e8..a357128d3c 100755 --- a/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh @@ -49,13 +49,16 @@ else KV_EXTRA_CONFIG='' fi +# Connector: default pull NixlConnector; NixlPushConnector enables PP prefill. +KV_CONNECTOR=${KV_CONNECTOR:-NixlConnector} + # Build the kv-transfer-config for P and D if [[ "$KV_BUFFER_DEVICE" == "cuda" ]]; then - KV_CONFIG_P='{"kv_connector":"NixlConnector","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}' - KV_CONFIG_D='{"kv_connector":"NixlConnector","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}' + KV_CONFIG_P='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_producer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}' + KV_CONFIG_D='{"kv_connector":"'"$KV_CONNECTOR"'","kv_role":"kv_consumer"'${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}'}' else - KV_CONFIG_P="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}" - KV_CONFIG_D="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}" + KV_CONFIG_P="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_producer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}" + KV_CONFIG_D="{\"kv_connector\":\"$KV_CONNECTOR\",\"kv_role\":\"kv_consumer\",\"kv_buffer_device\":\"$KV_BUFFER_DEVICE\""${KV_CONFIG_HETERO_LAYOUT}${KV_EXTRA_CONFIG}"}" fi # Models to run @@ -72,6 +75,7 @@ fi NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} # Default to 1 NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} # Default to 1 PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} +PREFILLER_PP_SIZE=${PREFILLER_PP_SIZE:-1} # >1 requires NixlPushConnector DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2} PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128} @@ -138,8 +142,9 @@ run_tests_for_model() { # Calculate GPU ID - we'll distribute across available GPUs GPU_ID=$((i % $(get_num_gpus))) NEXT_GPU=${GPU_ID} - # If PREFILLER_TP_SIZE is more than 1 - for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do + # Reserve TP*PP GPUs for the prefiller (TP shards across PP stages). + PREFILLER_WORLD_SIZE=$((PREFILLER_TP_SIZE * PREFILLER_PP_SIZE)) + for (( j=1; j < PREFILLER_WORLD_SIZE; j++ )); do NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus))) GPU_ID="${GPU_ID},${NEXT_GPU}" done @@ -161,6 +166,7 @@ run_tests_for_model() { --block-size ${PREFILL_BLOCK_SIZE} \ --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ --tensor-parallel-size $PREFILLER_TP_SIZE \ + --pipeline-parallel-size $PREFILLER_PP_SIZE \ --kv-transfer-config '$KV_CONFIG_P'" if [[ "$ENFORCE_EAGER" == "1" ]]; then BASE_CMD="${BASE_CMD} --enforce-eager" diff --git a/tests/v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh new file mode 100755 index 0000000000..c5c886a239 --- /dev/null +++ b/tests/v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +# NixlPushConnector (push-mode PD) integration tests. Kept separate from the +# pull-mode sweep in ../nixl_integration so push coverage can grow on its own; +# the per-config runner is shared. +SCRIPT="v1/kv_connector/nixl_integration/run_accuracy_test.sh" +IMPORT_CANARY="v1/kv_connector/nixl_integration/test_nixl_imports.py" + +echo "=== Running NIXL import canary ===" +python3 -m pytest -s -x "${IMPORT_CANARY}" + +# PP=2 prefill -> TP=2 decode: each PP stage writes its layer slice into the +# decoder's matching region range (non-MLA and MLA contiguous-slice paths). +configs=( + "KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.6 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2" # non-MLA, Qwen3-0.6B + "KV_CONNECTOR=NixlPushConnector GPU_MEMORY_UTILIZATION=0.8 PREFILLER_PP_SIZE=2 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA +) + +run_tests() { + local label=$1 + local extra_args=$2 + + echo "=== Running push tests (${label}) ===" + for cfg in "${configs[@]}"; do + local -a cfg_parts extra_args_parts + read -r -a cfg_parts <<< "$cfg" + read -r -a extra_args_parts <<< "$extra_args" + + echo "-> Running with ${cfg} ${extra_args:+and ${extra_args}}" + # Use 'env' to safely set variables without eval; keep argv splitting + # safe and SC2086-clean via arrays. + if ! env "${cfg_parts[@]}" bash "${SCRIPT}" "${extra_args_parts[@]}"; then + echo "❌ Test failed for config: ${cfg} ${extra_args:+(${extra_args})}" + exit 1 + fi + done + echo "✅ All ${label} push tests passed!" +} + +run_tests "default backend" "" diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index cd13efd451..49d75e193f 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -408,11 +408,10 @@ def test_kv_transfer_handshake(dist_init): decoder = msgspec.msgpack.Decoder(NixlAgentMetadata) expected_agent_metadata = decoder.decode(metadata.agent_metadata_bytes) - # The scheduler connector expects metadata to be in - # dict[int, KVConnectorHandshakeMetadata], where the first key is - # the dp_rank, the second key is the tp_rank. + # The scheduler connector expects metadata keyed by + # (pp_rank, tp_rank). scheduler_connector = scheduler.get_kv_connector() - scheduler_connector.set_xfer_handshake_metadata({0: metadata}) + scheduler_connector.set_xfer_handshake_metadata_pp_aware({(0, 0): metadata}) # Simulate a request that finishes prefill, which returns # corresponding NixlConnectorMetadata for decode instance. @@ -502,8 +501,14 @@ class FakeNixlConnectorWorker(NixlConnectorWorker): ) def _nixl_handshake( - self, host: str, port: int, remote_tp_size: int, expected_engine_id: str - ) -> dict[int, str]: + self, + host: str, + port: int, + remote_tp_size: int, + expected_engine_id: str, + remote_pp_size: int = 1, + notif_agents_only: bool = False, + ) -> dict[tuple[int, int], str]: # Mimic slow _nixl_handshake, as well as bypass zmq communication. time.sleep(self._hand_shake_latency) # These should've been done in register_kv_caches(), called by @@ -533,7 +538,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker): # When remote tp_size > local tp_size, handshake with multiple # remote ranks. num_handshakes = 1 if tp_ratio > 0 else -tp_ratio - remote_agents: dict[int, str] = {} + remote_agents: dict[tuple[int, int], str] = {} for remote_tp_rank in range(num_handshakes): remote_agent_name = self.add_remote_agent( NixlAgentMetadata( @@ -554,7 +559,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker): remote_tp_rank=remote_tp_rank, remote_tp_size=remote_tp_size, ) - remote_agents[remote_tp_rank] = remote_agent_name + remote_agents[(0, remote_tp_rank)] = remote_agent_name return remote_agents @@ -749,7 +754,7 @@ class TestNixlHandshake: def check_handshake(remote_tp_size: int): tp_ratio = remote_tp_size // local_tp_size - assert set(remote_agents.keys()) == set(range(tp_ratio)) + assert set(remote_agents.keys()) == {(0, r) for r in range(tp_ratio)} remote_engine_id = worker.REMOTE_ENGINE_ID remote_info = worker.transfer_topo.get_engine_info(remote_engine_id) @@ -2044,7 +2049,7 @@ def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): # P TP = 2 * D TP case, we should register 2 local handles worker.src_xfer_handles_by_tp_ratio = {-2: [456, 457]} worker.dst_xfer_side_handles = {"engine1": {0: 789}} - worker._remote_agents = {"engine1": {0: "agent1"}} + worker._remote_agents = {"engine1": {(0, 0): "agent1"}} # _cleanup_remote_engine (called by shutdown) also clears these: worker.kv_caches_base_addr["engine1"] = {0: [0xABC]} worker.dst_num_blocks["engine1"] = 50 @@ -2097,7 +2102,7 @@ def _setup_worker_with_remote_engine( ) engine_id = "remote-engine-1" - worker._remote_agents[engine_id] = {0: "agent_0", 1: "agent_1"} + worker._remote_agents[engine_id] = {(0, 0): "agent_0", (0, 1): "agent_1"} worker.dst_xfer_side_handles[engine_id] = {0: 100, 1: 200} worker.kv_caches_base_addr[engine_id] = {0: [0xABC]} worker.dst_num_blocks[engine_id] = 50 @@ -2993,7 +2998,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario) local_block_len=worker.block_size * 4096, ) worker._remote_agents[remote_engine_id] = { - rank: f"agent_p{rank}" for rank in range(prefill_tp_size) + (0, rank): f"agent_p{rank}" for rank in range(prefill_tp_size) } worker.dst_xfer_side_handles = { remote_engine_id: {rank: 100 + rank for rank in range(prefill_tp_size)} @@ -3044,7 +3049,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario) # Broadcast goes to ranks {1, 2, 3} only, never to the read target. expected_recipients = { - worker._remote_agents[remote_engine_id][r] + worker._remote_agents[remote_engine_id][(0, r)] for r in range(1, prefill_tp_size) } assert {agent for agent, _ in send_notif_calls} == expected_recipients diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index fe67c1ac73..3c9cce1bd6 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -29,9 +29,11 @@ from typing import Any from unittest.mock import MagicMock, patch import msgspec +import pytest from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( PUSH_REG_NOTIF_PREFIX, + NixlAgentMetadata, NixlConnectorMetadata, ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( @@ -512,7 +514,7 @@ class TestPushWriterNotifs: # Pretend the writer thread already forwarded a completion notif # for a request whose KV is being received. request_id = "req-recv-1" - w._recving_metadata[request_id] = MagicMock() + w._recving_metadata[request_id] = MagicMock(pp_size=1) # Compose the standard completion notif: req_id:tp_size. notif_msg = f"{request_id}:1".encode() w._pending_completion_notifs.put(notif_msg) @@ -813,3 +815,177 @@ class TestPushWriterNegative: assert w._reqs_to_send[rid] >= now # Unknown request must not be inserted by the heartbeat path. assert "req-unknown" not in w._reqs_to_send + + +# ----------------------------------------------------------------- # +# Pipeline-parallel producer (push-mode PP-disagg) # +# ----------------------------------------------------------------- # + + +class TestPushPipelineParallel: + """PP-sharded producer: per-stage completion counting, pp_size plumbing, + and remote-region slicing.""" + + def test_completion_waits_for_one_notif_per_pp_stage(self): + """Each PP stage WRITEs its own layers and sends one notif; D must + collect pp_size notifs before reporting the recv done.""" + w = _StubWriterWorker.fresh() + w.transfer_topo = MagicMock() + request_id = "req-pp-2" + w._recving_metadata[request_id] = MagicMock(pp_size=2) + notif = f"{request_id}:1".encode() + + # First stage: counted, not yet done. + w._pending_completion_notifs.put(notif) + assert w._get_new_notifs() == set() + assert request_id not in w._recving_transfers + assert w.consumer_notification_counts_by_req[request_id] == 1 + + # Second (final) stage: now reported done. + w._pending_completion_notifs.put(notif) + assert w._get_new_notifs() == set() + assert request_id in w._recving_transfers + assert request_id not in w.consumer_notification_counts_by_req + + def test_req_meta_reads_pp_size_from_kv_transfer_params(self): + """D learns the producer's pp_size from kv_transfer_params (forwarded + by the proxy) and defaults to 1 when absent.""" + metadata = NixlConnectorMetadata() + params = { + "remote_block_ids": ([0],), + "remote_engine_id": "p-engine", + "remote_request_id": "p-req", + "remote_host": "localhost", + "remote_port": 1234, + "tp_size": 1, + "pp_size": 2, + } + metadata.add_new_req_to_recv("req", ([0],), params) + assert metadata.reqs_to_recv["req"].pp_size == 2 + + params.pop("pp_size") + metadata.add_new_req_to_recv("req-default", ([0],), params) + assert metadata.reqs_to_recv["req-default"].pp_size == 1 + + def test_add_remote_agent_slices_remote_regions_to_local_pp_window(self): + """With PP>1 the producer registered regions for the full model, but + this worker holds only a contiguous layer slice; add_remote_agent + trims the remote region list to [offset : offset + num_local_regions] + before building descriptors. We stop right after the slice via a + sentinel on the next collaborator call.""" + block_len = 4096 * 16 + w = _StubWriterWorker.fresh() # seeds writer-thread state for teardown + w.pp_size = 2 + w._remote_region_offset = 2 # this worker owns layers [2, 4) + w.block_len_per_layer = [block_len, block_len] # 2 local layers + w.nixl_wrapper = MagicMock() + + class _StopAfterSlice(RuntimeError): + pass + + w.transfer_topo = MagicMock() + w.transfer_topo.register_remote_engine.side_effect = _StopAfterSlice() + + meta = NixlAgentMetadata( + engine_id="p-engine", + agent_metadata=b"agent", + kv_caches_base_addr=[10, 11, 12, 13], # full 4-layer model + device_id=0, + num_blocks=4, + block_lens=[block_len] * 4, + kv_cache_layout="HND", + block_size=16, + ssm_sizes=(0, 0), + attn_backend_name="FLASH_ATTN", + physical_blocks_per_logical_kv_block=1, + ) + + with pytest.raises(_StopAfterSlice): + w.add_remote_agent(meta, remote_tp_rank=0, remote_tp_size=1) + + # Sliced down to this worker's layer window (regions [2:4]). + assert meta.kv_caches_base_addr == [12, 13] + assert meta.block_lens == [block_len, block_len] + + +class TestPushWriterMlaReplication: + """MLA latent KV is replicated across D's TP ranks, so when D_TP > P_TP + (``tp_ratio < 0``) the producer must *WRITE* the latent into every D rank + it handshook -- not write one and merely notify the rest, which would + leave the un-written ranks decoding against stale KV.""" + + @staticmethod + def _mla_worker_writing_to(d_ranks): + from types import SimpleNamespace + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + TPMapping, + ) + + engine_id = "decode-engine" + w = _StubWriterWorker.fresh() + w.use_mla = True + w.nixl_wrapper = MagicMock() + w.transfer_topo = MagicMock() + w.transfer_topo.get_engine_info.return_value = SimpleNamespace( + remote_tp_size=len(d_ranks), + remote_block_size=16, + remote_physical_blocks_per_logical=1, + ) + # D_TP > P_TP => negative ratio (one P rank feeds |ratio| D ranks). + w.transfer_topo.tp_ratio.return_value = -len(d_ranks) + # The tp-mapping collapses MLA to a single source rank (correct for + # the pull/read direction); the push path must fan it back out. + w.tp_mappings = { + engine_id: TPMapping( + source_ranks_per_group=((0,),), + all_source_ranks=(0,), + rank_to_attention_slot={0: 0}, + rank_offset_factor=0, + ) + } + w._logical_to_remote_kernel_block_ids = lambda block_ids, ratio: block_ids + w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}} + w.src_xfer_handles_by_block_size = {16: 2000} + w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}} + return w, engine_id + + def test_mla_hetero_tp_writes_every_d_rank(self): + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + RemoteMeta, + ReqMeta, + ) + + w, engine_id = self._mla_worker_writing_to(d_ranks=(0, 1)) + + written: list[int] = [] + + def _fake_xfer(**kw): + rank = kw["read_spec"].remote_rank + written.append(rank) + return 1000 + rank # in-flight handle, as the real method returns + + w._xfer_blocks = _fake_xfer + + meta = ReqMeta( + local_block_ids=([100, 101],), + local_physical_block_ids=([100, 101],), + tp_size=1, + remote=RemoteMeta( + block_ids=([200, 201],), + host="", + port=0, + engine_id=engine_id, + request_id="d-req", + ), + ) + + w._xfer_blocks_for_req(req_id="p-req", meta=meta) + + # Every handshook D rank must receive a real WRITE of the latent... + assert sorted(written) == [0, 1] + # ...and no rank may be fobbed off with a bare completion notif. + assert w.nixl_wrapper.send_notif.call_count == 0 + # All of the request's WRITE handles must be tracked together, so the + # engine thread never sees a partial set and double-frees the request. + assert sorted(w._sending_transfers["p-req"]) == [1000, 1001] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index e506850bd7..942ef90926 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -185,6 +185,7 @@ class NixlBaseConnectorScheduler: host = params.get("remote_host") port = params.get("remote_port") tp_size = params.get("tp_size") + pp_size = params.get("pp_size", 1) if ( remote_engine_id is None or remote_request_id is None @@ -199,6 +200,7 @@ class NixlBaseConnectorScheduler: host=host, port=port, tp_size=tp_size, + pp_size=pp_size, ) self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id) self._heartbeat_req_engine[request.request_id] = ( @@ -244,7 +246,7 @@ class NixlBaseConnectorScheduler: ) def set_xfer_handshake_metadata( - self, metadata: dict[int, KVConnectorHandshakeMetadata] + self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata] ) -> None: """ Set the KV connector handshake metadata for this connector. @@ -252,19 +254,20 @@ class NixlBaseConnectorScheduler: Args: metadata (dict): the handshake metadata to set. """ - encoded_data: dict[int, bytes] = {} + encoded_data: dict[tuple[int, int], bytes] = {} encoder = msgspec.msgpack.Encoder() - for tp_rank, rank_metadata in metadata.items(): + for (pp_rank, tp_rank), rank_metadata in metadata.items(): if not isinstance(rank_metadata, NixlHandshakePayload): raise ValueError( "NixlConnectorScheduler expects NixlHandshakePayload for " "handshake metadata." ) - encoded_data[tp_rank] = encoder.encode(rank_metadata) + encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata) logger.debug( - "Tp rank %d: encoded NixlHandshakePayload size: %s bytes", + "PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes", + pp_rank, tp_rank, - str(len(encoded_data[tp_rank])), + str(len(encoded_data[(pp_rank, tp_rank)])), ) # Only start the listener when we have metadata to serve. @@ -287,7 +290,7 @@ class NixlBaseConnectorScheduler: @staticmethod def _nixl_handshake_listener( - encoded_data: dict[int, Any], + encoded_data: dict[tuple[int, int], Any], ready_event: threading.Event, stop_event: threading.Event, host: str, @@ -310,15 +313,18 @@ class NixlBaseConnectorScheduler: if stop_event.is_set(): break continue - # Decode the message which contains (GET_META_MSG, rank) - msg, target_tp_rank = msgspec.msgpack.decode(msg) + # Decode (GET_META_MSG, pp_rank, tp_rank). + msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg) logger.debug( - "Received message for tp rank %s", + "Received message for pp rank %s, tp rank %s", + target_pp_rank, target_tp_rank, ) if msg != GET_META_MSG: logger.warning("Connection listener got unexpected message %s", msg) - sock.send_multipart((identity, b"", encoded_data[target_tp_rank])) + sock.send_multipart( + (identity, b"", encoded_data[(target_pp_rank, target_tp_rank)]) + ) def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int: """D-side only. Returns N-1 for Mamba models since the decoder diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 66bee55f28..eeee594cc3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Base worker-side logic for the NIXL connector.""" +import itertools import logging import os import queue @@ -336,8 +337,11 @@ class NixlBaseConnectorWorker: ) self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config) - # Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}. - self._remote_agents: dict[EngineId, dict[int, str]] = defaultdict(dict) + # Map of engine_id -> {(pp_rank, tp_rank): agent_name, ...}. + # non-PP remote uses pp_rank 0, i.e. (0, tp_rank). + self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict( + dict + ) # Metadata. self.engine_id: EngineId = engine_id @@ -411,6 +415,25 @@ class NixlBaseConnectorWorker: # (so 1 per layer for MLA, otherwise 2 per layer) self.num_regions = 0 + # PP>1 (push mode): this worker holds a contiguous layer slice and + # transfers into the matching sub-range of a PP=1 remote's regions. + self.pp_size = vllm_config.parallel_config.pipeline_parallel_size + self._remote_region_offset = 0 + # PP push slices regions per layer (uniform count); HMA breaks that. + if self.pp_size > 1 and self._is_hma_required: + raise NotImplementedError( + "NixlPushConnector does not support pipeline_parallel_size > 1 " + "with hybrid KV cache layouts (HMA) yet." + ) + # Decode-side PP is unsupported (completions counted per consumer rank). + if vllm_config.kv_transfer_config.kv_role == "kv_consumer" and self.pp_size > 1: + raise NotImplementedError( + "NixlPushConnector consumer (decode) does not support " + "pipeline_parallel_size > 1." + ) + # Keep heartbeat handshakes to a PP-sharded producer notif-only. + self._hb_handshake_notif_only = False + # nixl_prepped_dlist_handle. self.src_xfer_handles_by_block_size: dict[int, int] = {} # Populated dynamically during handshake based on remote configuration. @@ -449,7 +472,7 @@ class NixlBaseConnectorWorker: thread_name_prefix="vllm-nixl-handshake-initiator", ) self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]() - self._handshake_futures: dict[EngineId, Future[dict[int, str]]] = {} + self._handshake_futures: dict[EngineId, Future[dict[tuple[int, int], str]]] = {} # Protects _handshake_futures and _remote_agents. self._handshake_lock = threading.RLock() @@ -536,7 +559,9 @@ class NixlBaseConnectorWorker: port: int, remote_tp_size: int, expected_engine_id: str, - ) -> dict[int, str]: + remote_pp_size: int = 1, + notif_agents_only: bool = False, + ) -> dict[tuple[int, int], str]: """Do a NIXL handshake with a remote instance.""" # the first time we connect to a remote agent. @@ -558,20 +583,25 @@ class NixlBaseConnectorWorker: # this happens to be the same single rank_i. assert self.transfer_topo is not None p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size) - remote_rank_to_agent_name = {} + remote_rank_to_agent_name: dict[tuple[int, int], str] = {} path = make_zmq_path("tcp", host, port) with zmq_ctx(zmq.REQ, path) as sock: - for remote_rank in p_remote_ranks: + for remote_pp_rank, remote_rank in itertools.product( + range(remote_pp_size), p_remote_ranks + ): logger.debug( - "Querying metadata on path: %s at remote tp rank %s", + "Querying metadata on path: %s at remote pp rank %s, tp rank %s", path, + remote_pp_rank, remote_rank, ) start_time = time.perf_counter() # Send query for the request. - msg = msgspec.msgpack.encode((GET_META_MSG, remote_rank)) + msg = msgspec.msgpack.encode( + (GET_META_MSG, remote_pp_rank, remote_rank) + ) # Set receive timeout to 5 seconds to avoid hanging on dead server sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds sock.send(msg) @@ -638,17 +668,44 @@ class NixlBaseConnectorWorker: ) # Register Remote agent. - remote_agent_name = self.add_remote_agent( - metadata, remote_rank, remote_tp_size - ) + if notif_agents_only: + remote_agent_name = self._add_notif_only_remote_agent( + metadata, remote_tp_size + ) + else: + remote_agent_name = self.add_remote_agent( + metadata, remote_rank, remote_tp_size + ) setup_agent_time = time.perf_counter() logger.debug( "NIXL handshake: add agent took: %s", setup_agent_time - got_metadata_time, ) - remote_rank_to_agent_name[remote_rank] = remote_agent_name + remote_ranks = (remote_pp_rank, remote_rank) + remote_rank_to_agent_name[remote_ranks] = remote_agent_name return remote_rank_to_agent_name + def _add_notif_only_remote_agent( + self, metadata: NixlAgentMetadata, remote_tp_size: int + ) -> str: + """Load a remote agent for notifs only on the push-mode decode side. + + Skips descriptor setup but records engine info for block accounting. + """ + assert self.transfer_topo is not None + self.transfer_topo.register_remote_engine( + metadata.engine_id, + EngineTransferInfo( + remote_tp_size=remote_tp_size, + remote_block_size=metadata.block_size, + remote_block_len=metadata.block_lens[0], + remote_physical_blocks_per_logical=( + metadata.physical_blocks_per_logical_kv_block + ), + ), + ) + return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata) + def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None: """ Initialize transfer buffer in CPU mem for accelerators @@ -761,7 +818,9 @@ class NixlBaseConnectorWorker: host: str, port: int, tp_size: int, - ) -> Future[dict[int, str]] | None: + pp_size: int = 1, + notif_agents_only: bool = False, + ) -> Future[dict[tuple[int, int], str]] | None: """ Ensure a handshake is in-flight (or already done) for *engine_id*. @@ -784,10 +843,12 @@ class NixlBaseConnectorWorker: port, tp_size, engine_id, + pp_size, + notif_agents_only, ) self._handshake_futures[engine_id] = fut - def done_callback(f: Future[dict[int, str]], eid=engine_id): + def done_callback(f: Future[dict[tuple[int, int], str]], eid=engine_id): with self._handshake_lock: del self._handshake_futures[eid] try: @@ -1114,6 +1175,15 @@ class NixlBaseConnectorWorker: self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses self.num_regions = len(caches_data) + if self.pp_size > 1: + start_layer, end_layer = self.model_config.get_layers_start_end_indices( + self.vllm_config.parallel_config + ) + num_local_layers = end_layer - start_layer + assert num_local_layers > 0 and self.num_regions % num_local_layers == 0 + regions_per_layer = self.num_regions // num_local_layers + self._remote_region_offset = regions_per_layer * start_layer + if self.transfer_topo.virtually_split_kv_in_blocks: # NOTE (NickLucche) When FlashInfer is used, memory is registered # with joint KV for each block. This minimizes the overhead in @@ -1464,14 +1534,32 @@ class NixlBaseConnectorWorker: """ # noqa: E501 engine_id = nixl_agent_meta.engine_id # TODO re-evaluate refreshing for scaling/recovery - if remote_tp_rank in self._remote_agents.get(engine_id, {}): + if (0, remote_tp_rank) in self._remote_agents.get(engine_id, {}): logger.debug( "Remote agent with engine_id %s and rank" "%s already exchanged metadata, skip handshake.", engine_id, remote_tp_rank, ) - return self._remote_agents[engine_id][remote_tp_rank] + return self._remote_agents[engine_id][(0, remote_tp_rank)] + + # Compare physical regions, not self.num_regions (doubled by + # FlashInfer's virtual K/V split). + num_local_regions = len(self.block_len_per_layer) + if ( + self.pp_size > 1 + and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions + ): + # This worker holds a PP layer-slice; the PP=1 remote registered + # the full model. Slice its regions to our layer window so the + # logic below sees congruent local/remote lists. + start = self._remote_region_offset + end = start + num_local_regions + assert len(nixl_agent_meta.kv_caches_base_addr) >= end + nixl_agent_meta.kv_caches_base_addr = nixl_agent_meta.kv_caches_base_addr[ + start:end + ] + nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end] ### Register remote engine in TransferTopology (idempotent). assert self.transfer_topo is not None @@ -2098,7 +2186,12 @@ class NixlBaseConnectorWorker: # the **next** heartbeat for this remote can go through. if ( self._ensure_handshake( - engine_id, hb_info.host, hb_info.port, hb_info.tp_size + engine_id, + hb_info.host, + hb_info.port, + hb_info.tp_size, + hb_info.pp_size, + self._hb_handshake_notif_only and hb_info.pp_size > 1, ) is not None ): @@ -2374,19 +2467,23 @@ class NixlBaseConnectorWorker: """ assert engine_id in self._remote_agents - for handle in self.dst_xfer_side_handles.pop(engine_id).values(): + # Notif-only engines (push-mode D side) have no descriptor state. + for handle in self.dst_xfer_side_handles.pop(engine_id, {}).values(): self.nixl_wrapper.release_dlist_handle(handle) for agent_name in self._remote_agents.pop(engine_id).values(): self.nixl_wrapper.remove_remote_agent(agent_name) - del self.kv_caches_base_addr[engine_id] - del self.dst_num_blocks[engine_id] - del self.tp_mappings[engine_id] + self.kv_caches_base_addr.pop(engine_id, None) + self.dst_num_blocks.pop(engine_id, None) + self.tp_mappings.pop(engine_id, None) if self.transfer_topo is not None: self.transfer_topo.unregister_remote_engine(engine_id) - last_active = self._engine_last_active.pop(engine_id) - if log_eviction: + # Push P-side engines are tracked in _remote_agents but not in + # _engine_last_active (they don't participate in stale eviction), so + # tolerate a missing entry. + last_active = self._engine_last_active.pop(engine_id, None) + if log_eviction and last_active is not None: logger.info( "Evicted stale remote engine %s (inactive for %.1fs).", engine_id, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py index b321450530..7d9a34406b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py @@ -207,11 +207,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA): assert self.connector_scheduler is not None return self.connector_scheduler.request_finished(request, block_ids) - def set_xfer_handshake_metadata( - self, metadata: dict[int, KVConnectorHandshakeMetadata] + def set_xfer_handshake_metadata_pp_aware( + self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata] ) -> None: """ - Set the KV connector handshake metadata for this connector. + Set handshake metadata keyed by (pp_rank, tp_rank) so the side + channel can serve every PP stage's agent metadata. Args: metadata (dict): the handshake metadata to set. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index c120f939af..8ca10c2dc4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -147,6 +147,7 @@ class HeartbeatInfo: host: str port: int tp_size: int + pp_size: int = 1 @dataclass @@ -167,6 +168,8 @@ class ReqMeta: remote: RemoteMeta | None = None # Remote block size, discovered during NIXL handshake (push mode). remote_block_size: int | None = None + # Remote producer pipeline-parallel size (push mode, D side). + pp_size: int = 1 class NixlConnectorMetadata(KVConnectorMetadata): @@ -196,6 +199,7 @@ class NixlConnectorMetadata(KVConnectorMetadata): # P workers don't need to receive tp_size from proxy here. tp_size=kv_transfer_params.get("tp_size", 1), remote_block_size=kv_transfer_params.get("remote_block_size"), + pp_size=kv_transfer_params.get("pp_size", 1), ) def add_new_req_to_save( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 26f5fde24d..84e8d29285 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -183,7 +183,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): notif_id = f"{meta.remote.request_id}:{self.world_size}".encode() remote_agents = self._remote_agents[meta.remote.engine_id] for rank_to_notify, agent in remote_agents.items(): - if rank_to_notify != read_specs[0].remote_rank: + if rank_to_notify != (0, read_specs[0].remote_rank): self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) def _read_blocks( @@ -250,7 +250,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): # just notify P worker that we have the blocks we need. if len(local_block_ids) == 0: # A full prefix cache hit is indicated with an empty list. - agent_name = self._remote_agents[dst_engine_id][remote_rank] + agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)] try: self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id) except Exception as e: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py index 23c6e6cd35..413ebdcdb0 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py @@ -186,6 +186,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): "remote_host": params["remote_host"], "remote_port": params["remote_port"], "remote_tp_size": params["tp_size"], + "remote_pp_size": params.get("pp_size", 1), } self._push_registration_deadlines[request.request_id] = ( time.perf_counter() + self._push_registration_timeout @@ -238,6 +239,8 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): # serving layer via abort_immediately. To keep P from # stranding the prefill blocks, we still register an empty # recv so the worker emits a notif that lets P free them. + # Seed remote_block_ids so add_new_req_to_recv won't KeyError. + params["remote_block_ids"] = () self._reqs_need_recv[request.request_id] = (request, []) params["do_remote_prefill"] = False return False, None @@ -285,6 +288,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): remote_host=self.side_channel_host, remote_port=self.side_channel_port, tp_size=self.vllm_config.parallel_config.tensor_parallel_size, + pp_size=self.vllm_config.parallel_config.pipeline_parallel_size, remote_num_tokens=remote_num_tokens, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index a15fc204d2..0d1ab49767 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -80,6 +80,10 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): ): super().__init__(vllm_config, engine_id, kv_cache_config) + # Heartbeat handshakes to a PP-sharded producer must be notif-only, + # like the PUSH_REG path. + self._hb_handshake_notif_only = True + # Push-specific state. # P-side: outgoing WRITE handles awaiting completion, keyed by # request_id. Mutated by writer (submit) and main thread @@ -275,18 +279,22 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): and the request is re-queued onto ``_reg_send_inbox`` once it completes (at which point ``_ensure_handshake`` returns ``None`` and we send directly).""" + remote_pp_size = reg_data.get("remote_pp_size", 1) fut = self._ensure_handshake( reg_data["remote_engine_id"], reg_data["remote_host"], reg_data["remote_port"], reg_data["remote_tp_size"], + pp_size=remote_pp_size, + # D never addresses P memory in push mode; just load P's agents. + notif_agents_only=remote_pp_size > 1, ) if fut is None: self._do_send_reg_notif(req_id, reg_data) return def _on_handshake( - f: Future[dict[int, str]], + f: Future[dict[tuple[int, int], str]], rid: str = req_id, rd: dict[str, Any] = reg_data, ) -> None: @@ -500,28 +508,43 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): remote_block_ids = meta.remote.block_ids local_block_ids = meta.local_physical_block_ids num_groups = len(local_block_ids) - read_specs = [ - ReadSpec( - remote_rank=rank, - local_block_ids=[ - list(local_block_ids[g]) - if rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ], - remote_block_ids=[ - list(remote_block_ids[g]) - if rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ], - ) - for rank in plan.all_source_ranks - ] if self.use_mla and tp_ratio < 0: - assert len(read_specs) == 1 + # MLA latent is replicated across D's TP ranks: the tp-mapping + # collapses to one rank (fine for reads), but push must WRITE every + # D rank or the rest decode stale KV; only the dst differs per rank. + assert len(plan.all_source_ranks) == 1 + mla_local_ids = [list(ids) for ids in local_block_ids] + mla_remote_ids = [list(ids) for ids in remote_block_ids] + read_specs = [ + ReadSpec( + remote_rank=rank, + local_block_ids=mla_local_ids, + remote_block_ids=mla_remote_ids, + ) + for rank in self.dst_xfer_side_handles[engine_id] + ] + else: + read_specs = [ + ReadSpec( + remote_rank=rank, + local_block_ids=[ + list(local_block_ids[g]) + if rank in plan.source_ranks_per_group[g] + else [] + for g in range(num_groups) + ], + remote_block_ids=[ + list(remote_block_ids[g]) + if rank in plan.source_ranks_per_group[g] + else [] + for g in range(num_groups) + ], + ) + for rank in plan.all_source_ranks + ] + handles: list[int] = [] for i, spec in enumerate(read_specs): remote_block_size = remote_info.remote_block_size logger.debug( @@ -544,7 +567,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): spec.remote_rank ] - self._xfer_blocks( + handle = self._xfer_blocks( read_spec=spec, request_id=req_id, dst_engine_id=meta.remote.engine_id, @@ -552,13 +575,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): local_xfer_side_handle=local_xfer_side_handle, remote_xfer_side_handle=remote_xfer_side_handle, ) + if handle is not None: + handles.append(handle) - if self.use_mla and tp_ratio < 0 and read_specs: - notif_id = f"{meta.remote.request_id}:{self.world_size}".encode() - remote_agents = self._remote_agents[meta.remote.engine_id] - for rank_to_notify, agent in remote_agents.items(): - if rank_to_notify != read_specs[0].remote_rank: - self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + # Publish all the request's WRITE handles in one locked update: a + # partial set would let ``_pop_done_transfers`` finish the request + # early, then double-report it as the remaining writes land. + if handles: + with self._sending_transfers_lock: + self._sending_transfers[req_id].extend(handles) def _xfer_blocks( self, @@ -568,8 +593,12 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): remote_request_id: str, local_xfer_side_handle: int, remote_xfer_side_handle: int, - ): - """Post a WRITE point-to-point xfer request.""" + ) -> int | None: + """Post a WRITE point-to-point xfer request. + + Returns the in-flight transfer handle (so the caller can track all of + a request's handles atomically), or ``None`` if nothing was submitted. + """ assert self.transfer_topo is not None remote_rank = read_spec.remote_rank local_block_ids = read_spec.local_block_ids @@ -597,7 +626,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): if len(local_block_ids) == 0: logger.warning("No blocks to push for request %s", request_id) - return + return None # Align per-group block counts for push. local_block_ids = list(local_block_ids) @@ -637,9 +666,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): notif_msg=notif_id, ) self.nixl_wrapper.transfer(handle) - # Track push WRITE handles so P can free blocks once done. - with self._sending_transfers_lock: - self._sending_transfers[request_id].append(handle) + # Caller tracks the handle (atomically with the request's other + # writes) so P can free blocks once all of them are done. + return handle except Exception as e: self._log_failure( failure_type="transfer_setup_failed", @@ -656,6 +685,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): if handle is not None: self.nixl_wrapper.release_xfer_handle(handle) self.xfer_stats.record_failed_transfer() + return None # --- Notification handling on engine main thread ------------------ # @@ -683,12 +713,20 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # Not tracked as a P-side send/process for this notif. if req_id not in self._reqs_to_send and req_id not in self._reqs_to_process: - if req_id in self._recving_metadata: - # D-side: P signalled push completion. The transfer was - # driven entirely by P (we don't own a NIXL handle here), - # so materialise an empty entry in ``_recving_transfers`` - # and let ``_pop_done_transfers`` report it done on the - # next ``get_finished``. + if (meta := self._recving_metadata.get(req_id)) is not None: + # Consumer waits for one notif per producer rank writing + # here: pp_size stages * producers-per-consumer (>1 when + # producer TP > consumer TP; tp_size is the producer TP). + producers_per_consumer = max(1, int(tp_size) // self.world_size) + expected_notifs = meta.pp_size * producers_per_consumer + self.consumer_notification_counts_by_req[req_id] += 1 + notifs = self.consumer_notification_counts_by_req[req_id] + if notifs < expected_notifs: + continue + del self.consumer_notification_counts_by_req[req_id] + # P drove the transfer (we own no NIXL handle), so + # materialise an empty ``_recving_transfers`` entry for + # ``_pop_done_transfers`` to report done. self._recving_transfers.setdefault(req_id, []) else: # Not tracked on either side (lease may have expired