[KVConnector][NIXL] Support pipeline-parallel prefill in push mode (#45880)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-07-08 16:49:23 -07:00
committed by GitHub
co-authored by Nick Hill
parent 95d6d6f4bb
commit 7802c20c4e
15 changed files with 506 additions and 104 deletions
@@ -185,6 +185,7 @@ class NixlBaseConnectorScheduler:
host = params.get("remote_host")
port = params.get("remote_port")
tp_size = params.get("tp_size")
pp_size = params.get("pp_size", 1)
if (
remote_engine_id is None
or remote_request_id is None
@@ -199,6 +200,7 @@ class NixlBaseConnectorScheduler:
host=host,
port=port,
tp_size=tp_size,
pp_size=pp_size,
)
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
self._heartbeat_req_engine[request.request_id] = (
@@ -244,7 +246,7 @@ class NixlBaseConnectorScheduler:
)
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for this connector.
@@ -252,19 +254,20 @@ class NixlBaseConnectorScheduler:
Args:
metadata (dict): the handshake metadata to set.
"""
encoded_data: dict[int, bytes] = {}
encoded_data: dict[tuple[int, int], bytes] = {}
encoder = msgspec.msgpack.Encoder()
for tp_rank, rank_metadata in metadata.items():
for (pp_rank, tp_rank), rank_metadata in metadata.items():
if not isinstance(rank_metadata, NixlHandshakePayload):
raise ValueError(
"NixlConnectorScheduler expects NixlHandshakePayload for "
"handshake metadata."
)
encoded_data[tp_rank] = encoder.encode(rank_metadata)
encoded_data[(pp_rank, tp_rank)] = encoder.encode(rank_metadata)
logger.debug(
"Tp rank %d: encoded NixlHandshakePayload size: %s bytes",
"PP rank %d, TP rank %d: encoded NixlHandshakePayload size: %s bytes",
pp_rank,
tp_rank,
str(len(encoded_data[tp_rank])),
str(len(encoded_data[(pp_rank, tp_rank)])),
)
# Only start the listener when we have metadata to serve.
@@ -287,7 +290,7 @@ class NixlBaseConnectorScheduler:
@staticmethod
def _nixl_handshake_listener(
encoded_data: dict[int, Any],
encoded_data: dict[tuple[int, int], Any],
ready_event: threading.Event,
stop_event: threading.Event,
host: str,
@@ -310,15 +313,18 @@ class NixlBaseConnectorScheduler:
if stop_event.is_set():
break
continue
# Decode the message which contains (GET_META_MSG, rank)
msg, target_tp_rank = msgspec.msgpack.decode(msg)
# Decode (GET_META_MSG, pp_rank, tp_rank).
msg, target_pp_rank, target_tp_rank = msgspec.msgpack.decode(msg)
logger.debug(
"Received message for tp rank %s",
"Received message for pp rank %s, tp rank %s",
target_pp_rank,
target_tp_rank,
)
if msg != GET_META_MSG:
logger.warning("Connection listener got unexpected message %s", msg)
sock.send_multipart((identity, b"", encoded_data[target_tp_rank]))
sock.send_multipart(
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)])
)
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. Returns N-1 for Mamba models since the decoder
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base worker-side logic for the NIXL connector."""
import itertools
import logging
import os
import queue
@@ -336,8 +337,11 @@ class NixlBaseConnectorWorker:
)
self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config)
# Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}.
self._remote_agents: dict[EngineId, dict[int, str]] = defaultdict(dict)
# Map of engine_id -> {(pp_rank, tp_rank): agent_name, ...}.
# non-PP remote uses pp_rank 0, i.e. (0, tp_rank).
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
dict
)
# Metadata.
self.engine_id: EngineId = engine_id
@@ -411,6 +415,25 @@ class NixlBaseConnectorWorker:
# (so 1 per layer for MLA, otherwise 2 per layer)
self.num_regions = 0
# PP>1 (push mode): this worker holds a contiguous layer slice and
# transfers into the matching sub-range of a PP=1 remote's regions.
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
self._remote_region_offset = 0
# PP push slices regions per layer (uniform count); HMA breaks that.
if self.pp_size > 1 and self._is_hma_required:
raise NotImplementedError(
"NixlPushConnector does not support pipeline_parallel_size > 1 "
"with hybrid KV cache layouts (HMA) yet."
)
# Decode-side PP is unsupported (completions counted per consumer rank).
if vllm_config.kv_transfer_config.kv_role == "kv_consumer" and self.pp_size > 1:
raise NotImplementedError(
"NixlPushConnector consumer (decode) does not support "
"pipeline_parallel_size > 1."
)
# Keep heartbeat handshakes to a PP-sharded producer notif-only.
self._hb_handshake_notif_only = False
# nixl_prepped_dlist_handle.
self.src_xfer_handles_by_block_size: dict[int, int] = {}
# Populated dynamically during handshake based on remote configuration.
@@ -449,7 +472,7 @@ class NixlBaseConnectorWorker:
thread_name_prefix="vllm-nixl-handshake-initiator",
)
self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]()
self._handshake_futures: dict[EngineId, Future[dict[int, str]]] = {}
self._handshake_futures: dict[EngineId, Future[dict[tuple[int, int], str]]] = {}
# Protects _handshake_futures and _remote_agents.
self._handshake_lock = threading.RLock()
@@ -536,7 +559,9 @@ class NixlBaseConnectorWorker:
port: int,
remote_tp_size: int,
expected_engine_id: str,
) -> dict[int, str]:
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> dict[tuple[int, int], str]:
"""Do a NIXL handshake with a remote instance."""
# the first time we connect to a remote agent.
@@ -558,20 +583,25 @@ class NixlBaseConnectorWorker:
# this happens to be the same single rank_i.
assert self.transfer_topo is not None
p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size)
remote_rank_to_agent_name = {}
remote_rank_to_agent_name: dict[tuple[int, int], str] = {}
path = make_zmq_path("tcp", host, port)
with zmq_ctx(zmq.REQ, path) as sock:
for remote_rank in p_remote_ranks:
for remote_pp_rank, remote_rank in itertools.product(
range(remote_pp_size), p_remote_ranks
):
logger.debug(
"Querying metadata on path: %s at remote tp rank %s",
"Querying metadata on path: %s at remote pp rank %s, tp rank %s",
path,
remote_pp_rank,
remote_rank,
)
start_time = time.perf_counter()
# Send query for the request.
msg = msgspec.msgpack.encode((GET_META_MSG, remote_rank))
msg = msgspec.msgpack.encode(
(GET_META_MSG, remote_pp_rank, remote_rank)
)
# Set receive timeout to 5 seconds to avoid hanging on dead server
sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds
sock.send(msg)
@@ -638,17 +668,44 @@ class NixlBaseConnectorWorker:
)
# Register Remote agent.
remote_agent_name = self.add_remote_agent(
metadata, remote_rank, remote_tp_size
)
if notif_agents_only:
remote_agent_name = self._add_notif_only_remote_agent(
metadata, remote_tp_size
)
else:
remote_agent_name = self.add_remote_agent(
metadata, remote_rank, remote_tp_size
)
setup_agent_time = time.perf_counter()
logger.debug(
"NIXL handshake: add agent took: %s",
setup_agent_time - got_metadata_time,
)
remote_rank_to_agent_name[remote_rank] = remote_agent_name
remote_ranks = (remote_pp_rank, remote_rank)
remote_rank_to_agent_name[remote_ranks] = remote_agent_name
return remote_rank_to_agent_name
def _add_notif_only_remote_agent(
self, metadata: NixlAgentMetadata, remote_tp_size: int
) -> str:
"""Load a remote agent for notifs only on the push-mode decode side.
Skips descriptor setup but records engine info for block accounting.
"""
assert self.transfer_topo is not None
self.transfer_topo.register_remote_engine(
metadata.engine_id,
EngineTransferInfo(
remote_tp_size=remote_tp_size,
remote_block_size=metadata.block_size,
remote_block_len=metadata.block_lens[0],
remote_physical_blocks_per_logical=(
metadata.physical_blocks_per_logical_kv_block
),
),
)
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""
Initialize transfer buffer in CPU mem for accelerators
@@ -761,7 +818,9 @@ class NixlBaseConnectorWorker:
host: str,
port: int,
tp_size: int,
) -> Future[dict[int, str]] | None:
pp_size: int = 1,
notif_agents_only: bool = False,
) -> Future[dict[tuple[int, int], str]] | None:
"""
Ensure a handshake is in-flight (or already done) for *engine_id*.
@@ -784,10 +843,12 @@ class NixlBaseConnectorWorker:
port,
tp_size,
engine_id,
pp_size,
notif_agents_only,
)
self._handshake_futures[engine_id] = fut
def done_callback(f: Future[dict[int, str]], eid=engine_id):
def done_callback(f: Future[dict[tuple[int, int], str]], eid=engine_id):
with self._handshake_lock:
del self._handshake_futures[eid]
try:
@@ -1114,6 +1175,15 @@ class NixlBaseConnectorWorker:
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
self.num_regions = len(caches_data)
if self.pp_size > 1:
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
self.vllm_config.parallel_config
)
num_local_layers = end_layer - start_layer
assert num_local_layers > 0 and self.num_regions % num_local_layers == 0
regions_per_layer = self.num_regions // num_local_layers
self._remote_region_offset = regions_per_layer * start_layer
if self.transfer_topo.virtually_split_kv_in_blocks:
# NOTE (NickLucche) When FlashInfer is used, memory is registered
# with joint KV for each block. This minimizes the overhead in
@@ -1464,14 +1534,32 @@ class NixlBaseConnectorWorker:
""" # noqa: E501
engine_id = nixl_agent_meta.engine_id
# TODO re-evaluate refreshing for scaling/recovery
if remote_tp_rank in self._remote_agents.get(engine_id, {}):
if (0, remote_tp_rank) in self._remote_agents.get(engine_id, {}):
logger.debug(
"Remote agent with engine_id %s and rank"
"%s already exchanged metadata, skip handshake.",
engine_id,
remote_tp_rank,
)
return self._remote_agents[engine_id][remote_tp_rank]
return self._remote_agents[engine_id][(0, remote_tp_rank)]
# Compare physical regions, not self.num_regions (doubled by
# FlashInfer's virtual K/V split).
num_local_regions = len(self.block_len_per_layer)
if (
self.pp_size > 1
and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions
):
# This worker holds a PP layer-slice; the PP=1 remote registered
# the full model. Slice its regions to our layer window so the
# logic below sees congruent local/remote lists.
start = self._remote_region_offset
end = start + num_local_regions
assert len(nixl_agent_meta.kv_caches_base_addr) >= end
nixl_agent_meta.kv_caches_base_addr = nixl_agent_meta.kv_caches_base_addr[
start:end
]
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
### Register remote engine in TransferTopology (idempotent).
assert self.transfer_topo is not None
@@ -2098,7 +2186,12 @@ class NixlBaseConnectorWorker:
# the **next** heartbeat for this remote can go through.
if (
self._ensure_handshake(
engine_id, hb_info.host, hb_info.port, hb_info.tp_size
engine_id,
hb_info.host,
hb_info.port,
hb_info.tp_size,
hb_info.pp_size,
self._hb_handshake_notif_only and hb_info.pp_size > 1,
)
is not None
):
@@ -2374,19 +2467,23 @@ class NixlBaseConnectorWorker:
"""
assert engine_id in self._remote_agents
for handle in self.dst_xfer_side_handles.pop(engine_id).values():
# Notif-only engines (push-mode D side) have no descriptor state.
for handle in self.dst_xfer_side_handles.pop(engine_id, {}).values():
self.nixl_wrapper.release_dlist_handle(handle)
for agent_name in self._remote_agents.pop(engine_id).values():
self.nixl_wrapper.remove_remote_agent(agent_name)
del self.kv_caches_base_addr[engine_id]
del self.dst_num_blocks[engine_id]
del self.tp_mappings[engine_id]
self.kv_caches_base_addr.pop(engine_id, None)
self.dst_num_blocks.pop(engine_id, None)
self.tp_mappings.pop(engine_id, None)
if self.transfer_topo is not None:
self.transfer_topo.unregister_remote_engine(engine_id)
last_active = self._engine_last_active.pop(engine_id)
if log_eviction:
# Push P-side engines are tracked in _remote_agents but not in
# _engine_last_active (they don't participate in stale eviction), so
# tolerate a missing entry.
last_active = self._engine_last_active.pop(engine_id, None)
if log_eviction and last_active is not None:
logger.info(
"Evicted stale remote engine %s (inactive for %.1fs).",
engine_id,
@@ -207,11 +207,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for this connector.
Set handshake metadata keyed by (pp_rank, tp_rank) so the side
channel can serve every PP stage's agent metadata.
Args:
metadata (dict): the handshake metadata to set.
@@ -147,6 +147,7 @@ class HeartbeatInfo:
host: str
port: int
tp_size: int
pp_size: int = 1
@dataclass
@@ -167,6 +168,8 @@ class ReqMeta:
remote: RemoteMeta | None = None
# Remote block size, discovered during NIXL handshake (push mode).
remote_block_size: int | None = None
# Remote producer pipeline-parallel size (push mode, D side).
pp_size: int = 1
class NixlConnectorMetadata(KVConnectorMetadata):
@@ -196,6 +199,7 @@ class NixlConnectorMetadata(KVConnectorMetadata):
# P workers don't need to receive tp_size from proxy here.
tp_size=kv_transfer_params.get("tp_size", 1),
remote_block_size=kv_transfer_params.get("remote_block_size"),
pp_size=kv_transfer_params.get("pp_size", 1),
)
def add_new_req_to_save(
@@ -183,7 +183,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
notif_id = f"{meta.remote.request_id}:{self.world_size}".encode()
remote_agents = self._remote_agents[meta.remote.engine_id]
for rank_to_notify, agent in remote_agents.items():
if rank_to_notify != read_specs[0].remote_rank:
if rank_to_notify != (0, read_specs[0].remote_rank):
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
def _read_blocks(
@@ -250,7 +250,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# just notify P worker that we have the blocks we need.
if len(local_block_ids) == 0:
# A full prefix cache hit is indicated with an empty list.
agent_name = self._remote_agents[dst_engine_id][remote_rank]
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
try:
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
except Exception as e:
@@ -186,6 +186,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
"remote_host": params["remote_host"],
"remote_port": params["remote_port"],
"remote_tp_size": params["tp_size"],
"remote_pp_size": params.get("pp_size", 1),
}
self._push_registration_deadlines[request.request_id] = (
time.perf_counter() + self._push_registration_timeout
@@ -238,6 +239,8 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
# serving layer via abort_immediately. To keep P from
# stranding the prefill blocks, we still register an empty
# recv so the worker emits a notif that lets P free them.
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, [])
params["do_remote_prefill"] = False
return False, None
@@ -285,6 +288,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
remote_num_tokens=remote_num_tokens,
)
@@ -80,6 +80,10 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
):
super().__init__(vllm_config, engine_id, kv_cache_config)
# Heartbeat handshakes to a PP-sharded producer must be notif-only,
# like the PUSH_REG path.
self._hb_handshake_notif_only = True
# Push-specific state.
# P-side: outgoing WRITE handles awaiting completion, keyed by
# request_id. Mutated by writer (submit) and main thread
@@ -275,18 +279,22 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
and the request is re-queued onto ``_reg_send_inbox`` once it
completes (at which point ``_ensure_handshake`` returns ``None`` and we
send directly)."""
remote_pp_size = reg_data.get("remote_pp_size", 1)
fut = self._ensure_handshake(
reg_data["remote_engine_id"],
reg_data["remote_host"],
reg_data["remote_port"],
reg_data["remote_tp_size"],
pp_size=remote_pp_size,
# D never addresses P memory in push mode; just load P's agents.
notif_agents_only=remote_pp_size > 1,
)
if fut is None:
self._do_send_reg_notif(req_id, reg_data)
return
def _on_handshake(
f: Future[dict[int, str]],
f: Future[dict[tuple[int, int], str]],
rid: str = req_id,
rd: dict[str, Any] = reg_data,
) -> None:
@@ -500,28 +508,43 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
remote_block_ids = meta.remote.block_ids
local_block_ids = meta.local_physical_block_ids
num_groups = len(local_block_ids)
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=[
list(local_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
remote_block_ids=[
list(remote_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
)
for rank in plan.all_source_ranks
]
if self.use_mla and tp_ratio < 0:
assert len(read_specs) == 1
# MLA latent is replicated across D's TP ranks: the tp-mapping
# collapses to one rank (fine for reads), but push must WRITE every
# D rank or the rest decode stale KV; only the dst differs per rank.
assert len(plan.all_source_ranks) == 1
mla_local_ids = [list(ids) for ids in local_block_ids]
mla_remote_ids = [list(ids) for ids in remote_block_ids]
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=mla_local_ids,
remote_block_ids=mla_remote_ids,
)
for rank in self.dst_xfer_side_handles[engine_id]
]
else:
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=[
list(local_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
remote_block_ids=[
list(remote_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
)
for rank in plan.all_source_ranks
]
handles: list[int] = []
for i, spec in enumerate(read_specs):
remote_block_size = remote_info.remote_block_size
logger.debug(
@@ -544,7 +567,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
spec.remote_rank
]
self._xfer_blocks(
handle = self._xfer_blocks(
read_spec=spec,
request_id=req_id,
dst_engine_id=meta.remote.engine_id,
@@ -552,13 +575,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
local_xfer_side_handle=local_xfer_side_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
)
if handle is not None:
handles.append(handle)
if self.use_mla and tp_ratio < 0 and read_specs:
notif_id = f"{meta.remote.request_id}:{self.world_size}".encode()
remote_agents = self._remote_agents[meta.remote.engine_id]
for rank_to_notify, agent in remote_agents.items():
if rank_to_notify != read_specs[0].remote_rank:
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
# Publish all the request's WRITE handles in one locked update: a
# partial set would let ``_pop_done_transfers`` finish the request
# early, then double-report it as the remaining writes land.
if handles:
with self._sending_transfers_lock:
self._sending_transfers[req_id].extend(handles)
def _xfer_blocks(
self,
@@ -568,8 +593,12 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
remote_request_id: str,
local_xfer_side_handle: int,
remote_xfer_side_handle: int,
):
"""Post a WRITE point-to-point xfer request."""
) -> int | None:
"""Post a WRITE point-to-point xfer request.
Returns the in-flight transfer handle (so the caller can track all of
a request's handles atomically), or ``None`` if nothing was submitted.
"""
assert self.transfer_topo is not None
remote_rank = read_spec.remote_rank
local_block_ids = read_spec.local_block_ids
@@ -597,7 +626,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
if len(local_block_ids) == 0:
logger.warning("No blocks to push for request %s", request_id)
return
return None
# Align per-group block counts for push.
local_block_ids = list(local_block_ids)
@@ -637,9 +666,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
notif_msg=notif_id,
)
self.nixl_wrapper.transfer(handle)
# Track push WRITE handles so P can free blocks once done.
with self._sending_transfers_lock:
self._sending_transfers[request_id].append(handle)
# Caller tracks the handle (atomically with the request's other
# writes) so P can free blocks once all of them are done.
return handle
except Exception as e:
self._log_failure(
failure_type="transfer_setup_failed",
@@ -656,6 +685,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
if handle is not None:
self.nixl_wrapper.release_xfer_handle(handle)
self.xfer_stats.record_failed_transfer()
return None
# --- Notification handling on engine main thread ------------------ #
@@ -683,12 +713,20 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# Not tracked as a P-side send/process for this notif.
if req_id not in self._reqs_to_send and req_id not in self._reqs_to_process:
if req_id in self._recving_metadata:
# D-side: P signalled push completion. The transfer was
# driven entirely by P (we don't own a NIXL handle here),
# so materialise an empty entry in ``_recving_transfers``
# and let ``_pop_done_transfers`` report it done on the
# next ``get_finished``.
if (meta := self._recving_metadata.get(req_id)) is not None:
# Consumer waits for one notif per producer rank writing
# here: pp_size stages * producers-per-consumer (>1 when
# producer TP > consumer TP; tp_size is the producer TP).
producers_per_consumer = max(1, int(tp_size) // self.world_size)
expected_notifs = meta.pp_size * producers_per_consumer
self.consumer_notification_counts_by_req[req_id] += 1
notifs = self.consumer_notification_counts_by_req[req_id]
if notifs < expected_notifs:
continue
del self.consumer_notification_counts_by_req[req_id]
# P drove the transfer (we own no NIXL handle), so
# materialise an empty ``_recving_transfers`` entry for
# ``_pop_done_transfers`` to report done.
self._recving_transfers.setdefault(req_id, [])
else:
# Not tracked on either side (lease may have expired