[NIXL][PCP] Report replicated-PCP ranks > 0 as done sending instead of hiding them (#53903)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
08426d51ef
commit
93911fcf68
@@ -584,7 +584,7 @@ class TestNixlHandshake:
|
||||
def test_pcp_producer_uses_canonical_replica(
|
||||
self, default_vllm_config, dist_init, pcp_rank
|
||||
):
|
||||
"""Only PCP rank zero publishes and reports sending completion."""
|
||||
"""Only PCP rank zero publishes, but every rank reports completion."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
vllm_config = create_vllm_config(kv_role="kv_producer")
|
||||
@@ -622,13 +622,18 @@ class TestNixlHandshake:
|
||||
|
||||
payload = MagicMock(spec=NixlHandshakePayload)
|
||||
worker.xfer_handshake_metadata = payload
|
||||
worker.get_finished = MagicMock(return_value=({"sent"}, set()))
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker._get_new_notifs = MagicMock(
|
||||
side_effect=lambda: {"sent"} if pcp_rank == 0 else set()
|
||||
)
|
||||
|
||||
expected_payload = payload if pcp_rank == 0 else None
|
||||
assert connector.get_handshake_metadata() is expected_payload
|
||||
done_sending, done_recving = connector.get_finished(set())
|
||||
assert done_sending == ({"sent"} if pcp_rank == 0 else set())
|
||||
assert done_sending == ({"sent"} if pcp_rank == 0 else {req_id})
|
||||
assert done_recving == set()
|
||||
if pcp_rank > 0:
|
||||
assert connector.get_finished(set()) == (set(), set())
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
|
||||
@@ -656,6 +656,9 @@ class NixlBaseConnectorWorker:
|
||||
self._recving_transfers = defaultdict[ReqId, list[TransferHandle]](list)
|
||||
# Track the expiration time of requests that are waiting to be sent.
|
||||
self._reqs_to_send: dict[ReqId, float] = {}
|
||||
# Replicated-KV PCP ranks > 0 never transfer; requests they are asked
|
||||
# to send are reported finished immediately (see get_finished).
|
||||
self._replicated_pcp_done_sending: set[ReqId] = set()
|
||||
# Set of requests that have been part of a batch, regardless of status.
|
||||
self._reqs_to_process: set[ReqId] = set()
|
||||
|
||||
@@ -2543,6 +2546,9 @@ class NixlBaseConnectorWorker:
|
||||
done_sending = self._get_new_notifs()
|
||||
done_recving = self._pop_done_transfers(self._recving_transfers)
|
||||
|
||||
done_sending.update(self._replicated_pcp_done_sending)
|
||||
self._replicated_pcp_done_sending.clear()
|
||||
|
||||
# Drain queue of requests where handshake or transfer setup failed.
|
||||
failed_recv_reqs = set[ReqId]()
|
||||
while not self._failed_recv_reqs.empty():
|
||||
|
||||
@@ -156,15 +156,6 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
############################################################
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
parallel_config = self._vllm_config.parallel_config
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and parallel_config.prefill_context_parallel_size > 1
|
||||
):
|
||||
return (
|
||||
parallel_config.tensor_parallel_size
|
||||
* parallel_config.pipeline_parallel_size
|
||||
)
|
||||
return None
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
@@ -241,13 +232,7 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""Get the finished recving and sending requests."""
|
||||
assert self.connector_worker is not None
|
||||
done_sending, done_recving = self.connector_worker.get_finished()
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
):
|
||||
done_sending.clear()
|
||||
return done_sending, done_recving
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs that failed to load via NIXL."""
|
||||
|
||||
@@ -80,6 +80,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
|
||||
if self.pcp_rank > 0:
|
||||
# Replicated-KV PCP: only PCP rank 0 serves the KV, so this rank
|
||||
# has nothing to send. Report the requests as sent right away so
|
||||
# the scheduler-side aggregation (world_size workers, and any
|
||||
# sibling connector inside a MultiConnector) still completes.
|
||||
self._replicated_pcp_done_sending.update(metadata.reqs_to_send)
|
||||
return
|
||||
|
||||
# Keep around the requests that have been part of a batch. This is
|
||||
|
||||
Reference in New Issue
Block a user