[Bugfix][KV Connector][NIXL] Support PCP producers (#52779)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
3e47a9a824
commit
0a3a4f7f35
@@ -363,9 +363,18 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
c.handle_preemptions(cm)
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
# TODO(https://github.com/vllm-project/vllm/issues/33400)
|
||||
# Currently no connectors return non-None
|
||||
return None
|
||||
child_counts = [
|
||||
connector.get_finished_count() for connector in self._connectors
|
||||
]
|
||||
if any(count is None for count in child_counts):
|
||||
return None
|
||||
counts = {count for count in child_counts if count is not None}
|
||||
if len(counts) > 1:
|
||||
raise ValueError(
|
||||
"MultiConnector children returned incompatible finished counts: "
|
||||
f"{sorted(counts)}"
|
||||
)
|
||||
return next(iter(counts), None)
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
metadata_list: list[KVConnectorWorkerMetadata | None] | None = None
|
||||
|
||||
@@ -62,6 +62,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
@@ -369,6 +370,9 @@ class NixlBaseConnectorWorker:
|
||||
self.engine_id: EngineId = engine_id
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.world_size = get_tensor_model_parallel_world_size()
|
||||
self.pcp_size = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
|
||||
|
||||
self.num_blocks = kv_cache_config.num_blocks
|
||||
self.enable_permute_local_kv = False
|
||||
@@ -561,6 +565,23 @@ class NixlBaseConnectorWorker:
|
||||
"enforce_handshake_compat", True
|
||||
)
|
||||
|
||||
def _validate_remote_parallel_config(
|
||||
self, agent_metadata: NixlAgentMetadata
|
||||
) -> None:
|
||||
local_pcp_size = self.pcp_size
|
||||
local_dcp_size = self.dcp_size
|
||||
remote_pcp_size = agent_metadata.pcp_size
|
||||
remote_dcp_size = agent_metadata.dcp_size
|
||||
if (local_pcp_size > 1 and remote_dcp_size > 1) or (
|
||||
remote_pcp_size > 1 and local_dcp_size > 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP requires decode_context_parallel_size=1 "
|
||||
"on both instances. "
|
||||
f"Local PCP/DCP={local_pcp_size}/{local_dcp_size}; "
|
||||
f"remote PCP/DCP={remote_pcp_size}/{remote_dcp_size}."
|
||||
)
|
||||
|
||||
def _sync_block_size_with_kernel(self) -> None:
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
@@ -700,6 +721,8 @@ class NixlBaseConnectorWorker:
|
||||
f"Failed to decode NixlAgentMetadata. Error: {e}"
|
||||
) from e
|
||||
|
||||
self._validate_remote_parallel_config(metadata)
|
||||
|
||||
# Ensure engine id matches.
|
||||
if metadata.engine_id != expected_engine_id:
|
||||
raise RuntimeError(
|
||||
@@ -1033,6 +1056,8 @@ class NixlBaseConnectorWorker:
|
||||
physical_blocks_per_logical_kv_block=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
),
|
||||
dcp_size=self.dcp_size,
|
||||
pcp_size=self.pcp_size,
|
||||
)
|
||||
assert self.compat_hash is not None
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
@@ -1277,6 +1302,8 @@ class NixlBaseConnectorWorker:
|
||||
physical_blocks_per_logical_kv_block=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
),
|
||||
dcp_size=self.dcp_size,
|
||||
pcp_size=self.pcp_size,
|
||||
)
|
||||
# Wrap metadata in payload with hash for defensive decoding
|
||||
assert self.compat_hash is not None
|
||||
|
||||
@@ -131,6 +131,29 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"kv_role='kv_consumer' for decode instances. "
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
pcp_size = parallel_config.prefill_context_parallel_size
|
||||
kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
if pcp_size > 1 and kv_role in ("kv_consumer", "kv_both"):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP currently supports kv_producer only. "
|
||||
"Consumers and kv_both require "
|
||||
"prefill_context_parallel_size=1."
|
||||
)
|
||||
if pcp_size > 1 and parallel_config.decode_context_parallel_size > 1:
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP producers currently require "
|
||||
"decode_context_parallel_size=1."
|
||||
)
|
||||
# TODO: Support PCP with bidirectional KV transfer by tracking separate
|
||||
# send and receive completion counts.
|
||||
if pcp_size > 1 and vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"bidirectional_kv_xfer", False
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"NixlConnector PCP producers do not support bidirectional KV transfer."
|
||||
)
|
||||
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
|
||||
self.kv_transfer_config = vllm_config.kv_transfer_config
|
||||
@@ -164,6 +187,18 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Scheduler Side Methods
|
||||
############################################################
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
parallel_config = self._vllm_config.parallel_config
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and parallel_config.prefill_context_parallel_size > 1
|
||||
):
|
||||
return (
|
||||
parallel_config.tensor_parallel_size
|
||||
* parallel_config.pipeline_parallel_size
|
||||
)
|
||||
return None
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
@@ -244,7 +279,13 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""Get the finished recving and sending requests."""
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
done_sending, done_recving = self.connector_worker.get_finished()
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
):
|
||||
done_sending.clear()
|
||||
return done_sending, done_recving
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs that failed to load via NIXL."""
|
||||
@@ -320,6 +361,11 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
None if no handshake metadata is available.
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
):
|
||||
return None
|
||||
return self.connector_worker.xfer_handshake_metadata
|
||||
|
||||
|
||||
|
||||
@@ -43,8 +43,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# clock-sync timestamp
|
||||
# 6: Validate EAGLE/MTP speculative configuration compatibility
|
||||
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
|
||||
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 7
|
||||
NIXL_CONNECTOR_VERSION: int = 8
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -60,6 +61,8 @@ class NixlAgentMetadata:
|
||||
ssm_sizes: tuple[int, int]
|
||||
attn_backend_name: str
|
||||
physical_blocks_per_logical_kv_block: int
|
||||
dcp_size: int = 1
|
||||
pcp_size: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -76,6 +76,9 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
while not self._ready_requests.empty():
|
||||
self._read_blocks_for_req(*self._ready_requests.get_nowait())
|
||||
|
||||
if self.pcp_rank > 0:
|
||||
return
|
||||
|
||||
# Keep around the requests that have been part of a batch. This is
|
||||
# needed because async scheduling pushes the misalignment between the
|
||||
# moment in which requests expiration is set (P side) and the moment in
|
||||
|
||||
@@ -159,6 +159,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""Pre-process metadata; defer NIXL ops to the writer thread."""
|
||||
if self.pcp_rank > 0:
|
||||
return
|
||||
|
||||
# D-side: track reqs waiting for P to push.
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
meta.local_physical_block_ids = self._logical_to_kernel_block_ids(
|
||||
|
||||
Reference in New Issue
Block a user