[NIXL] Avoid reading expired blocks in bidirectional turn-2 read (#47021)
Signed-off-by: Tomer Gilad <tgilad@nvidia.com> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
NickLucche
parent
b6754f536e
commit
7a74a9662b
@@ -322,8 +322,13 @@ class NixlBaseConnectorScheduler:
|
||||
)
|
||||
if msg != GET_META_MSG:
|
||||
logger.warning("Connection listener got unexpected message %s", msg)
|
||||
# Echo our perf_counter so P can estimate the clock offset.
|
||||
# perf_counter is only comparable within a process, so this
|
||||
# listener must run in the same process that stamps the block
|
||||
# expiry deadline (`_reqs_need_send`).
|
||||
ts = msgspec.msgpack.encode(time.perf_counter())
|
||||
sock.send_multipart(
|
||||
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)])
|
||||
(identity, b"", encoded_data[(target_pp_rank, target_tp_rank)], ts)
|
||||
)
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
|
||||
@@ -266,6 +266,12 @@ class NixlBaseConnectorWorker:
|
||||
# NOTE (NickLucche): For now we use a hardcoded value for a simpler interface.
|
||||
self._lease_extension = kv_lease_duration * 2 // 3
|
||||
|
||||
self._bidirectional_kv_xfer_enabled: bool = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"bidirectional_kv_xfer", False
|
||||
)
|
||||
)
|
||||
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
@@ -340,6 +346,8 @@ class NixlBaseConnectorWorker:
|
||||
self._remote_agents: dict[EngineId, dict[tuple[int, int], str]] = defaultdict(
|
||||
dict
|
||||
)
|
||||
# Map of engine_id -> clock offset.
|
||||
self._engine_clock_offset: dict[EngineId, float] = {}
|
||||
|
||||
# Metadata.
|
||||
self.engine_id: EngineId = engine_id
|
||||
@@ -470,7 +478,9 @@ class NixlBaseConnectorWorker:
|
||||
thread_name_prefix="vllm-nixl-handshake-initiator",
|
||||
)
|
||||
self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]()
|
||||
self._handshake_futures: dict[EngineId, Future[dict[tuple[int, int], str]]] = {}
|
||||
self._handshake_futures: dict[
|
||||
EngineId, Future[tuple[dict[tuple[int, int], str], float]]
|
||||
] = {}
|
||||
# Protects _handshake_futures and _remote_agents.
|
||||
self._handshake_lock = threading.RLock()
|
||||
|
||||
@@ -559,7 +569,7 @@ class NixlBaseConnectorWorker:
|
||||
expected_engine_id: str,
|
||||
remote_pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> dict[tuple[int, int], str]:
|
||||
) -> tuple[dict[tuple[int, int], str], float]:
|
||||
"""Do a NIXL handshake with a remote instance."""
|
||||
|
||||
# the first time we connect to a remote agent.
|
||||
@@ -583,6 +593,11 @@ class NixlBaseConnectorWorker:
|
||||
p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size)
|
||||
remote_rank_to_agent_name: dict[tuple[int, int], str] = {}
|
||||
path = make_zmq_path("tcp", host, port)
|
||||
# Clock offset to the peer, estimated from the handshake round-trip.
|
||||
# Keep the lowest-RTT sample: hop cost is ~uniform across ranks, so a
|
||||
# higher RTT is just noise that skews the midpoint estimate.
|
||||
best_rtt = float("inf")
|
||||
best_offset: float | None = None
|
||||
|
||||
with zmq_ctx(zmq.REQ, path) as sock:
|
||||
for remote_pp_rank, remote_rank in itertools.product(
|
||||
@@ -595,15 +610,24 @@ class NixlBaseConnectorWorker:
|
||||
remote_rank,
|
||||
)
|
||||
|
||||
start_time = time.perf_counter()
|
||||
# Send query for the request.
|
||||
msg = msgspec.msgpack.encode(
|
||||
(GET_META_MSG, remote_pp_rank, remote_rank)
|
||||
)
|
||||
# Set receive timeout to 5 seconds to avoid hanging on dead server
|
||||
sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds
|
||||
start_time = time.perf_counter()
|
||||
sock.send(msg)
|
||||
handshake_bytes = sock.recv()
|
||||
reply_parts = sock.recv_multipart()
|
||||
recv_time = time.perf_counter()
|
||||
assert len(reply_parts) == 2
|
||||
handshake_bytes = reply_parts[0]
|
||||
|
||||
remote_perf = msgspec.msgpack.decode(reply_parts[1])
|
||||
rtt = recv_time - start_time
|
||||
if rtt < best_rtt:
|
||||
best_rtt = rtt
|
||||
best_offset = remote_perf - (start_time + recv_time) / 2
|
||||
|
||||
# Decode handshake payload to get compatibility hash
|
||||
handshake_decoder = msgspec.msgpack.Decoder(NixlHandshakePayload)
|
||||
@@ -681,7 +705,9 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
remote_ranks = (remote_pp_rank, remote_rank)
|
||||
remote_rank_to_agent_name[remote_ranks] = remote_agent_name
|
||||
return remote_rank_to_agent_name
|
||||
|
||||
assert best_offset is not None
|
||||
return remote_rank_to_agent_name, best_offset
|
||||
|
||||
def _add_notif_only_remote_agent(
|
||||
self, metadata: NixlAgentMetadata, remote_tp_size: int
|
||||
@@ -818,7 +844,7 @@ class NixlBaseConnectorWorker:
|
||||
tp_size: int,
|
||||
pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> Future[dict[tuple[int, int], str]] | None:
|
||||
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
|
||||
"""
|
||||
Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
|
||||
@@ -846,11 +872,16 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
self._handshake_futures[engine_id] = fut
|
||||
|
||||
def done_callback(f: Future[dict[tuple[int, int], str]], eid=engine_id):
|
||||
def done_callback(
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
eid=engine_id,
|
||||
):
|
||||
with self._handshake_lock:
|
||||
del self._handshake_futures[eid]
|
||||
try:
|
||||
self._remote_agents[eid] = f.result()
|
||||
remote_agents, clock_offset = f.result()
|
||||
self._remote_agents[eid] = remote_agents
|
||||
self._engine_clock_offset[eid] = clock_offset
|
||||
self._engine_last_active[eid] = time.perf_counter()
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
@@ -2421,6 +2452,8 @@ class NixlBaseConnectorWorker:
|
||||
if self.transfer_topo is not None:
|
||||
self.transfer_topo.unregister_remote_engine(engine_id)
|
||||
|
||||
# Drop the cached clock offset; it is re-measured on the next handshake.
|
||||
self._engine_clock_offset.pop(engine_id, None)
|
||||
# Push P-side engines are tracked in _remote_agents but not in
|
||||
# _engine_last_active (they don't participate in stale eviction), so
|
||||
# tolerate a missing entry.
|
||||
|
||||
@@ -39,8 +39,10 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 2: Add remote_request_id to kv_transfer_params
|
||||
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
|
||||
# 4: Add KV block lease renewal through heartbeats
|
||||
# 5: Add remote_blocks_expiry_time to kv_transfer_params + handshake
|
||||
# clock-sync timestamp
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 4
|
||||
NIXL_CONNECTOR_VERSION: int = 5
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -157,6 +159,7 @@ class RemoteMeta:
|
||||
port: int
|
||||
engine_id: str
|
||||
request_id: str
|
||||
blocks_expiry_time: float | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -225,5 +228,6 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
request_id=kv_transfer_params["remote_request_id"],
|
||||
host=kv_transfer_params["remote_host"],
|
||||
port=kv_transfer_params["remote_port"],
|
||||
blocks_expiry_time=kv_transfer_params.get("remote_blocks_expiry_time"),
|
||||
)
|
||||
self.reqs_to_recv[request_id] = req
|
||||
|
||||
@@ -238,6 +238,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
# remove the conditional below
|
||||
delay_free_blocks = any(len(group) > 0 for group in block_ids)
|
||||
remote_num_tokens = 0
|
||||
blocks_expiry_time = None
|
||||
if delay_free_blocks:
|
||||
# Prefill request on remote. It will be read from D upon completion
|
||||
request_kv_blocks_ttl = self._kv_lease_duration
|
||||
@@ -254,6 +255,9 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
self._reqs_need_send[request.request_id] = (
|
||||
time.perf_counter() + request_kv_blocks_ttl
|
||||
)
|
||||
if is_d_node:
|
||||
# D blocks expiry time exported for the turn-2 readback.
|
||||
blocks_expiry_time = self._reqs_need_send[request.request_id]
|
||||
# NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones),
|
||||
# trimming down after allocating for the whole sequence length. Empty
|
||||
# blocks are always at the start of the list.
|
||||
@@ -272,4 +276,5 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
remote_port=self.side_channel_port,
|
||||
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
|
||||
remote_num_tokens=remote_num_tokens,
|
||||
remote_blocks_expiry_time=blocks_expiry_time,
|
||||
)
|
||||
|
||||
@@ -25,6 +25,10 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Slack (seconds) subtracted from D's exported block-expiry deadline on the turn-2
|
||||
# readback, absorbing clock-offset error and read latency.
|
||||
_KV_BLOCKS_EXPIRY_SAFETY_MARGIN = 5.0
|
||||
|
||||
|
||||
class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""Pull-specific (READ) worker logic."""
|
||||
@@ -98,12 +102,34 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
# requests sit in the D scheduler WAITING queue.
|
||||
self._send_heartbeats(metadata)
|
||||
|
||||
def _is_turn2_read_expired(self, meta: ReqMeta) -> bool:
|
||||
"""Whether D's cached blocks for this turn-2 readback have (nearly) expired."""
|
||||
assert meta.remote is not None
|
||||
blocks_expiry_time = meta.remote.blocks_expiry_time
|
||||
# Deadline may be absent (router may not forward it) -> read as usual.
|
||||
if blocks_expiry_time is None or not meta.local_physical_block_ids:
|
||||
return False
|
||||
clock_offset = self._engine_clock_offset[meta.remote.engine_id]
|
||||
deadline = blocks_expiry_time - clock_offset
|
||||
return time.perf_counter() + _KV_BLOCKS_EXPIRY_SAFETY_MARGIN >= deadline
|
||||
|
||||
def _read_blocks_for_req(self, req_id: str, meta: ReqMeta):
|
||||
assert meta.remote is not None and self.transfer_topo is not None
|
||||
engine_id = meta.remote.engine_id
|
||||
# Update last activity from this remote. Mind that cleanup is done on main
|
||||
# thread (this one), so we don't race on this structure.
|
||||
self._engine_last_active[engine_id] = time.perf_counter()
|
||||
|
||||
if self._bidirectional_kv_xfer_enabled and self._is_turn2_read_expired(meta):
|
||||
logger.warning(
|
||||
"Declining expired remote read for %s from engine %s.",
|
||||
req_id,
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
self._handle_failed_transfer(req_id, None)
|
||||
return
|
||||
|
||||
plan = self.tp_mappings[engine_id]
|
||||
remote_info = self.transfer_topo.get_engine_info(engine_id)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
|
||||
|
||||
@@ -294,7 +294,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
return
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[dict[tuple[int, int], str]],
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
rid: str = req_id,
|
||||
rd: dict[str, Any] = reg_data,
|
||||
) -> None:
|
||||
@@ -456,7 +456,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
if decode_engine_id in self._remote_agents:
|
||||
return True
|
||||
try:
|
||||
remote_agents = self._nixl_handshake(
|
||||
remote_agents, _ = self._nixl_handshake(
|
||||
decode_host,
|
||||
decode_port,
|
||||
decode_tp_size,
|
||||
|
||||
Reference in New Issue
Block a user