[2/N] Elastic EP Milestone 2: Integrating NIXL-EP (#35627)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
Co-authored-by: Yongji Wu <wuyongji317@gmail.com>
Co-authored-by: Ron Tourgeman <rtourgeman@nvidia.com>
This commit is contained in:
Itay Alroy
2026-03-13 09:25:33 -04:00
committed by GitHub
co-authored by Yongji Wu Ron Tourgeman
parent 82f836d976
commit d5af196c18
14 changed files with 635 additions and 11 deletions
+15 -3
View File
@@ -544,6 +544,11 @@ class MPClient(EngineCoreClient):
try:
# State used for data parallel.
self.engines_running = False
parallel_config = vllm_config.parallel_config
# Elastic EP can remove a rank and later add it back with the same
# identity. The client input ROUTER needs handover to allow the new
# engine to replace the dead connection.
enable_input_socket_handover = parallel_config.enable_elastic_ep
self.stats_update_address: str | None = None
if client_addresses:
@@ -552,7 +557,11 @@ class MPClient(EngineCoreClient):
output_address = client_addresses["output_address"]
self.stats_update_address = client_addresses.get("stats_update_address")
self.input_socket = self.resources.input_socket = make_zmq_socket(
self.ctx, input_address, zmq.ROUTER, bind=True
self.ctx,
input_address,
zmq.ROUTER,
bind=True,
router_handover=enable_input_socket_handover,
)
self.resources.output_socket = make_zmq_socket(
self.ctx, output_address, zmq.PULL
@@ -561,7 +570,11 @@ class MPClient(EngineCoreClient):
# Engines are managed by this client.
addresses = get_engine_zmq_addresses(vllm_config)
self.input_socket = self.resources.input_socket = make_zmq_socket(
self.ctx, addresses.inputs[0], zmq.ROUTER, bind=True
self.ctx,
addresses.inputs[0],
zmq.ROUTER,
bind=True,
router_handover=enable_input_socket_handover,
)
self.resources.output_socket = make_zmq_socket(
self.ctx, addresses.outputs[0], zmq.PULL
@@ -582,7 +595,6 @@ class MPClient(EngineCoreClient):
coordinator.get_stats_publish_address()
)
parallel_config = vllm_config.parallel_config
dp_size = parallel_config.data_parallel_size
dp_rank = parallel_config.data_parallel_index
dp_local_size = parallel_config.data_parallel_size_local