[2/N] Elastic EP Milestone 2: Integrating NIXL-EP (#35627)
Signed-off-by: Itay Alroy <ialroy@nvidia.com> Co-authored-by: Yongji Wu <wuyongji317@gmail.com> Co-authored-by: Ron Tourgeman <rtourgeman@nvidia.com>
This commit is contained in:
co-authored by
Yongji Wu
Ron Tourgeman
parent
82f836d976
commit
d5af196c18
@@ -544,6 +544,11 @@ class MPClient(EngineCoreClient):
|
||||
try:
|
||||
# State used for data parallel.
|
||||
self.engines_running = False
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# Elastic EP can remove a rank and later add it back with the same
|
||||
# identity. The client input ROUTER needs handover to allow the new
|
||||
# engine to replace the dead connection.
|
||||
enable_input_socket_handover = parallel_config.enable_elastic_ep
|
||||
|
||||
self.stats_update_address: str | None = None
|
||||
if client_addresses:
|
||||
@@ -552,7 +557,11 @@ class MPClient(EngineCoreClient):
|
||||
output_address = client_addresses["output_address"]
|
||||
self.stats_update_address = client_addresses.get("stats_update_address")
|
||||
self.input_socket = self.resources.input_socket = make_zmq_socket(
|
||||
self.ctx, input_address, zmq.ROUTER, bind=True
|
||||
self.ctx,
|
||||
input_address,
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
router_handover=enable_input_socket_handover,
|
||||
)
|
||||
self.resources.output_socket = make_zmq_socket(
|
||||
self.ctx, output_address, zmq.PULL
|
||||
@@ -561,7 +570,11 @@ class MPClient(EngineCoreClient):
|
||||
# Engines are managed by this client.
|
||||
addresses = get_engine_zmq_addresses(vllm_config)
|
||||
self.input_socket = self.resources.input_socket = make_zmq_socket(
|
||||
self.ctx, addresses.inputs[0], zmq.ROUTER, bind=True
|
||||
self.ctx,
|
||||
addresses.inputs[0],
|
||||
zmq.ROUTER,
|
||||
bind=True,
|
||||
router_handover=enable_input_socket_handover,
|
||||
)
|
||||
self.resources.output_socket = make_zmq_socket(
|
||||
self.ctx, addresses.outputs[0], zmq.PULL
|
||||
@@ -582,7 +595,6 @@ class MPClient(EngineCoreClient):
|
||||
coordinator.get_stats_publish_address()
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
dp_size = parallel_config.data_parallel_size
|
||||
dp_rank = parallel_config.data_parallel_index
|
||||
dp_local_size = parallel_config.data_parallel_size_local
|
||||
|
||||
Reference in New Issue
Block a user