[Core] Simplify API server handshake (#39364)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-04-09 18:56:15 +08:00
committed by GitHub
parent d87fb264df
commit c8d98f81f6
5 changed files with 53 additions and 67 deletions
+30 -18
View File
@@ -457,22 +457,6 @@ class ElasticScalingCache:
pending_notifications: dict[EEPNotificationType, set[int]]
_ready_response_decoder = msgspec.msgpack.Decoder(EngineCoreReadyResponse)
def _apply_ready_response(payload: bytes, vllm_config: VllmConfig) -> None:
"""Decode an EngineCoreReadyResponse and sync any post-initialization
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
if not payload:
return
response = _ready_response_decoder.decode(payload)
if response.max_model_len is not None:
vllm_config.model_config.max_model_len = min(
vllm_config.model_config.max_model_len,
response.max_model_len,
)
class MPClient(EngineCoreClient):
"""
MPClient: base client for multi-proc EngineCore.
@@ -608,7 +592,7 @@ class MPClient(EngineCoreClient):
)
identity, payload = sync_input_socket.recv_multipart()
identities.remove(identity)
_apply_ready_response(payload, vllm_config)
self._apply_ready_response(payload)
self.core_engine: EngineIdentity = self.core_engines[0]
self.utility_results: dict[int, AnyFuture] = {}
@@ -680,6 +664,34 @@ class MPClient(EngineCoreClient):
target=monitor_engine_cores, daemon=True, name="MPClientEngineMonitor"
).start()
def _apply_ready_response(self, payload: bytes) -> None:
"""Decode an EngineCoreReadyResponse and sync any post-initialization
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
if not payload:
return
vllm_config = self.vllm_config
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
if response.max_model_len is not None:
vllm_config.model_config.max_model_len = min(
vllm_config.model_config.max_model_len,
response.max_model_len,
)
# Setup KV cache config with initialization state from
# engine core process. Sum values from all engines in DP case.
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0
num_gpu_blocks += response.num_gpu_blocks
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
# In external DP LB mode, the coordinator address that the
# front-end procs connect to is obtained by each engine via it's
# initial handshake with the rank 0 front-end.
if response.dp_stats_address is not None:
if self.stats_update_address is None:
self.stats_update_address = response.dp_stats_address
else:
assert response.dp_stats_address == self.stats_update_address
def _process_utility_output(
output: UtilityOutput, utility_results: dict[int, AnyFuture]
@@ -1602,7 +1614,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
)
identity, payload = sync_input_socket.recv_multipart()
new_engine_identities.discard(identity)
_apply_ready_response(payload, self.vllm_config)
self._apply_ready_response(payload)
# NOTE(yongji): Before we schedule any requests on the new workers,
# we should wait for them to switch to the new setup.