[Core] Simplify API server handshake (#39364)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -457,22 +457,6 @@ class ElasticScalingCache:
|
||||
pending_notifications: dict[EEPNotificationType, set[int]]
|
||||
|
||||
|
||||
_ready_response_decoder = msgspec.msgpack.Decoder(EngineCoreReadyResponse)
|
||||
|
||||
|
||||
def _apply_ready_response(payload: bytes, vllm_config: VllmConfig) -> None:
|
||||
"""Decode an EngineCoreReadyResponse and sync any post-initialization
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
|
||||
if not payload:
|
||||
return
|
||||
response = _ready_response_decoder.decode(payload)
|
||||
if response.max_model_len is not None:
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len,
|
||||
response.max_model_len,
|
||||
)
|
||||
|
||||
|
||||
class MPClient(EngineCoreClient):
|
||||
"""
|
||||
MPClient: base client for multi-proc EngineCore.
|
||||
@@ -608,7 +592,7 @@ class MPClient(EngineCoreClient):
|
||||
)
|
||||
identity, payload = sync_input_socket.recv_multipart()
|
||||
identities.remove(identity)
|
||||
_apply_ready_response(payload, vllm_config)
|
||||
self._apply_ready_response(payload)
|
||||
|
||||
self.core_engine: EngineIdentity = self.core_engines[0]
|
||||
self.utility_results: dict[int, AnyFuture] = {}
|
||||
@@ -680,6 +664,34 @@ class MPClient(EngineCoreClient):
|
||||
target=monitor_engine_cores, daemon=True, name="MPClientEngineMonitor"
|
||||
).start()
|
||||
|
||||
def _apply_ready_response(self, payload: bytes) -> None:
|
||||
"""Decode an EngineCoreReadyResponse and sync any post-initialization
|
||||
config changes (e.g. auto-fitted max_model_len) back to the frontend."""
|
||||
if not payload:
|
||||
return
|
||||
vllm_config = self.vllm_config
|
||||
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
if response.max_model_len is not None:
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len,
|
||||
response.max_model_len,
|
||||
)
|
||||
|
||||
# Setup KV cache config with initialization state from
|
||||
# engine core process. Sum values from all engines in DP case.
|
||||
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0
|
||||
num_gpu_blocks += response.num_gpu_blocks
|
||||
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
|
||||
|
||||
# In external DP LB mode, the coordinator address that the
|
||||
# front-end procs connect to is obtained by each engine via it's
|
||||
# initial handshake with the rank 0 front-end.
|
||||
if response.dp_stats_address is not None:
|
||||
if self.stats_update_address is None:
|
||||
self.stats_update_address = response.dp_stats_address
|
||||
else:
|
||||
assert response.dp_stats_address == self.stats_update_address
|
||||
|
||||
|
||||
def _process_utility_output(
|
||||
output: UtilityOutput, utility_results: dict[int, AnyFuture]
|
||||
@@ -1602,7 +1614,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
)
|
||||
identity, payload = sync_input_socket.recv_multipart()
|
||||
new_engine_identities.discard(identity)
|
||||
_apply_ready_response(payload, self.vllm_config)
|
||||
self._apply_ready_response(payload)
|
||||
|
||||
# NOTE(yongji): Before we schedule any requests on the new workers,
|
||||
# we should wait for them to switch to the new setup.
|
||||
|
||||
Reference in New Issue
Block a user