[Core] Change max_model_len in EngineCoreReadyResponse to be non-None (#39442)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -72,9 +72,9 @@ class EngineCoreReadyResponse:
|
||||
values (e.g. max_model_len after KV cache auto-fitting).
|
||||
"""
|
||||
|
||||
max_model_len: int
|
||||
num_gpu_blocks: int
|
||||
dp_stats_address: str | None
|
||||
max_model_len: int | None = None
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
|
||||
@@ -671,11 +671,9 @@ class MPClient(EngineCoreClient):
|
||||
return
|
||||
vllm_config = self.vllm_config
|
||||
response = msgspec.msgpack.decode(payload, type=EngineCoreReadyResponse)
|
||||
if response.max_model_len is not None:
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len,
|
||||
response.max_model_len,
|
||||
)
|
||||
vllm_config.model_config.max_model_len = min(
|
||||
vllm_config.model_config.max_model_len, response.max_model_len
|
||||
)
|
||||
|
||||
# Setup KV cache config with initialization state from
|
||||
# engine core process. Sum values from all engines in DP case.
|
||||
|
||||
Reference in New Issue
Block a user