[V1][Bugfix]: vllm v1 verison metric num_gpu_blocks is None (#15755)
Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>
This commit is contained in:
@@ -120,7 +120,8 @@ class AsyncLLM(EngineClient):
|
||||
executor_class=executor_class,
|
||||
log_stats=self.log_stats,
|
||||
)
|
||||
|
||||
for stat_logger in self.stat_loggers[0]:
|
||||
stat_logger.log_engine_initialized()
|
||||
self.output_handler: Optional[asyncio.Task] = None
|
||||
try:
|
||||
# Start output handler eagerly if we are in the asyncio eventloop.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
@@ -116,6 +117,7 @@ class EngineCore:
|
||||
logger.info("Batch queue is enabled with size %d",
|
||||
self.batch_queue_size)
|
||||
self.batch_queue = queue.Queue(self.batch_queue_size)
|
||||
self.vllm_config = vllm_config
|
||||
|
||||
def _initialize_kv_caches(
|
||||
self, vllm_config: VllmConfig) -> tuple[int, int, KVCacheConfig]:
|
||||
@@ -507,7 +509,12 @@ class EngineCoreProc(EngineCore):
|
||||
bind=False) as socket:
|
||||
|
||||
# Send ready message to front-end once input socket is connected.
|
||||
socket.send(b'READY')
|
||||
message_dict = {
|
||||
'type': 'READY',
|
||||
'num_gpu_blocks': self.vllm_config.cache_config.num_gpu_blocks,
|
||||
}
|
||||
message = json.dumps(message_dict).encode('utf-8')
|
||||
socket.send(message)
|
||||
|
||||
while True:
|
||||
# (RequestType, RequestData)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import queue
|
||||
import uuid
|
||||
import weakref
|
||||
@@ -362,6 +363,7 @@ class MPClient(EngineCoreClient):
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
# Serialization setup.
|
||||
self.encoder = MsgpackEncoder()
|
||||
self.decoder = MsgpackDecoder(EngineCoreOutputs)
|
||||
@@ -430,14 +432,19 @@ class MPClient(EngineCoreClient):
|
||||
raise RuntimeError("Engine core initialization failed. "
|
||||
"See root cause above.")
|
||||
|
||||
eng_id_bytes, msg = sync_input_socket.recv_multipart()
|
||||
eng_id_bytes, data = sync_input_socket.recv_multipart()
|
||||
eng_id = int.from_bytes(eng_id_bytes, byteorder="little")
|
||||
if eng_id not in identities:
|
||||
raise RuntimeError(f"Unexpected or duplicate engine: {eng_id}")
|
||||
if msg != b'READY':
|
||||
raise RuntimeError(f"Engine {eng_id} failed: {msg.decode()}")
|
||||
message_dict = json.loads(data.decode('utf-8'))
|
||||
if message_dict['type'] != 'READY':
|
||||
raise RuntimeError(f"Engine {eng_id} failed: {data.decode()}")
|
||||
logger.info("Core engine process %d ready.", eng_id)
|
||||
identities.discard(eng_id)
|
||||
# Setup KV cache config with initialization state from
|
||||
# engine core process.
|
||||
self.vllm_config.cache_config.num_gpu_blocks = message_dict[
|
||||
'num_gpu_blocks']
|
||||
|
||||
def _init_core_engines(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user