[Rust Frontend][gRPC] Add server and model discovery (#49491)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Connor Carpenter
2026-07-27 09:53:27 -07:00
committed by GitHub
co-authored by Nick Hill
parent 2b465b2c42
commit e3c2fc3b3c
18 changed files with 615 additions and 269 deletions
+7
View File
@@ -82,6 +82,13 @@ class EngineCoreReadyResponse:
vllm_version: str
world_size: int
data_parallel_size: int
tensor_parallel_size: int
pipeline_parallel_size: int
decode_context_parallel_size: int
data_parallel_rank: int
max_num_seqs: int
max_num_batched_tokens: int
instance_id: str
# KV cache capacity (None for encoder-only/attention-free models).
kv_cache_size_tokens: int | None = None
kv_cache_max_concurrency: float | None = None
+26 -16
View File
@@ -1613,6 +1613,31 @@ class EngineCoreProc(EngineCore):
"to send. Please report this issue."
)
def _make_ready_response(self) -> EngineCoreReadyResponse:
parallel_config = self.vllm_config.parallel_config
scheduler_config = self.vllm_config.scheduler_config
return EngineCoreReadyResponse(
max_model_len=self.vllm_config.model_config.max_model_len,
num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
block_size=self.vllm_config.cache_config.block_size,
dp_stats_address=self.frontend_stats_publish_address,
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
vllm_version=VLLM_VERSION,
world_size=self.vllm_config.parallel_config.world_size,
data_parallel_size=parallel_config.data_parallel_size,
kv_cache_size_tokens=self.vllm_config.cache_config.kv_cache_size_tokens,
kv_cache_max_concurrency=(
self.vllm_config.cache_config.kv_cache_max_concurrency
),
tensor_parallel_size=parallel_config.tensor_parallel_size,
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
decode_context_parallel_size=parallel_config.decode_context_parallel_size,
data_parallel_rank=self.engine_index,
max_num_seqs=scheduler_config.max_num_seqs,
max_num_batched_tokens=scheduler_config.max_num_batched_tokens,
instance_id=self.vllm_config.instance_id,
)
def process_input_sockets(
self,
input_addresses: list[str],
@@ -1654,22 +1679,7 @@ class EngineCoreProc(EngineCore):
# Register sockets with poller.
poller = zmq.Poller()
ready_response = EngineCoreReadyResponse(
max_model_len=self.vllm_config.model_config.max_model_len,
num_gpu_blocks=self.vllm_config.cache_config.num_gpu_blocks or 0,
block_size=self.vllm_config.cache_config.block_size,
dp_stats_address=self.frontend_stats_publish_address,
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
vllm_version=VLLM_VERSION,
world_size=self.vllm_config.parallel_config.world_size,
data_parallel_size=self.vllm_config.parallel_config.data_parallel_size,
kv_cache_size_tokens=(
self.vllm_config.cache_config.kv_cache_size_tokens
),
kv_cache_max_concurrency=(
self.vllm_config.cache_config.kv_cache_max_concurrency
),
)
ready_response = self._make_ready_response()
ready_payload = msgspec.msgpack.encode(ready_response)
for input_socket in input_sockets:
# Send initial message to each input socket - this is required