[grpc] Support gRPC server entrypoint (#30190)
Signed-off-by: Chang Su <chang.s.su@oracle.com> Signed-off-by: njhill <nickhill123@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: njhill <nickhill123@gmail.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
co-authored by
njhill
Simon Mo
parent
be6a81f31b
commit
791b2fc30a
Executable
+531
@@ -0,0 +1,531 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# mypy: ignore-errors
|
||||
"""
|
||||
vLLM gRPC Server
|
||||
|
||||
Starts a gRPC server for vLLM using the VllmEngine protocol.
|
||||
|
||||
Usage:
|
||||
python -m vllm.entrypoints.grpc_server --model <model_path>
|
||||
|
||||
Example:
|
||||
python -m vllm.entrypoints.grpc_server \
|
||||
--model meta-llama/Llama-2-7b-hf \
|
||||
--host 0.0.0.0 \
|
||||
--port 50051
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
import grpc
|
||||
import uvloop
|
||||
from grpc_reflection.v1alpha import reflection
|
||||
|
||||
from vllm import SamplingParams, TextPrompt, TokensPrompt
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.grpc import vllm_engine_pb2, vllm_engine_pb2_grpc
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import RequestOutputKind, StructuredOutputsParams
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmEngineServicer(vllm_engine_pb2_grpc.VllmEngineServicer):
|
||||
"""
|
||||
gRPC servicer implementing the VllmEngine service.
|
||||
|
||||
Handles 6 RPCs:
|
||||
- Generate: Streaming text generation
|
||||
- Embed: Embeddings (TODO)
|
||||
- HealthCheck: Health probe
|
||||
- Abort: Cancel requests out-of-band
|
||||
- GetModelInfo: Model metadata
|
||||
- GetServerInfo: Server state
|
||||
"""
|
||||
|
||||
def __init__(self, async_llm: AsyncLLM, start_time: float):
|
||||
"""
|
||||
Initialize the servicer.
|
||||
|
||||
Args:
|
||||
async_llm: The AsyncLLM instance
|
||||
start_time: The server start time, in seconds since epoch
|
||||
"""
|
||||
self.async_llm = async_llm
|
||||
self.start_time = start_time
|
||||
logger.info("VllmEngineServicer initialized")
|
||||
|
||||
async def Generate(
|
||||
self,
|
||||
request: vllm_engine_pb2.GenerateRequest,
|
||||
context: grpc.aio.ServicerContext,
|
||||
) -> AsyncGenerator[vllm_engine_pb2.GenerateResponse, None]:
|
||||
"""
|
||||
Handle streaming generation requests.
|
||||
|
||||
Args:
|
||||
request: The GenerateRequest protobuf
|
||||
context: gRPC context
|
||||
|
||||
Yields:
|
||||
GenerateResponse protobuf messages (streaming)
|
||||
"""
|
||||
request_id = request.request_id
|
||||
logger.debug("Generate request %s received.", request_id)
|
||||
|
||||
try:
|
||||
# Extract tokenized input
|
||||
if request.WhichOneof("input") == "tokenized":
|
||||
prompt: TokensPrompt = {
|
||||
"prompt_token_ids": list(request.tokenized.input_ids)
|
||||
}
|
||||
if request.tokenized.original_text:
|
||||
prompt["prompt"] = request.tokenized.original_text
|
||||
else:
|
||||
prompt: TextPrompt = {"prompt": request.text}
|
||||
|
||||
# Build sampling params with detokenize=False
|
||||
sampling_params = self._sampling_params_from_proto(
|
||||
request.sampling_params, stream=request.stream
|
||||
)
|
||||
|
||||
async for output in self.async_llm.generate(
|
||||
prompt=prompt,
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
):
|
||||
# Convert vLLM output to protobuf
|
||||
# For streaming, always send chunks
|
||||
if request.stream:
|
||||
yield self._chunk_response(output)
|
||||
|
||||
# Send complete response when finished
|
||||
if output.finished:
|
||||
yield self._complete_response(output)
|
||||
|
||||
except ValueError as e:
|
||||
# Invalid request error (equiv to 400).
|
||||
await context.abort(grpc.StatusCode.INVALID_ARGUMENT, str(e))
|
||||
except Exception as e:
|
||||
logger.exception("Error in Generate for request %s", request_id)
|
||||
await context.abort(grpc.StatusCode.INTERNAL, str(e))
|
||||
|
||||
async def Embed(
|
||||
self,
|
||||
request: vllm_engine_pb2.EmbedRequest,
|
||||
context: grpc.aio.ServicerContext,
|
||||
) -> vllm_engine_pb2.EmbedResponse:
|
||||
"""
|
||||
Handle embedding requests.
|
||||
|
||||
TODO: Implement in Phase 4
|
||||
|
||||
Args:
|
||||
request: The EmbedRequest protobuf
|
||||
context: gRPC context
|
||||
|
||||
Returns:
|
||||
EmbedResponse protobuf
|
||||
"""
|
||||
logger.warning("Embed RPC not yet implemented")
|
||||
await context.abort(
|
||||
grpc.StatusCode.UNIMPLEMENTED, "Embed RPC not yet implemented"
|
||||
)
|
||||
|
||||
async def HealthCheck(
|
||||
self,
|
||||
request: vllm_engine_pb2.HealthCheckRequest,
|
||||
context: grpc.aio.ServicerContext,
|
||||
) -> vllm_engine_pb2.HealthCheckResponse:
|
||||
"""
|
||||
Handle health check requests.
|
||||
|
||||
Args:
|
||||
request: The HealthCheckRequest protobuf
|
||||
context: gRPC context
|
||||
|
||||
Returns:
|
||||
HealthCheckResponse protobuf
|
||||
"""
|
||||
is_healthy = not self.async_llm.errored
|
||||
message = "Health" if is_healthy else "Engine is not alive"
|
||||
|
||||
logger.debug("HealthCheck request: healthy=%s, message=%s", is_healthy, message)
|
||||
|
||||
return vllm_engine_pb2.HealthCheckResponse(healthy=is_healthy, message=message)
|
||||
|
||||
async def Abort(
|
||||
self,
|
||||
request: vllm_engine_pb2.AbortRequest,
|
||||
context: grpc.aio.ServicerContext,
|
||||
) -> vllm_engine_pb2.AbortResponse:
|
||||
"""
|
||||
Out-of-band abort requests.
|
||||
|
||||
Args:
|
||||
request: The AbortRequest protobuf
|
||||
context: gRPC context
|
||||
|
||||
Returns:
|
||||
AbortResponse protobuf
|
||||
"""
|
||||
request_ids = request.request_ids
|
||||
logger.debug("Abort requests: %s", request_ids)
|
||||
|
||||
await self.async_llm.abort(request_ids)
|
||||
return vllm_engine_pb2.AbortResponse()
|
||||
|
||||
async def GetModelInfo(
|
||||
self,
|
||||
request: vllm_engine_pb2.GetModelInfoRequest,
|
||||
context: grpc.aio.ServicerContext,
|
||||
) -> vllm_engine_pb2.GetModelInfoResponse:
|
||||
"""
|
||||
Handle model info requests.
|
||||
|
||||
Args:
|
||||
request: The GetModelInfoRequest protobuf
|
||||
context: gRPC context
|
||||
|
||||
Returns:
|
||||
GetModelInfoResponse protobuf
|
||||
"""
|
||||
model_config = self.async_llm.model_config
|
||||
|
||||
return vllm_engine_pb2.GetModelInfoResponse(
|
||||
model_path=model_config.model,
|
||||
is_generation=model_config.runner_type == "generate",
|
||||
max_context_length=model_config.max_model_len,
|
||||
vocab_size=model_config.get_vocab_size(),
|
||||
supports_vision=model_config.is_multimodal_model,
|
||||
)
|
||||
|
||||
async def GetServerInfo(
|
||||
self,
|
||||
request: vllm_engine_pb2.GetServerInfoRequest,
|
||||
context: grpc.aio.ServicerContext,
|
||||
) -> vllm_engine_pb2.GetServerInfoResponse:
|
||||
"""
|
||||
Handle server info requests.
|
||||
|
||||
Args:
|
||||
request: The GetServerInfoRequest protobuf
|
||||
context: gRPC context
|
||||
|
||||
Returns:
|
||||
GetServerInfoResponse protobuf
|
||||
"""
|
||||
num_requests = self.async_llm.output_processor.get_num_unfinished_requests()
|
||||
|
||||
return vllm_engine_pb2.GetServerInfoResponse(
|
||||
active_requests=num_requests,
|
||||
is_paused=False, # TODO
|
||||
last_receive_timestamp=time.time(), # TODO looks wrong?
|
||||
uptime_seconds=time.time() - self.start_time,
|
||||
server_type="vllm-grpc",
|
||||
)
|
||||
|
||||
# ========== Helper methods ==========
|
||||
|
||||
@staticmethod
|
||||
def _sampling_params_from_proto(
|
||||
params: vllm_engine_pb2.SamplingParams, stream: bool = True
|
||||
) -> SamplingParams:
|
||||
"""
|
||||
Convert protobuf SamplingParams to vLLM SamplingParams.
|
||||
|
||||
Args:
|
||||
params: Protobuf SamplingParams message
|
||||
stream: Whether streaming is enabled
|
||||
|
||||
Returns:
|
||||
vLLM SamplingParams with detokenize=False and structured_outputs
|
||||
"""
|
||||
# Build stop sequences
|
||||
stop = list(params.stop) if params.stop else None
|
||||
stop_token_ids = list(params.stop_token_ids) if params.stop_token_ids else None
|
||||
|
||||
# Handle structured outputs constraints
|
||||
structured_outputs = None
|
||||
constraint_field = params.WhichOneof("constraint")
|
||||
if constraint_field:
|
||||
if constraint_field == "json_schema":
|
||||
structured_outputs = StructuredOutputsParams(json=params.json_schema)
|
||||
elif constraint_field == "regex":
|
||||
structured_outputs = StructuredOutputsParams(regex=params.regex)
|
||||
elif constraint_field == "grammar":
|
||||
structured_outputs = StructuredOutputsParams(grammar=params.grammar)
|
||||
elif constraint_field == "structural_tag":
|
||||
structured_outputs = StructuredOutputsParams(
|
||||
structural_tag=params.structural_tag
|
||||
)
|
||||
elif constraint_field == "json_object":
|
||||
structured_outputs = StructuredOutputsParams(
|
||||
json_object=params.json_object
|
||||
)
|
||||
elif constraint_field == "choice":
|
||||
structured_outputs = StructuredOutputsParams(
|
||||
choice=list(params.choice.choices)
|
||||
)
|
||||
|
||||
# Create SamplingParams
|
||||
# output_kind=DELTA: Return only new tokens in each chunk (for streaming)
|
||||
return SamplingParams(
|
||||
temperature=params.temperature if params.HasField("temperature") else 1.0,
|
||||
top_p=params.top_p if params.top_p != 0.0 else 1.0,
|
||||
top_k=params.top_k,
|
||||
min_p=params.min_p,
|
||||
frequency_penalty=params.frequency_penalty,
|
||||
presence_penalty=params.presence_penalty,
|
||||
repetition_penalty=params.repetition_penalty
|
||||
if params.repetition_penalty != 0.0
|
||||
else 1.0,
|
||||
max_tokens=params.max_tokens if params.HasField("max_tokens") else None,
|
||||
min_tokens=params.min_tokens,
|
||||
stop=stop,
|
||||
stop_token_ids=stop_token_ids,
|
||||
skip_special_tokens=params.skip_special_tokens,
|
||||
spaces_between_special_tokens=params.spaces_between_special_tokens,
|
||||
ignore_eos=params.ignore_eos,
|
||||
n=params.n if params.n > 0 else 1,
|
||||
logprobs=params.logprobs if params.HasField("logprobs") else None,
|
||||
prompt_logprobs=params.prompt_logprobs
|
||||
if params.HasField("prompt_logprobs")
|
||||
else None,
|
||||
seed=params.seed if params.HasField("seed") else None,
|
||||
include_stop_str_in_output=params.include_stop_str_in_output,
|
||||
logit_bias=dict(params.logit_bias) if params.logit_bias else None,
|
||||
truncate_prompt_tokens=params.truncate_prompt_tokens
|
||||
if params.HasField("truncate_prompt_tokens")
|
||||
else None,
|
||||
structured_outputs=structured_outputs,
|
||||
# detokenize must be True if stop strings are used
|
||||
detokenize=bool(stop),
|
||||
output_kind=RequestOutputKind.DELTA
|
||||
if stream
|
||||
else RequestOutputKind.FINAL_ONLY,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _chunk_response(output: RequestOutput) -> vllm_engine_pb2.GenerateResponse:
|
||||
"""
|
||||
Build a streaming chunk response from vLLM output.
|
||||
When output_kind=DELTA, vLLM returns only new tokens automatically.
|
||||
|
||||
Args:
|
||||
output: vLLM RequestOutput (with delta tokens when output_kind=DELTA)
|
||||
|
||||
Returns:
|
||||
GenerateResponse with chunk field set
|
||||
"""
|
||||
# Get the completion output (first one if n > 1)
|
||||
completion = output.outputs[0] if output.outputs else None
|
||||
|
||||
if completion is None:
|
||||
# Empty chunk
|
||||
return vllm_engine_pb2.GenerateResponse(
|
||||
chunk=vllm_engine_pb2.GenerateStreamChunk(
|
||||
token_ids=[],
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
cached_tokens=0,
|
||||
),
|
||||
)
|
||||
|
||||
# When output_kind=DELTA, completion.token_ids contains only new tokens
|
||||
# vLLM handles the delta logic internally
|
||||
# completion_tokens = delta count (client will accumulate)
|
||||
return vllm_engine_pb2.GenerateResponse(
|
||||
chunk=vllm_engine_pb2.GenerateStreamChunk(
|
||||
token_ids=completion.token_ids,
|
||||
prompt_tokens=len(output.prompt_token_ids)
|
||||
if output.prompt_token_ids
|
||||
else 0,
|
||||
completion_tokens=len(completion.token_ids), # Delta count
|
||||
cached_tokens=output.num_cached_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _complete_response(output: RequestOutput) -> vllm_engine_pb2.GenerateResponse:
|
||||
"""
|
||||
Build a final completion response from vLLM output.
|
||||
|
||||
Args:
|
||||
output: vLLM RequestOutput (finished=True)
|
||||
|
||||
Returns:
|
||||
GenerateResponse with complete field set
|
||||
"""
|
||||
# Get the completion output (first one if n > 1)
|
||||
completion = output.outputs[0] if output.outputs else None
|
||||
|
||||
if completion is None:
|
||||
# Empty completion
|
||||
return vllm_engine_pb2.GenerateResponse(
|
||||
complete=vllm_engine_pb2.GenerateComplete(
|
||||
output_ids=[],
|
||||
finish_reason="error",
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
cached_tokens=0,
|
||||
),
|
||||
)
|
||||
|
||||
# Build complete response
|
||||
# When streaming (DELTA mode): completion.token_ids will be empty/last delta
|
||||
# When non-streaming (FINAL_ONLY mode): completion.token_ids has all tokens
|
||||
# Client will accumulate token counts for streaming
|
||||
return vllm_engine_pb2.GenerateResponse(
|
||||
complete=vllm_engine_pb2.GenerateComplete(
|
||||
output_ids=completion.token_ids,
|
||||
finish_reason=completion.finish_reason or "stop",
|
||||
prompt_tokens=len(output.prompt_token_ids)
|
||||
if output.prompt_token_ids
|
||||
else 0,
|
||||
completion_tokens=len(completion.token_ids),
|
||||
cached_tokens=output.num_cached_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def serve_grpc(args: argparse.Namespace):
|
||||
"""
|
||||
Main serving function.
|
||||
|
||||
Args:
|
||||
args: Parsed command line arguments
|
||||
"""
|
||||
logger.info("vLLM gRPC server version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
# Create engine args
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
|
||||
# Build vLLM config
|
||||
vllm_config = engine_args.create_engine_config(
|
||||
usage_context=UsageContext.OPENAI_API_SERVER
|
||||
)
|
||||
|
||||
# Create AsyncLLM
|
||||
async_llm = AsyncLLM.from_vllm_config(
|
||||
vllm_config=vllm_config,
|
||||
usage_context=UsageContext.OPENAI_API_SERVER,
|
||||
enable_log_requests=args.enable_log_requests,
|
||||
disable_log_stats=args.disable_log_stats_server,
|
||||
)
|
||||
|
||||
# Create servicer
|
||||
servicer = VllmEngineServicer(async_llm, start_time)
|
||||
|
||||
# Create gRPC server
|
||||
server = grpc.aio.server(
|
||||
options=[
|
||||
("grpc.max_send_message_length", -1),
|
||||
("grpc.max_receive_message_length", -1),
|
||||
],
|
||||
)
|
||||
|
||||
# Add servicer to server
|
||||
vllm_engine_pb2_grpc.add_VllmEngineServicer_to_server(servicer, server)
|
||||
|
||||
# Enable reflection for grpcurl and other tools
|
||||
service_names = (
|
||||
vllm_engine_pb2.DESCRIPTOR.services_by_name["VllmEngine"].full_name,
|
||||
reflection.SERVICE_NAME,
|
||||
)
|
||||
reflection.enable_server_reflection(service_names, server)
|
||||
|
||||
# Bind to address
|
||||
address = f"{args.host}:{args.port}"
|
||||
server.add_insecure_port(address)
|
||||
|
||||
# Start server
|
||||
await server.start()
|
||||
logger.info("vLLM gRPC server started on %s", address)
|
||||
logger.info("Server is ready to accept requests")
|
||||
|
||||
# Handle shutdown signals
|
||||
loop = asyncio.get_running_loop()
|
||||
stop_event = asyncio.Event()
|
||||
|
||||
def signal_handler():
|
||||
logger.info("Received shutdown signal")
|
||||
stop_event.set()
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, signal_handler)
|
||||
|
||||
# Serve until shutdown signal
|
||||
try:
|
||||
await stop_event.wait()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Interrupted by user")
|
||||
finally:
|
||||
logger.info("Shutting down vLLM gRPC server...")
|
||||
|
||||
# Stop gRPC server
|
||||
await server.stop(grace=5.0)
|
||||
logger.info("gRPC server stopped")
|
||||
|
||||
# Shutdown AsyncLLM
|
||||
async_llm.shutdown()
|
||||
logger.info("AsyncLLM engine stopped")
|
||||
|
||||
logger.info("Shutdown complete")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point."""
|
||||
parser = FlexibleArgumentParser(
|
||||
description="vLLM gRPC Server",
|
||||
)
|
||||
|
||||
# Server args
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Host to bind gRPC server to",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=50051,
|
||||
help="Port to bind gRPC server to",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-log-stats-server",
|
||||
action="store_true",
|
||||
help="Disable stats logging on server side",
|
||||
)
|
||||
|
||||
# Add vLLM engine args
|
||||
parser = AsyncEngineArgs.add_cli_args(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Run server
|
||||
try:
|
||||
uvloop.run(serve_grpc(args))
|
||||
except Exception as e:
|
||||
logger.exception("Server failed: %s", e)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
vLLM gRPC protocol definitions.
|
||||
|
||||
This module contains the protocol buffer definitions for vLLM's gRPC API.
|
||||
The protobuf files are compiled into Python code using grpcio-tools.
|
||||
"""
|
||||
|
||||
# These imports will be available after protobuf compilation
|
||||
# from vllm.grpc import vllm_engine_pb2
|
||||
# from vllm.grpc import vllm_engine_pb2_grpc
|
||||
|
||||
__all__ = [
|
||||
"vllm_engine_pb2",
|
||||
"vllm_engine_pb2_grpc",
|
||||
]
|
||||
Executable
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Compile vLLM protobuf definitions into Python code.
|
||||
|
||||
This script uses grpcio-tools to generate *_pb2.py, *_pb2_grpc.py, and
|
||||
*_pb2.pyi (type stubs) files from the vllm_engine.proto definition.
|
||||
|
||||
NOTE: Proto compilation happens automatically during package build (via setup.py).
|
||||
This script is provided for developers who want to regenerate protos manually,
|
||||
e.g., after modifying vllm_engine.proto.
|
||||
|
||||
Usage:
|
||||
python vllm/grpc/compile_protos.py
|
||||
|
||||
Requirements:
|
||||
pip install grpcio-tools
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def compile_protos():
|
||||
"""Compile protobuf definitions."""
|
||||
# Get the vllm package root directory
|
||||
script_dir = Path(__file__).parent
|
||||
vllm_package_root = script_dir.parent.parent # vllm/vllm/grpc -> vllm/
|
||||
|
||||
proto_file = script_dir / "vllm_engine.proto"
|
||||
|
||||
if not proto_file.exists():
|
||||
print(f"Error: Proto file not found at {proto_file}")
|
||||
return 1
|
||||
|
||||
print(f"Compiling protobuf: {proto_file}")
|
||||
print(f"Output directory: {script_dir}")
|
||||
|
||||
# Compile the proto file
|
||||
# We use vllm/vllm as the proto_path so that the package is vllm.grpc.engine
|
||||
try:
|
||||
from grpc_tools import protoc
|
||||
|
||||
result = protoc.main(
|
||||
[
|
||||
"grpc_tools.protoc",
|
||||
f"--proto_path={vllm_package_root}",
|
||||
f"--python_out={vllm_package_root}",
|
||||
f"--grpc_python_out={vllm_package_root}",
|
||||
f"--pyi_out={vllm_package_root}", # Generate type stubs
|
||||
str(script_dir / "vllm_engine.proto"),
|
||||
]
|
||||
)
|
||||
|
||||
if result == 0:
|
||||
# Add SPDX headers to generated files
|
||||
spdx_header = (
|
||||
"# SPDX-License-Identifier: Apache-2.0\n"
|
||||
"# SPDX-FileCopyrightText: Copyright contributors to the vLLM project\n"
|
||||
)
|
||||
|
||||
for generated_file in [
|
||||
script_dir / "vllm_engine_pb2.py",
|
||||
script_dir / "vllm_engine_pb2_grpc.py",
|
||||
script_dir / "vllm_engine_pb2.pyi",
|
||||
]:
|
||||
if generated_file.exists():
|
||||
content = generated_file.read_text()
|
||||
if not content.startswith("# SPDX-License-Identifier"):
|
||||
# Add mypy ignore-errors comment for all generated files
|
||||
header = spdx_header + "# mypy: ignore-errors\n"
|
||||
generated_file.write_text(header + content)
|
||||
|
||||
print("✓ Protobuf compilation successful!")
|
||||
print(f" Generated: {script_dir / 'vllm_engine_pb2.py'}")
|
||||
print(f" Generated: {script_dir / 'vllm_engine_pb2_grpc.py'}")
|
||||
print(f" Generated: {script_dir / 'vllm_engine_pb2.pyi'} (type stubs)")
|
||||
return 0
|
||||
else:
|
||||
print(f"Error: protoc returned {result}")
|
||||
return result
|
||||
|
||||
except ImportError:
|
||||
print("Error: grpcio-tools not installed")
|
||||
print("Install with: pip install grpcio-tools")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"Error during compilation: {e}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(compile_protos())
|
||||
@@ -0,0 +1,195 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package vllm.grpc.engine;
|
||||
|
||||
// Service definition for vLLM engine communication
|
||||
// This protocol is designed for efficient binary communication between
|
||||
// the Rust router and vLLM Python engine (AsyncLLM).
|
||||
service VllmEngine {
|
||||
// Submit a generation request (supports streaming)
|
||||
rpc Generate(GenerateRequest) returns (stream GenerateResponse);
|
||||
|
||||
// Submit an embedding request
|
||||
rpc Embed(EmbedRequest) returns (EmbedResponse);
|
||||
|
||||
// Health check
|
||||
rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse);
|
||||
|
||||
// Abort a running request
|
||||
rpc Abort(AbortRequest) returns (AbortResponse);
|
||||
|
||||
// Get model information
|
||||
rpc GetModelInfo(GetModelInfoRequest) returns (GetModelInfoResponse);
|
||||
|
||||
// Get server information
|
||||
rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse);
|
||||
}
|
||||
|
||||
// =====================
|
||||
// Common Types
|
||||
// =====================
|
||||
|
||||
// Sampling parameters for text generation
|
||||
message SamplingParams {
|
||||
optional float temperature = 1;
|
||||
float top_p = 2;
|
||||
uint32 top_k = 3;
|
||||
float min_p = 4;
|
||||
float frequency_penalty = 5;
|
||||
float presence_penalty = 6;
|
||||
float repetition_penalty = 7;
|
||||
|
||||
optional uint32 max_tokens = 8;
|
||||
uint32 min_tokens = 9;
|
||||
|
||||
repeated string stop = 10;
|
||||
repeated uint32 stop_token_ids = 11;
|
||||
|
||||
bool skip_special_tokens = 12;
|
||||
bool spaces_between_special_tokens = 13;
|
||||
bool ignore_eos = 14;
|
||||
|
||||
uint32 n = 15; // Number of parallel samples
|
||||
|
||||
// Logprobs configuration
|
||||
optional int32 logprobs = 22; // Number of log probabilities per output token (-1 for all)
|
||||
optional int32 prompt_logprobs = 23; // Number of log probabilities per prompt token (-1 for all)
|
||||
|
||||
// Additional vLLM fields
|
||||
optional int32 seed = 24; // Random seed for reproducibility
|
||||
bool include_stop_str_in_output = 25; // Whether to include stop strings in output
|
||||
map<int32, float> logit_bias = 26; // Token ID to bias mapping (-100 to 100)
|
||||
optional int32 truncate_prompt_tokens = 27; // Prompt truncation (-1 for model max)
|
||||
|
||||
// Structured outputs (one of) - matches vLLM's StructuredOutputsParams
|
||||
oneof constraint {
|
||||
string json_schema = 16; // JSON schema for structured output
|
||||
string regex = 17; // Regex pattern
|
||||
string grammar = 18; // Grammar/EBNF for structured output
|
||||
string structural_tag = 19; // Structural tag (e.g., Harmony models)
|
||||
bool json_object = 20; // Force JSON object output
|
||||
ChoiceConstraint choice = 21; // List of allowed choices
|
||||
}
|
||||
}
|
||||
|
||||
// Choice constraint for structured outputs
|
||||
message ChoiceConstraint {
|
||||
repeated string choices = 1;
|
||||
}
|
||||
|
||||
// Pre-tokenized input from Rust router
|
||||
message TokenizedInput {
|
||||
string original_text = 1; // For reference/debugging
|
||||
repeated uint32 input_ids = 2; // Actual token IDs to process
|
||||
}
|
||||
|
||||
// =====================
|
||||
// Generate Request
|
||||
// =====================
|
||||
|
||||
message GenerateRequest {
|
||||
string request_id = 1;
|
||||
|
||||
// Prompt input
|
||||
oneof input {
|
||||
TokenizedInput tokenized = 2;
|
||||
string text = 3;
|
||||
}
|
||||
|
||||
// Generation parameters (includes logprobs config)
|
||||
SamplingParams sampling_params = 4;
|
||||
|
||||
// Streaming
|
||||
bool stream = 5;
|
||||
}
|
||||
|
||||
// =====================
|
||||
// Generate Response
|
||||
// =====================
|
||||
|
||||
message GenerateResponse {
|
||||
oneof response {
|
||||
GenerateStreamChunk chunk = 1; // For streaming
|
||||
GenerateComplete complete = 2; // For final/non-streaming
|
||||
}
|
||||
}
|
||||
|
||||
message GenerateStreamChunk {
|
||||
repeated uint32 token_ids = 1; // Incremental tokens
|
||||
uint32 prompt_tokens = 2;
|
||||
uint32 completion_tokens = 3;
|
||||
uint32 cached_tokens = 4;
|
||||
|
||||
// Logprobs support (TODO: implement in Phase 4)
|
||||
// OutputLogProbs output_logprobs = 5;
|
||||
// InputLogProbs input_logprobs = 6; // Only in first chunk
|
||||
}
|
||||
|
||||
message GenerateComplete {
|
||||
repeated uint32 output_ids = 1; // All output tokens
|
||||
string finish_reason = 2; // "stop", "length", "abort"
|
||||
uint32 prompt_tokens = 3;
|
||||
uint32 completion_tokens = 4;
|
||||
uint32 cached_tokens = 5;
|
||||
|
||||
// Logprobs support (TODO: implement in Phase 4)
|
||||
// OutputLogProbs output_logprobs = 6;
|
||||
// InputLogProbs input_logprobs = 7;
|
||||
}
|
||||
|
||||
// =====================
|
||||
// Embedding Request
|
||||
// =====================
|
||||
|
||||
message EmbedRequest {
|
||||
string request_id = 1;
|
||||
TokenizedInput tokenized = 2;
|
||||
}
|
||||
|
||||
message EmbedResponse {
|
||||
repeated float embedding = 1;
|
||||
uint32 prompt_tokens = 2;
|
||||
uint32 embedding_dim = 3;
|
||||
}
|
||||
|
||||
// =====================
|
||||
// Management Operations
|
||||
// =====================
|
||||
|
||||
message HealthCheckRequest {}
|
||||
|
||||
message HealthCheckResponse {
|
||||
bool healthy = 1;
|
||||
string message = 2;
|
||||
}
|
||||
|
||||
message AbortRequest {
|
||||
repeated string request_ids = 1;
|
||||
}
|
||||
|
||||
message AbortResponse {
|
||||
}
|
||||
|
||||
// =====================
|
||||
// Model and Server Info
|
||||
// =====================
|
||||
|
||||
message GetModelInfoRequest {}
|
||||
|
||||
message GetModelInfoResponse {
|
||||
string model_path = 1;
|
||||
bool is_generation = 2;
|
||||
uint32 max_context_length = 3;
|
||||
uint32 vocab_size = 4;
|
||||
bool supports_vision = 5;
|
||||
}
|
||||
|
||||
message GetServerInfoRequest {}
|
||||
|
||||
message GetServerInfoResponse {
|
||||
uint32 active_requests = 1;
|
||||
bool is_paused = 2;
|
||||
double last_receive_timestamp = 3;
|
||||
double uptime_seconds = 4;
|
||||
string server_type = 5; // "vllm-grpc"
|
||||
}
|
||||
Reference in New Issue
Block a user