feat(grpc): extract gRPC servicer into smg-grpc-servicer package, add --grpc flag to vllm serve (#36169)
Signed-off-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -51,6 +51,12 @@ class ServeSubcommand(CLISubcommand):
|
||||
if hasattr(args, "model_tag") and args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
|
||||
if getattr(args, "grpc", False):
|
||||
from vllm.entrypoints.grpc_server import serve_grpc
|
||||
|
||||
uvloop.run(serve_grpc(args))
|
||||
return
|
||||
|
||||
if args.headless:
|
||||
if args.api_server_count is not None and args.api_server_count > 0:
|
||||
raise ValueError(
|
||||
@@ -127,6 +133,13 @@ class ServeSubcommand(CLISubcommand):
|
||||
)
|
||||
|
||||
serve_parser = make_arg_parser(serve_parser)
|
||||
serve_parser.add_argument(
|
||||
"--grpc",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Launch a gRPC server instead of the HTTP OpenAI-compatible "
|
||||
"server. Requires: pip install vllm[grpc].",
|
||||
)
|
||||
serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
|
||||
return serve_parser
|
||||
|
||||
|
||||
Reference in New Issue
Block a user