[Feature] vLLM CLI (#5090)

Co-authored-by: simon-mo <simon.mo@hey.com>
This commit is contained in:
Ethan Xu
2024-07-14 15:36:43 -07:00
committed by GitHub
co-authored by simon-mo
parent 73030b7dae
commit dbfe254eda
7 changed files with 223 additions and 36 deletions
+4 -2
View File
@@ -14,7 +14,7 @@ import requests
from vllm.distributed import (ensure_model_parallel_initialized,
init_distributed_environment)
from vllm.entrypoints.openai.cli_args import make_arg_parser
from vllm.utils import get_open_port, is_hip
from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
if is_hip():
from amdsmi import (amdsmi_get_gpu_vram_usage,
@@ -57,7 +57,9 @@ class RemoteOpenAIServer:
cli_args = cli_args + ["--port", str(get_open_port())]
parser = make_arg_parser()
parser = FlexibleArgumentParser(
description="vLLM's remote OpenAI server.")
parser = make_arg_parser(parser)
args = parser.parse_args(cli_args)
self.host = str(args.host or 'localhost')
self.port = int(args.port)