[Feature] specify model in config.yaml (#15798)
Signed-off-by: weizeng <weizeng@roblox.com>
This commit is contained in:
@@ -4,7 +4,6 @@ import argparse
|
||||
|
||||
import uvloop
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.openai.api_server import run_server
|
||||
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
|
||||
@@ -21,14 +20,9 @@ class ServeSubcommand(CLISubcommand):
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
# The default value of `--model`
|
||||
if args.model != EngineArgs.model:
|
||||
raise ValueError(
|
||||
"With `vllm serve`, you should provide the model as a "
|
||||
"positional argument instead of via the `--model` option.")
|
||||
|
||||
# EngineArgs expects the model name to be passed as --model.
|
||||
args.model = args.model_tag
|
||||
# If model is specified in CLI (as positional arg), it takes precedence
|
||||
if hasattr(args, 'model_tag') and args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
|
||||
uvloop.run(run_server(args))
|
||||
|
||||
@@ -41,10 +35,12 @@ class ServeSubcommand(CLISubcommand):
|
||||
serve_parser = subparsers.add_parser(
|
||||
"serve",
|
||||
help="Start the vLLM OpenAI Compatible API server",
|
||||
usage="vllm serve <model_tag> [options]")
|
||||
usage="vllm serve [model_tag] [options]")
|
||||
serve_parser.add_argument("model_tag",
|
||||
type=str,
|
||||
help="The model tag to serve")
|
||||
nargs='?',
|
||||
help="The model tag to serve "
|
||||
"(optional if specified in config)")
|
||||
serve_parser.add_argument(
|
||||
"--config",
|
||||
type=str,
|
||||
|
||||
+27
-9
@@ -1241,6 +1241,16 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
|
||||
if args is None:
|
||||
args = sys.argv[1:]
|
||||
|
||||
# Check for --model in command line arguments first
|
||||
if args and args[0] == "serve":
|
||||
model_in_cli_args = any(arg == '--model' for arg in args)
|
||||
|
||||
if model_in_cli_args:
|
||||
raise ValueError(
|
||||
"With `vllm serve`, you should provide the model as a "
|
||||
"positional argument or in a config file instead of via "
|
||||
"the `--model` option.")
|
||||
|
||||
if '--config' in args:
|
||||
args = self._pull_args_from_config(args)
|
||||
|
||||
@@ -1324,19 +1334,29 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
|
||||
config_args = self._load_config_file(file_path)
|
||||
|
||||
# 0th index is for {serve,chat,complete}
|
||||
# followed by model_tag (only for serve)
|
||||
# optionally followed by model_tag (only for serve)
|
||||
# followed by config args
|
||||
# followed by rest of cli args.
|
||||
# maintaining this order will enforce the precedence
|
||||
# of cli > config > defaults
|
||||
if args[0] == "serve":
|
||||
if index == 1:
|
||||
model_in_cli = len(args) > 1 and not args[1].startswith('-')
|
||||
model_in_config = any(arg == '--model' for arg in config_args)
|
||||
|
||||
if not model_in_cli and not model_in_config:
|
||||
raise ValueError(
|
||||
"No model_tag specified! Please check your command-line"
|
||||
" arguments.")
|
||||
args = [args[0]] + [
|
||||
args[1]
|
||||
] + config_args + args[2:index] + args[index + 2:]
|
||||
"No model specified! Please specify model either "
|
||||
"as a positional argument or in a config file.")
|
||||
|
||||
if model_in_cli:
|
||||
# Model specified as positional arg, keep CLI version
|
||||
args = [args[0]] + [
|
||||
args[1]
|
||||
] + config_args + args[2:index] + args[index + 2:]
|
||||
else:
|
||||
# No model in CLI, use config if available
|
||||
args = [args[0]
|
||||
] + config_args + args[1:index] + args[index + 2:]
|
||||
else:
|
||||
args = [args[0]] + config_args + args[1:index] + args[index + 2:]
|
||||
|
||||
@@ -1354,9 +1374,7 @@ class FlexibleArgumentParser(argparse.ArgumentParser):
|
||||
'--port': '12323',
|
||||
'--tensor-parallel-size': '4'
|
||||
]
|
||||
|
||||
"""
|
||||
|
||||
extension: str = file_path.split('.')[-1]
|
||||
if extension not in ('yaml', 'yml'):
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user