[Frontend] Delegate to vLLM Omni When --omni Passed (#40744)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
@@ -7,6 +7,7 @@ to avoid certain eager import breakage."""
|
|||||||
|
|
||||||
import importlib.metadata
|
import importlib.metadata
|
||||||
import sys
|
import sys
|
||||||
|
from importlib.util import find_spec
|
||||||
|
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
|
||||||
@@ -34,47 +35,63 @@ def main():
|
|||||||
|
|
||||||
cli_env_setup()
|
cli_env_setup()
|
||||||
|
|
||||||
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
|
# If `--omni` arg is passed to the CLI, delegate to vLLM Omni's entrypoint handling
|
||||||
if len(sys.argv) > 1 and sys.argv[1] == "bench":
|
if "--omni" in sys.argv:
|
||||||
logger.debug(
|
# NOTE: Check the spec instead of importing directly here, since things could
|
||||||
"Bench command detected, must ensure current platform is not "
|
# fail with ImportError due to mismatched versions if things are moved around.
|
||||||
"UnspecifiedPlatform to avoid device type inference error"
|
spec = find_spec("vllm_omni")
|
||||||
)
|
if spec is None:
|
||||||
from vllm import platforms
|
logger.error(
|
||||||
|
"--omni flag requires a valid instance of vllm-omni to be installed."
|
||||||
if platforms.current_platform.is_unspecified():
|
|
||||||
from vllm.platforms.cpu import CpuPlatform
|
|
||||||
|
|
||||||
platforms.current_platform = CpuPlatform()
|
|
||||||
logger.info(
|
|
||||||
"Unspecified platform detected, switching to CPU Platform instead."
|
|
||||||
)
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
parser = FlexibleArgumentParser(
|
from vllm_omni.entrypoints.cli.main import main as omni_main
|
||||||
description="vLLM CLI",
|
|
||||||
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"-v",
|
|
||||||
"--version",
|
|
||||||
action="version",
|
|
||||||
version=importlib.metadata.version("vllm"),
|
|
||||||
)
|
|
||||||
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
|
||||||
cmds = {}
|
|
||||||
for cmd_module in CMD_MODULES:
|
|
||||||
new_cmds = cmd_module.cmd_init()
|
|
||||||
for cmd in new_cmds:
|
|
||||||
cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
|
|
||||||
cmds[cmd.name] = cmd
|
|
||||||
args = parser.parse_args()
|
|
||||||
if args.subparser in cmds:
|
|
||||||
cmds[args.subparser].validate(args)
|
|
||||||
|
|
||||||
if hasattr(args, "dispatch_function"):
|
logger.info("Delegating entrypoint handling to vllm-omni")
|
||||||
args.dispatch_function(args)
|
omni_main()
|
||||||
else:
|
else:
|
||||||
parser.print_help()
|
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
|
||||||
|
if len(sys.argv) > 1 and sys.argv[1] == "bench":
|
||||||
|
logger.debug(
|
||||||
|
"Bench command detected, must ensure current platform is not "
|
||||||
|
"UnspecifiedPlatform to avoid device type inference error"
|
||||||
|
)
|
||||||
|
from vllm import platforms
|
||||||
|
|
||||||
|
if platforms.current_platform.is_unspecified():
|
||||||
|
from vllm.platforms.cpu import CpuPlatform
|
||||||
|
|
||||||
|
platforms.current_platform = CpuPlatform()
|
||||||
|
logger.info(
|
||||||
|
"Unspecified platform detected, switching to CPU Platform instead."
|
||||||
|
)
|
||||||
|
|
||||||
|
parser = FlexibleArgumentParser(
|
||||||
|
description="vLLM CLI",
|
||||||
|
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-v",
|
||||||
|
"--version",
|
||||||
|
action="version",
|
||||||
|
version=importlib.metadata.version("vllm"),
|
||||||
|
)
|
||||||
|
subparsers = parser.add_subparsers(required=False, dest="subparser")
|
||||||
|
cmds = {}
|
||||||
|
for cmd_module in CMD_MODULES:
|
||||||
|
new_cmds = cmd_module.cmd_init()
|
||||||
|
for cmd in new_cmds:
|
||||||
|
cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
|
||||||
|
cmds[cmd.name] = cmd
|
||||||
|
args = parser.parse_args()
|
||||||
|
if args.subparser in cmds:
|
||||||
|
cmds[args.subparser].validate(args)
|
||||||
|
|
||||||
|
if hasattr(args, "dispatch_function"):
|
||||||
|
args.dispatch_function(args)
|
||||||
|
else:
|
||||||
|
parser.print_help()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user