[V1] AsyncLLM Implementation (#9826)
Signed-off-by: Nick Hill <nickhill@us.ibm.com> Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Nick Hill <nhill@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Nick Hill
Varun Sundar Rabindranath
Nick Hill
Tyler Michael Smith
parent
08f93e7439
commit
6ace6fba2c
@@ -6,7 +6,6 @@ from typing import Iterator, List, Optional, Union
|
||||
import cloudpickle
|
||||
import zmq
|
||||
|
||||
import vllm.envs
|
||||
from vllm import AsyncEngineArgs, SamplingParams
|
||||
from vllm.engine.llm_engine import LLMEngine
|
||||
# yapf conflicts with isort for this block
|
||||
@@ -113,17 +112,9 @@ class MQLLMEngine:
|
||||
load_general_plugins()
|
||||
|
||||
engine_config = engine_args.create_engine_config()
|
||||
if vllm.envs.VLLM_USE_V1:
|
||||
# Lazy import: the v1 package isn't distributed
|
||||
from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
|
||||
engine_class = V1LLMEngine
|
||||
else:
|
||||
engine_class = LLMEngine
|
||||
executor_class = LLMEngine._get_executor_cls(engine_config)
|
||||
|
||||
executor_class = engine_class._get_executor_cls(engine_config)
|
||||
|
||||
use_async_sockets = (engine_config.model_config.use_async_output_proc
|
||||
and not vllm.envs.VLLM_USE_V1)
|
||||
use_async_sockets = engine_config.model_config.use_async_output_proc
|
||||
|
||||
return cls(ipc_path=ipc_path,
|
||||
use_async_sockets=use_async_sockets,
|
||||
|
||||
Reference in New Issue
Block a user