[V1] AsyncLLM Implementation (#9826)

Signed-off-by: Nick Hill <nickhill@us.ibm.com>
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Signed-off-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Nick Hill <nickhill@us.ibm.com>
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
Robert Shaw
2024-11-11 23:05:38 +00:00
committed by GitHub
co-authored by Nick Hill Varun Sundar Rabindranath Nick Hill Tyler Michael Smith
parent 08f93e7439
commit 6ace6fba2c
29 changed files with 2409 additions and 724 deletions
+2 -11
View File
@@ -6,7 +6,6 @@ from typing import Iterator, List, Optional, Union
import cloudpickle
import zmq
import vllm.envs
from vllm import AsyncEngineArgs, SamplingParams
from vllm.engine.llm_engine import LLMEngine
# yapf conflicts with isort for this block
@@ -113,17 +112,9 @@ class MQLLMEngine:
load_general_plugins()
engine_config = engine_args.create_engine_config()
if vllm.envs.VLLM_USE_V1:
# Lazy import: the v1 package isn't distributed
from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
engine_class = V1LLMEngine
else:
engine_class = LLMEngine
executor_class = LLMEngine._get_executor_cls(engine_config)
executor_class = engine_class._get_executor_cls(engine_config)
use_async_sockets = (engine_config.model_config.use_async_output_proc
and not vllm.envs.VLLM_USE_V1)
use_async_sockets = engine_config.model_config.use_async_output_proc
return cls(ipc_path=ipc_path,
use_async_sockets=use_async_sockets,