[V1] Implement vLLM V1 [1/N] (#9289)
This commit is contained in:
@@ -8,7 +8,7 @@ from typing import Iterator, List, Optional, Union
|
||||
import cloudpickle
|
||||
import zmq
|
||||
|
||||
from vllm import AsyncEngineArgs, LLMEngine, SamplingParams
|
||||
from vllm import AsyncEngineArgs, SamplingParams
|
||||
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
|
||||
ParallelConfig, SchedulerConfig)
|
||||
# yapf conflicts with isort for this block
|
||||
@@ -21,12 +21,17 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
|
||||
RPCStartupRequest, RPCStartupResponse,
|
||||
RPCUProfileRequest)
|
||||
# yapf: enable
|
||||
from vllm.envs import VLLM_RPC_TIMEOUT
|
||||
from vllm.envs import VLLM_RPC_TIMEOUT, VLLM_USE_V1
|
||||
from vllm.executor.gpu_executor import GPUExecutor
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
|
||||
if VLLM_USE_V1:
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
else:
|
||||
from vllm.engine.llm_engine import LLMEngine
|
||||
|
||||
CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
|
||||
SchedulerConfig, LoRAConfig]
|
||||
|
||||
@@ -136,14 +141,16 @@ class MQLLMEngine:
|
||||
|
||||
executor_class = LLMEngine._get_executor_cls(engine_config)
|
||||
|
||||
return cls(
|
||||
ipc_path=ipc_path,
|
||||
use_async_sockets=engine_config.model_config.use_async_output_proc,
|
||||
**engine_config.to_dict(),
|
||||
executor_class=executor_class,
|
||||
log_requests=not engine_args.disable_log_requests,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
usage_context=usage_context)
|
||||
use_async_sockets = (engine_config.model_config.use_async_output_proc
|
||||
and not VLLM_USE_V1)
|
||||
|
||||
return cls(ipc_path=ipc_path,
|
||||
use_async_sockets=use_async_sockets,
|
||||
**engine_config.to_dict(),
|
||||
executor_class=executor_class,
|
||||
log_requests=not engine_args.disable_log_requests,
|
||||
log_stats=not engine_args.disable_log_stats,
|
||||
usage_context=usage_context)
|
||||
|
||||
def start(self):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user