[V1] V1 Enablement Oracle (#13726)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
co-authored by
rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Nicolò Lucchesi
Tyler Michael Smith
Michael Goin
parent
8c0d15d5c5
commit
d4d93db2c5
+5
-14
@@ -11,7 +11,6 @@ import torch.nn as nn
|
||||
from tqdm import tqdm
|
||||
from typing_extensions import TypeVar, deprecated
|
||||
|
||||
from vllm import envs
|
||||
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
|
||||
BeamSearchSequence, get_beam_search_score)
|
||||
from vllm.config import CompilationConfig
|
||||
@@ -238,23 +237,15 @@ class LLM:
|
||||
compilation_config=compilation_config_instance,
|
||||
**kwargs,
|
||||
)
|
||||
# Logic to switch between engines is done at runtime instead of import
|
||||
# to avoid import order issues
|
||||
self.engine_class = self.get_engine_class()
|
||||
self.llm_engine = self.engine_class.from_engine_args(
|
||||
engine_args, usage_context=UsageContext.LLM_CLASS)
|
||||
|
||||
# Create the Engine (autoselects V0 vs V1)
|
||||
self.llm_engine = LLMEngine.from_engine_args(
|
||||
engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
|
||||
self.engine_class = type(self.llm_engine)
|
||||
|
||||
self.request_counter = Counter()
|
||||
self.default_sampling_params: Union[dict[str, Any], None] = None
|
||||
|
||||
@staticmethod
|
||||
def get_engine_class() -> type[LLMEngine]:
|
||||
if envs.VLLM_USE_V1:
|
||||
# Lazy import: the v1 package isn't distributed
|
||||
from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
|
||||
return V1LLMEngine # type: ignore
|
||||
return LLMEngine
|
||||
|
||||
def get_tokenizer(self) -> AnyTokenizer:
|
||||
return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user