[Core] Subclass ModelRunner to support cross-attention & encoder sequences (towards eventual encoder/decoder model support) (#4942)
Co-authored-by: Andrew Feldman <afeld2012@gmail.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
This commit is contained in:
co-authored by
Andrew Feldman
Nick Hill
parent
660470e5a3
commit
fd95e026e0
+12
-1
@@ -19,8 +19,11 @@ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
from vllm.utils import (is_embedding_model_config,
|
||||
is_encoder_decoder_model_config)
|
||||
from vllm.worker.cache_engine import CacheEngine
|
||||
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
|
||||
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
|
||||
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
|
||||
from vllm.worker.worker_base import LocalOrDistributedWorkerBase, WorkerInput
|
||||
|
||||
@@ -85,8 +88,10 @@ class Worker(LocalOrDistributedWorkerBase):
|
||||
ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
|
||||
if model_runner_cls is not None:
|
||||
ModelRunnerClass = model_runner_cls
|
||||
elif self.model_config.embedding_mode:
|
||||
elif self._is_embedding_model():
|
||||
ModelRunnerClass = EmbeddingModelRunner
|
||||
elif self._is_encoder_decoder_model():
|
||||
ModelRunnerClass = EncoderDecoderModelRunner
|
||||
self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
|
||||
model_config,
|
||||
parallel_config,
|
||||
@@ -107,6 +112,12 @@ class Worker(LocalOrDistributedWorkerBase):
|
||||
# Initialize gpu_cache as embedding models don't initialize kv_caches
|
||||
self.gpu_cache: Optional[List[List[torch.Tensor]]] = None
|
||||
|
||||
def _is_encoder_decoder_model(self):
|
||||
return is_encoder_decoder_model_config(self.model_config)
|
||||
|
||||
def _is_embedding_model(self):
|
||||
return is_embedding_model_config(self.model_config)
|
||||
|
||||
def init_device(self) -> None:
|
||||
if self.device_config.device.type == "cuda":
|
||||
# torch.distributed.all_reduce does not free the input tensor until
|
||||
|
||||
Reference in New Issue
Block a user