[Core] Interface for accessing model from VllmRunner (#10353)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -2,6 +2,7 @@ import weakref
|
||||
from typing import List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
@@ -10,6 +11,10 @@ from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
|
||||
from vllm.spec_decode.top1_proposer import Top1Proposer
|
||||
|
||||
|
||||
class _DummyModel(nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
class NGramWorker(NonLLMProposerWorkerBase):
|
||||
"""NGramWorker provides a light drafter without need for model.
|
||||
|
||||
@@ -36,7 +41,6 @@ class NGramWorker(NonLLMProposerWorkerBase):
|
||||
|
||||
def init_device(self):
|
||||
self.device = torch.device(f"{self.device_type}:{self.local_rank}")
|
||||
self.load_model = lambda *args, **kwargs: None
|
||||
|
||||
# Current NGramWorker only supports Top1Proposer
|
||||
self._proposer = Top1Proposer(
|
||||
@@ -45,6 +49,12 @@ class NGramWorker(NonLLMProposerWorkerBase):
|
||||
vocab_size=self.vocab_size,
|
||||
)
|
||||
|
||||
def load_model(self) -> None:
|
||||
pass # Dummy
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
return _DummyModel()
|
||||
|
||||
def sampler_output(
|
||||
self,
|
||||
execute_model_req: ExecuteModelRequest,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from typing import List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.distributed.parallel_state import (get_tp_group,
|
||||
init_model_parallel_group,
|
||||
@@ -15,6 +16,10 @@ from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class _DummyModel(nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
class SmallerTpProposerWorker(ProposerWorkerBase):
|
||||
"""Class which allows a speculative draft model to run with smaller tensor
|
||||
parallel degree than target model.
|
||||
@@ -139,6 +144,13 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
|
||||
return self._worker.get_spec_proposals(
|
||||
execute_model_req, seq_ids_with_bonus_token_in_last_step)
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
if self._is_dummy:
|
||||
return _DummyModel()
|
||||
|
||||
with self._patch_tensor_parallel_group():
|
||||
return self._worker.get_model()
|
||||
|
||||
def execute_model(
|
||||
self,
|
||||
execute_model_req: Optional[ExecuteModelRequest] = None
|
||||
|
||||
@@ -4,6 +4,7 @@ from functools import cached_property
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple, Type
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ParallelConfig, SpeculativeConfig, VllmConfig
|
||||
from vllm.distributed.communication_op import broadcast_tensor_dict
|
||||
@@ -403,6 +404,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
|
||||
num_cpu_blocks=num_cpu_blocks)
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.scorer_worker.get_model()
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user