[Misc] Add a wrapper for torch.inference_mode (#6618)
This commit is contained in:
@@ -5,6 +5,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import (IntermediateTensors, SamplerOutput,
|
||||
SequenceGroupMetadata)
|
||||
|
||||
@@ -163,7 +164,7 @@ class ModelRunnerBase(ABC, Generic[T]):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@torch.inference_mode()
|
||||
@current_platform.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
model_input: T,
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from vllm.distributed import broadcast_tensor_dict, get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
|
||||
SamplerOutput)
|
||||
from vllm.utils import (enable_trace_function_call_for_thread,
|
||||
@@ -53,7 +54,7 @@ class WorkerBase(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@torch.inference_mode()
|
||||
@current_platform.inference_mode()
|
||||
def start_worker_execution_loop(self) -> None:
|
||||
"""Execute model loop in parallel worker.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user