[Misc] Add a wrapper for torch.inference_mode (#6618)

This commit is contained in:
Woosuk Kwon
2024-07-21 18:43:11 -07:00
committed by GitHub
parent c9eef37f32
commit 42de2cefcb
5 changed files with 49 additions and 4 deletions
+2 -1
View File
@@ -9,6 +9,7 @@ import torch
from vllm.distributed import broadcast_tensor_dict, get_pp_group
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.platforms import current_platform
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
SamplerOutput)
from vllm.utils import (enable_trace_function_call_for_thread,
@@ -53,7 +54,7 @@ class WorkerBase(ABC):
"""
raise NotImplementedError
@torch.inference_mode()
@current_platform.inference_mode()
def start_worker_execution_loop(self) -> None:
"""Execute model loop in parallel worker.