[misc] [doc] [frontend] LLM torch profiler support (#7943)

This commit is contained in:
William Lin
2024-09-06 17:48:48 -07:00
committed by GitHub
parent 29f49cd6e3
commit 12dd715807
6 changed files with 74 additions and 3 deletions
+6
View File
@@ -1914,6 +1914,12 @@ class LLMEngine:
self.tokenizer.check_health()
self.model_executor.check_health()
def start_profile(self) -> None:
self.model_executor.start_profile()
def stop_profile(self) -> None:
self.model_executor.stop_profile()
def is_tracing_enabled(self) -> bool:
return self.tracer is not None
+6
View File
@@ -560,6 +560,12 @@ class LLM:
outputs = self._run_engine(use_tqdm=use_tqdm)
return LLMEngine.validate_outputs(outputs, EmbeddingRequestOutput)
def start_profile(self) -> None:
self.llm_engine.start_profile()
def stop_profile(self) -> None:
self.llm_engine.stop_profile()
# LEGACY
def _convert_v1_inputs(
self,
+6
View File
@@ -296,6 +296,12 @@ class CPUExecutor(ExecutorBase):
for result in parallel_worker_tasks:
result.get()
def start_profile(self) -> None:
self.driver_method_invoker(self.driver_worker, "start_profile")
def stop_profile(self) -> None:
self.driver_method_invoker(self.driver_worker, "stop_profile")
class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
+6
View File
@@ -169,6 +169,12 @@ class GPUExecutor(ExecutorBase):
# it's running.
return
def start_profile(self) -> None:
self.driver_worker.start_profile()
def stop_profile(self) -> None:
self.driver_worker.stop_profile()
class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):