[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)
Signed-off-by: czhu <conway.zhu@cohere.com> Signed-off-by: Conway Zhu <conway.zhu@cohere.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
+17
-3
@@ -785,16 +785,30 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
def start_profile(self, profile_prefix: str | None = None) -> None:
|
||||
"""Start profiling with optional custom trace prefix.
|
||||
def start_profile(
|
||||
self,
|
||||
profile_prefix: str | None = None,
|
||||
*,
|
||||
delay_iterations: int | None = None,
|
||||
max_iterations: int | None = None,
|
||||
) -> None:
|
||||
"""Start profiling with optional per-session overrides.
|
||||
|
||||
Args:
|
||||
profile_prefix: Optional prefix for the trace file names. If provided,
|
||||
trace files will be named as "<prefix>_dp<X>_pp<Y>_tp<Z>".
|
||||
If not provided, default naming will be used.
|
||||
delay_iterations: Optional number of worker iterations to skip before
|
||||
profiling starts.
|
||||
max_iterations: Optional maximum number of worker iterations to profile.
|
||||
Zero means no limit.
|
||||
|
||||
"""
|
||||
self.llm_engine.start_profile(profile_prefix)
|
||||
self.llm_engine.start_profile(
|
||||
profile_prefix,
|
||||
delay_iterations=delay_iterations,
|
||||
max_iterations=max_iterations,
|
||||
)
|
||||
|
||||
def stop_profile(self) -> None:
|
||||
self.llm_engine.stop_profile()
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from fastapi import APIRouter, FastAPI, Request
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from vllm.config import ProfilerConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -14,14 +15,32 @@ logger = init_logger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class StartProfileRequest(BaseModel):
|
||||
"""Optional overrides for one profiling session.
|
||||
|
||||
Values are validated by the engine client.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="ignore", strict=True)
|
||||
|
||||
profile_prefix: str | None = None
|
||||
delay_iterations: int | None = None
|
||||
max_iterations: int | None = None
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
@router.post("/start_profile")
|
||||
async def start_profile(raw_request: Request):
|
||||
async def start_profile(
|
||||
raw_request: Request, profile_request: StartProfileRequest | None = None
|
||||
):
|
||||
logger.info("Starting profiler...")
|
||||
await engine_client(raw_request).start_profile()
|
||||
# Only pass supplied overrides so clients without them keep the
|
||||
# zero-argument call.
|
||||
overrides = profile_request.model_dump(exclude_none=True) if profile_request else {}
|
||||
await engine_client(raw_request).start_profile(**overrides)
|
||||
logger.info("Profiler started.")
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user