[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)

Signed-off-by: czhu <conway.zhu@cohere.com>
Signed-off-by: Conway Zhu <conway.zhu@cohere.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
czhu-cohere
2026-10-06 05:21:19 +00:00
committed by GitHub
co-authored by aoshen02
parent 9a1f6fd4cb
commit 6fddc9b21e
25 changed files with 1497 additions and 164 deletions
+17 -3
View File
@@ -785,16 +785,30 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
mm_processor_kwargs=mm_processor_kwargs,
)
def start_profile(self, profile_prefix: str | None = None) -> None:
"""Start profiling with optional custom trace prefix.
def start_profile(
self,
profile_prefix: str | None = None,
*,
delay_iterations: int | None = None,
max_iterations: int | None = None,
) -> None:
"""Start profiling with optional per-session overrides.
Args:
profile_prefix: Optional prefix for the trace file names. If provided,
trace files will be named as "<prefix>_dp<X>_pp<Y>_tp<Z>".
If not provided, default naming will be used.
delay_iterations: Optional number of worker iterations to skip before
profiling starts.
max_iterations: Optional maximum number of worker iterations to profile.
Zero means no limit.
"""
self.llm_engine.start_profile(profile_prefix)
self.llm_engine.start_profile(
profile_prefix,
delay_iterations=delay_iterations,
max_iterations=max_iterations,
)
def stop_profile(self) -> None:
self.llm_engine.stop_profile()
+21 -2
View File
@@ -4,6 +4,7 @@
from fastapi import APIRouter, FastAPI, Request
from fastapi.responses import Response
from pydantic import BaseModel, ConfigDict
from vllm.config import ProfilerConfig
from vllm.engine.protocol import EngineClient
@@ -14,14 +15,32 @@ logger = init_logger(__name__)
router = APIRouter()
class StartProfileRequest(BaseModel):
"""Optional overrides for one profiling session.
Values are validated by the engine client.
"""
model_config = ConfigDict(extra="ignore", strict=True)
profile_prefix: str | None = None
delay_iterations: int | None = None
max_iterations: int | None = None
def engine_client(request: Request) -> EngineClient:
return request.app.state.engine_client
@router.post("/start_profile")
async def start_profile(raw_request: Request):
async def start_profile(
raw_request: Request, profile_request: StartProfileRequest | None = None
):
logger.info("Starting profiler...")
await engine_client(raw_request).start_profile()
# Only pass supplied overrides so clients without them keep the
# zero-argument call.
overrides = profile_request.model_dump(exclude_none=True) if profile_request else {}
await engine_client(raw_request).start_profile(**overrides)
logger.info("Profiler started.")
return Response(status_code=200)