Make it easy to profile workers with nsight (#3162)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
This commit is contained in:
co-authored by
Roger Wang
parent
996d095c54
commit
17c3103c56
@@ -124,7 +124,20 @@ class LLMEngine:
|
||||
ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
|
||||
if ray_usage != "1":
|
||||
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
|
||||
self._init_workers_ray(placement_group)
|
||||
# Pass additional arguments to initialize the worker
|
||||
additional_ray_args = {}
|
||||
if self.parallel_config.ray_workers_use_nsight:
|
||||
logger.info("Configuring Ray workers to use nsight.")
|
||||
additional_ray_args = {
|
||||
"runtime_env": {
|
||||
"nsight": {
|
||||
"t": "cuda,cudnn,cublas",
|
||||
"o": "'worker_process_%p'",
|
||||
"cuda-graph-trace": "node",
|
||||
}
|
||||
}
|
||||
}
|
||||
self._init_workers_ray(placement_group, **additional_ray_args)
|
||||
else:
|
||||
self._init_workers()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user