Make it easy to profile workers with nsight (#3162)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>
This commit is contained in:
co-authored by
Roger Wang
parent
996d095c54
commit
17c3103c56
@@ -46,6 +46,7 @@ class EngineArgs:
|
||||
lora_dtype = 'auto'
|
||||
max_cpu_loras: Optional[int] = None
|
||||
device: str = 'auto'
|
||||
ray_workers_use_nsight: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
if self.tokenizer is None:
|
||||
@@ -168,6 +169,10 @@ class EngineArgs:
|
||||
help='load model sequentially in multiple batches, '
|
||||
'to avoid RAM OOM when using tensor '
|
||||
'parallel and large models')
|
||||
parser.add_argument(
|
||||
'--ray-workers-use-nsight',
|
||||
action='store_true',
|
||||
help='If specified, use nsight to profile ray workers')
|
||||
# KV cache arguments
|
||||
parser.add_argument('--block-size',
|
||||
type=int,
|
||||
@@ -305,7 +310,8 @@ class EngineArgs:
|
||||
self.tensor_parallel_size,
|
||||
self.worker_use_ray,
|
||||
self.max_parallel_loading_workers,
|
||||
self.disable_custom_all_reduce)
|
||||
self.disable_custom_all_reduce,
|
||||
self.ray_workers_use_nsight)
|
||||
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
|
||||
self.max_num_seqs,
|
||||
model_config.max_model_len,
|
||||
|
||||
@@ -124,7 +124,20 @@ class LLMEngine:
|
||||
ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
|
||||
if ray_usage != "1":
|
||||
os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
|
||||
self._init_workers_ray(placement_group)
|
||||
# Pass additional arguments to initialize the worker
|
||||
additional_ray_args = {}
|
||||
if self.parallel_config.ray_workers_use_nsight:
|
||||
logger.info("Configuring Ray workers to use nsight.")
|
||||
additional_ray_args = {
|
||||
"runtime_env": {
|
||||
"nsight": {
|
||||
"t": "cuda,cudnn,cublas",
|
||||
"o": "'worker_process_%p'",
|
||||
"cuda-graph-trace": "node",
|
||||
}
|
||||
}
|
||||
}
|
||||
self._init_workers_ray(placement_group, **additional_ray_args)
|
||||
else:
|
||||
self._init_workers()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user