[Feature] Add iteration level logging and enhance nvtx marker (#31193)
Signed-off-by: Max Hu <maxhu@nvidia.com> Signed-off-by: Max Hu <hyoung2991@gmail.com> Co-authored-by: Max Hu <maxhu@nvidia.com>
This commit is contained in:
@@ -525,6 +525,9 @@ class EngineArgs:
|
||||
ObservabilityConfig.enable_layerwise_nvtx_tracing
|
||||
)
|
||||
enable_mfu_metrics: bool = ObservabilityConfig.enable_mfu_metrics
|
||||
enable_logging_iteration_details: bool = (
|
||||
ObservabilityConfig.enable_logging_iteration_details
|
||||
)
|
||||
enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
|
||||
scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
|
||||
scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls
|
||||
@@ -1059,6 +1062,10 @@ class EngineArgs:
|
||||
"--enable-mfu-metrics",
|
||||
**observability_kwargs["enable_mfu_metrics"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--enable-logging-iteration-details",
|
||||
**observability_kwargs["enable_logging_iteration_details"],
|
||||
)
|
||||
|
||||
# Scheduler arguments
|
||||
scheduler_kwargs = get_kwargs(SchedulerConfig)
|
||||
@@ -1713,6 +1720,7 @@ class EngineArgs:
|
||||
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
|
||||
enable_mfu_metrics=self.enable_mfu_metrics,
|
||||
enable_mm_processor_stats=self.enable_mm_processor_stats,
|
||||
enable_logging_iteration_details=self.enable_logging_iteration_details,
|
||||
)
|
||||
|
||||
# Compilation config overrides
|
||||
|
||||
Reference in New Issue
Block a user