[Feature] Add iteration level logging and enhance nvtx marker (#31193)

Signed-off-by: Max Hu <maxhu@nvidia.com>
Signed-off-by: Max Hu <hyoung2991@gmail.com>
Co-authored-by: Max Hu <maxhu@nvidia.com>
This commit is contained in:
Max Hu
2026-01-09 00:13:39 +00:00
committed by GitHub
co-authored by Max Hu
parent 11cec296dd
commit 6ebe34d6fa
6 changed files with 137 additions and 9 deletions
+8
View File
@@ -525,6 +525,9 @@ class EngineArgs:
ObservabilityConfig.enable_layerwise_nvtx_tracing
)
enable_mfu_metrics: bool = ObservabilityConfig.enable_mfu_metrics
enable_logging_iteration_details: bool = (
ObservabilityConfig.enable_logging_iteration_details
)
enable_mm_processor_stats: bool = ObservabilityConfig.enable_mm_processor_stats
scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls
@@ -1059,6 +1062,10 @@ class EngineArgs:
"--enable-mfu-metrics",
**observability_kwargs["enable_mfu_metrics"],
)
observability_group.add_argument(
"--enable-logging-iteration-details",
**observability_kwargs["enable_logging_iteration_details"],
)
# Scheduler arguments
scheduler_kwargs = get_kwargs(SchedulerConfig)
@@ -1713,6 +1720,7 @@ class EngineArgs:
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
enable_mfu_metrics=self.enable_mfu_metrics,
enable_mm_processor_stats=self.enable_mm_processor_stats,
enable_logging_iteration_details=self.enable_logging_iteration_details,
)
# Compilation config overrides