[misc] CUDA Time Layerwise Profiler (#8337)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
Lucas Wilkinson
2024-10-17 10:36:09 -04:00
committed by GitHub
co-authored by Varun Sundar Rabindranath Michael Goin
parent 390be74649
commit 9d30a056e7
8 changed files with 1390 additions and 4 deletions
+4 -4
View File
@@ -1742,10 +1742,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
return [output]
class CUDAGraphRunner:
# NOTE: this is nn.Module so the profiler can properly capture/group
# kernels calls made within the graph
class CUDAGraphRunner(nn.Module):
def __init__(self, model: nn.Module, backend_name: str,
attn_state: AttentionState, is_encoder_decoder_model: bool):
super().__init__()
self.model = model
self.backend_name = backend_name
self.attn_state = attn_state
@@ -1892,9 +1895,6 @@ class CUDAGraphRunner:
return self.output_buffers
def __call__(self, *args, **kwargs):
return self.forward(*args, **kwargs)
def _get_graph_batch_size(batch_size: int) -> int:
"""Returns the padded batch size given actual batch size.