[misc] CUDA Time Layerwise Profiler (#8337)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
Michael Goin
parent
390be74649
commit
9d30a056e7
@@ -1742,10 +1742,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
|
||||
return [output]
|
||||
|
||||
|
||||
class CUDAGraphRunner:
|
||||
# NOTE: this is nn.Module so the profiler can properly capture/group
|
||||
# kernels calls made within the graph
|
||||
class CUDAGraphRunner(nn.Module):
|
||||
|
||||
def __init__(self, model: nn.Module, backend_name: str,
|
||||
attn_state: AttentionState, is_encoder_decoder_model: bool):
|
||||
super().__init__()
|
||||
self.model = model
|
||||
self.backend_name = backend_name
|
||||
self.attn_state = attn_state
|
||||
@@ -1892,9 +1895,6 @@ class CUDAGraphRunner:
|
||||
|
||||
return self.output_buffers
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self.forward(*args, **kwargs)
|
||||
|
||||
|
||||
def _get_graph_batch_size(batch_size: int) -> int:
|
||||
"""Returns the padded batch size given actual batch size.
|
||||
|
||||
Reference in New Issue
Block a user