[Kernel][Misc] register ops to prevent graph breaks (#6917)
Co-authored-by: Sage Moore <sage@neuralmagic.com>
This commit is contained in:
@@ -75,6 +75,10 @@ _NUM_WARMUP_ITERS = 2
|
||||
|
||||
TModelInputForGPU = TypeVar('TModelInputForGPU', bound="ModelInputForGPU")
|
||||
|
||||
# For now, bump up cache limits for recompilations during CUDA graph warmups.
|
||||
torch._dynamo.config.cache_size_limit = 128
|
||||
torch._dynamo.config.accumulated_cache_size_limit = 128
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ModelInputForGPU(ModelRunnerInputBase):
|
||||
@@ -1060,9 +1064,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
"This may lead to less accurate results!")
|
||||
|
||||
if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
|
||||
self.model = torch.compile(self.model,
|
||||
fullgraph=True,
|
||||
backend="eager")
|
||||
self.model = torch.compile(
|
||||
self.model,
|
||||
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
|
||||
backend="eager")
|
||||
|
||||
def save_sharded_state(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user