[torch.compile] Add patch for fullgraph compilation (#42686)
Signed-off-by: Luka Govedič <luka.govedic@gmail.com>
This commit is contained in:
@@ -758,3 +758,65 @@ def _patch_cpp_indirect_assert_if_needed():
|
||||
|
||||
|
||||
_patch_cpp_indirect_assert_if_needed()
|
||||
|
||||
|
||||
def _patch_should_realize_on_reuse():
|
||||
"""
|
||||
This patches the materialization heuristic in Inductor compilation.
|
||||
Patched in torch 2.12: https://github.com/pytorch/pytorch/pull/176994
|
||||
vLLM issue: https://github.com/vllm-project/vllm/issues/27828
|
||||
|
||||
Without this patch, Inductor inlined a chain of computations when compiling
|
||||
the whole model graph at once. For example, the residual in fused_add_rms_norm
|
||||
would get recomputed every time, and this would cascade through the model.
|
||||
|
||||
Note that pytorch/pytorch#176994 updated multiple callsites,
|
||||
this is a simplified version of the fix.
|
||||
"""
|
||||
|
||||
# Only apply for 2.11, fix included in 2.12
|
||||
if not is_torch_equal("2.11.0"):
|
||||
return
|
||||
|
||||
def should_realize_on_reuse_patched(self, users: int) -> bool:
|
||||
"""
|
||||
A heuristic to decide if we should realize a tensor
|
||||
that is used multiple times.
|
||||
"""
|
||||
from torch._inductor import config
|
||||
from torch._inductor.ir import Pointwise, Reduction, is_cpu
|
||||
from torch._inductor.virtualized import V
|
||||
|
||||
if users > 1 and isinstance(self.data, (Pointwise, Reduction)):
|
||||
if is_cpu(self.data):
|
||||
# Heuristic for realizing reused result of heavy ops on cpu
|
||||
opcount = self.data.inner_fn_opcount()
|
||||
heavy_ops = ["exp", "sigmoid"] # a list of heavy ops
|
||||
if any(x in opcount.used_ops for x in heavy_ops):
|
||||
return True
|
||||
if self.has_large_inner_fn():
|
||||
return True
|
||||
# Size-aware cost model comparing total memory traffic:
|
||||
# Inline: total_read_bytes * users
|
||||
# Materialize: total_read_bytes + output_bytes * (1 + users)
|
||||
# This naturally handles broadcast reads (small buffers are
|
||||
# cheap to re-read) and dtype promotions (fp32 outputs cost
|
||||
# more to write than bf16 inputs cost to read).
|
||||
total_read_bytes = sum(
|
||||
V.graph.get_dep_size_hint(dep) for dep in self.get_reads()
|
||||
)
|
||||
output_bytes = (
|
||||
V.graph.sizevars.optimization_hint(self.data.get_numel(), fallback=0)
|
||||
* self.data.dtype.itemsize
|
||||
)
|
||||
if total_read_bytes > 0 and output_bytes > 0:
|
||||
return total_read_bytes * (users - 1) >= output_bytes * (1 + users)
|
||||
return self.num_reads() > config.realize_reads_threshold
|
||||
return False
|
||||
|
||||
from torch._inductor.ir import StorageBox
|
||||
|
||||
StorageBox.should_realize_on_reuse = should_realize_on_reuse_patched
|
||||
|
||||
|
||||
_patch_should_realize_on_reuse()
|
||||
|
||||
Reference in New Issue
Block a user