From 966903eb93a053a908fbf8b931fcebfb28c4741a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luka=20Govedi=C4=8D?= Date: Sun, 17 May 2026 15:49:16 -0400 Subject: [PATCH] [torch.compile] Add patch for fullgraph compilation (#42686) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luka Govedič --- vllm/env_override.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/vllm/env_override.py b/vllm/env_override.py index 78270c2bee..47372144a2 100644 --- a/vllm/env_override.py +++ b/vllm/env_override.py @@ -758,3 +758,65 @@ def _patch_cpp_indirect_assert_if_needed(): _patch_cpp_indirect_assert_if_needed() + + +def _patch_should_realize_on_reuse(): + """ + This patches the materialization heuristic in Inductor compilation. + Patched in torch 2.12: https://github.com/pytorch/pytorch/pull/176994 + vLLM issue: https://github.com/vllm-project/vllm/issues/27828 + + Without this patch, Inductor inlined a chain of computations when compiling + the whole model graph at once. For example, the residual in fused_add_rms_norm + would get recomputed every time, and this would cascade through the model. + + Note that pytorch/pytorch#176994 updated multiple callsites, + this is a simplified version of the fix. + """ + + # Only apply for 2.11, fix included in 2.12 + if not is_torch_equal("2.11.0"): + return + + def should_realize_on_reuse_patched(self, users: int) -> bool: + """ + A heuristic to decide if we should realize a tensor + that is used multiple times. + """ + from torch._inductor import config + from torch._inductor.ir import Pointwise, Reduction, is_cpu + from torch._inductor.virtualized import V + + if users > 1 and isinstance(self.data, (Pointwise, Reduction)): + if is_cpu(self.data): + # Heuristic for realizing reused result of heavy ops on cpu + opcount = self.data.inner_fn_opcount() + heavy_ops = ["exp", "sigmoid"] # a list of heavy ops + if any(x in opcount.used_ops for x in heavy_ops): + return True + if self.has_large_inner_fn(): + return True + # Size-aware cost model comparing total memory traffic: + # Inline: total_read_bytes * users + # Materialize: total_read_bytes + output_bytes * (1 + users) + # This naturally handles broadcast reads (small buffers are + # cheap to re-read) and dtype promotions (fp32 outputs cost + # more to write than bf16 inputs cost to read). + total_read_bytes = sum( + V.graph.get_dep_size_hint(dep) for dep in self.get_reads() + ) + output_bytes = ( + V.graph.sizevars.optimization_hint(self.data.get_numel(), fallback=0) + * self.data.dtype.itemsize + ) + if total_read_bytes > 0 and output_bytes > 0: + return total_read_bytes * (users - 1) >= output_bytes * (1 + users) + return self.num_reads() > config.realize_reads_threshold + return False + + from torch._inductor.ir import StorageBox + + StorageBox.should_realize_on_reuse = should_realize_on_reuse_patched + + +_patch_should_realize_on_reuse()