[torch.compile] Use FakeTensors instead of real GPU tensors for single-size compilation (#36093)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
@@ -348,13 +348,39 @@ class InductorStandaloneAdaptor(CompilerInterface):
|
||||
# Can remove this after the following issue gets fixed
|
||||
# https://github.com/pytorch/pytorch/issues/174502
|
||||
if envs.VLLM_ENABLE_PREGRAD_PASSES:
|
||||
ctx: Any = contextlib.nullcontext()
|
||||
pregrad_ctx: Any = contextlib.nullcontext()
|
||||
else:
|
||||
ctx = patch(
|
||||
pregrad_ctx = patch(
|
||||
"torch._inductor.compile_fx._recursive_pre_grad_passes",
|
||||
lambda gm, _: gm,
|
||||
)
|
||||
with ctx, _patch_constrain_to_fx_strides():
|
||||
|
||||
# When inputs are FakeTensors (from create_concrete_args),
|
||||
# standalone_compile("from_example_inputs") would normally create
|
||||
# a fresh FakeTensorMode, causing a mode mismatch assertion.
|
||||
# Patch FakeTensorMode in standalone_compile so it reuses the
|
||||
# mode already attached to our FakeTensors. This gives us both
|
||||
# ignore_shape_env=True (from "from_example_inputs") and mode
|
||||
# consistency (from reusing our mode).
|
||||
# Can remove this after the following issue gets fixed:
|
||||
# https://github.com/pytorch/pytorch/issues/176562
|
||||
from torch._subclasses.fake_tensor import FakeTensor
|
||||
|
||||
input_fake_mode = None
|
||||
for x in example_inputs:
|
||||
if isinstance(x, FakeTensor):
|
||||
input_fake_mode = x.fake_mode
|
||||
break
|
||||
|
||||
if input_fake_mode is not None:
|
||||
fake_mode_ctx: Any = patch(
|
||||
"torch._inductor.standalone_compile.FakeTensorMode",
|
||||
lambda *a, **kw: input_fake_mode,
|
||||
)
|
||||
else:
|
||||
fake_mode_ctx = contextlib.nullcontext()
|
||||
|
||||
with pregrad_ctx, fake_mode_ctx, _patch_constrain_to_fx_strides():
|
||||
compiled_graph = standalone_compile(graph, example_inputs, **compile_kwargs)
|
||||
|
||||
if use_aot:
|
||||
|
||||
@@ -34,13 +34,14 @@ def get_fake_args_from_graph(graph: fx.GraphModule) -> list[Any]:
|
||||
|
||||
|
||||
def create_concrete_args(graph: fx.GraphModule, size: int) -> list[Any]:
|
||||
"""Create example inputs with symbolic dims replaced by a concrete size.
|
||||
"""Create Fake example inputs with symbolic dims replaced by a concrete size.
|
||||
|
||||
Used for single-size eager compilation where we need concrete-shaped
|
||||
inputs but don't have real runtime tensors yet.
|
||||
Used for single-size compilation where we need concrete-shaped inputs.
|
||||
The Dynamo-captured graph gives us example inputs with SymInts in them.
|
||||
"""
|
||||
from torch._prims_common import compute_required_storage_length
|
||||
from torch.fx.experimental.symbolic_shapes import is_symbolic
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
from torch.fx.experimental.symbolic_shapes import ShapeEnv, is_symbolic
|
||||
|
||||
def concretize(sym_val: Any) -> int:
|
||||
"""Replace all symbolic variables in a SymInt expression with size."""
|
||||
@@ -49,25 +50,28 @@ def create_concrete_args(graph: fx.GraphModule, size: int) -> list[Any]:
|
||||
expr = sym_val.node.expr
|
||||
return int(expr.subs({s: size for s in expr.free_symbols}))
|
||||
|
||||
fake_mode = FakeTensorMode(shape_env=ShapeEnv())
|
||||
|
||||
args: list[Any] = []
|
||||
for node in graph.graph.nodes:
|
||||
if node.op != "placeholder":
|
||||
break
|
||||
val = node.meta["example_value"]
|
||||
if isinstance(val, torch.SymInt):
|
||||
args.append(concretize(val))
|
||||
elif isinstance(val, torch.Tensor):
|
||||
new_shape = tuple(concretize(d) for d in val.shape)
|
||||
new_strides = tuple(concretize(s) for s in val.stride())
|
||||
new_storage_offset = concretize(val.storage_offset())
|
||||
needed_size = compute_required_storage_length(
|
||||
new_shape, new_strides, new_storage_offset
|
||||
)
|
||||
t = torch.empty(needed_size, dtype=val.dtype, device=val.device)
|
||||
t = t.as_strided(new_shape, new_strides, new_storage_offset)
|
||||
args.append(t)
|
||||
else:
|
||||
args.append(val)
|
||||
with fake_mode:
|
||||
for node in graph.graph.nodes:
|
||||
if node.op != "placeholder":
|
||||
break
|
||||
val = node.meta["example_value"]
|
||||
if isinstance(val, torch.SymInt):
|
||||
args.append(concretize(val))
|
||||
elif isinstance(val, torch.Tensor):
|
||||
new_shape = tuple(concretize(d) for d in val.shape)
|
||||
new_strides = tuple(concretize(s) for s in val.stride())
|
||||
new_storage_offset = concretize(val.storage_offset())
|
||||
needed_size = compute_required_storage_length(
|
||||
new_shape, new_strides, new_storage_offset
|
||||
)
|
||||
t = torch.empty(needed_size, dtype=val.dtype, device=val.device)
|
||||
t = t.as_strided(new_shape, new_strides, new_storage_offset)
|
||||
args.append(t)
|
||||
else:
|
||||
args.append(val)
|
||||
return args
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user