Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+9 -8
View File
@@ -7,9 +7,9 @@ import hashlib
import os
import pprint
import time
from collections.abc import Sequence
from collections.abc import Callable, Sequence
from contextlib import contextmanager
from typing import Any, Callable, Optional
from typing import Any
import torch
import torch.fx as fx
@@ -75,7 +75,7 @@ class CompilerManager:
"""
def __init__(self, compilation_config: CompilationConfig):
self.cache: dict[tuple[Optional[int], int, str], Any] = dict()
self.cache: dict[tuple[int | None, int, str], Any] = dict()
self.is_cache_updated = False
self.compilation_config = compilation_config
self.compiler = make_compiler(compilation_config)
@@ -84,7 +84,7 @@ class CompilerManager:
return self.compiler.compute_hash(vllm_config)
@contextmanager
def compile_context(self, runtime_shape: Optional[int] = None):
def compile_context(self, runtime_shape: int | None = None):
"""Provide compilation context for the duration of compilation to set
any torch global properties we want to scope to a single Inductor
compilation (e.g. partition rules, pass context)."""
@@ -145,8 +145,8 @@ class CompilerManager:
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
) -> Optional[Callable]:
runtime_shape: int | None = None,
) -> Callable | None:
if (runtime_shape, graph_index, self.compiler.name) not in self.cache:
return None
handle = self.cache[(runtime_shape, graph_index, self.compiler.name)]
@@ -178,7 +178,7 @@ class CompilerManager:
compilation_config: CompilationConfig,
graph_index: int = 0,
num_graphs: int = 1,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Any:
if graph_index == 0:
# before compiling the first graph, record the start time
@@ -656,7 +656,8 @@ class VllmBackend:
graph_path = os.path.join(local_cache_dir, "computation_graph.py")
if not os.path.exists(graph_path):
# code adapted from https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30 # noqa
# code adapted from
# https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30
# use `print_readable` because it can include submodules
src = (
"from __future__ import annotations\nimport torch\n"
+2 -1
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Callable, Protocol
from collections.abc import Callable
from typing import Any, Protocol
from vllm.config import CUDAGraphMode, VllmConfig
+9 -10
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from importlib.util import find_spec
from typing import Optional
import torch
import torch._inductor.pattern_matcher as pm
@@ -432,7 +431,7 @@ class AsyncTPPass(VllmPatternMatcherPass):
self.dump_patterns(config, self.patterns)
def is_applicable_for_shape(self, shape: Optional[int]) -> bool:
def is_applicable_for_shape(self, shape: int | None) -> bool:
# only do replace for specific shapes
tp_size = get_tensor_model_parallel_world_size()
return shape is not None and shape % tp_size == 0
@@ -485,10 +484,10 @@ if flashinfer_comm is not None:
max_token_num: int,
pattern_code: int,
fuse_rms_quant: bool,
norm_out: Optional[torch.Tensor] = None,
quant_out: Optional[torch.Tensor] = None,
scale_out: Optional[torch.Tensor] = None,
scale_factor: Optional[torch.Tensor] = None,
norm_out: torch.Tensor | None = None,
quant_out: torch.Tensor | None = None,
scale_out: torch.Tensor | None = None,
scale_factor: torch.Tensor | None = None,
) -> None:
num_tokens, hidden_size = allreduce_in.shape
element_size = allreduce_in.element_size()
@@ -589,10 +588,10 @@ if flashinfer_comm is not None:
max_token_num: int,
pattern_code: int,
fuse_rms_quant: bool,
norm_out: Optional[torch.Tensor] = None,
quant_out: Optional[torch.Tensor] = None,
scale_out: Optional[torch.Tensor] = None,
scale_factor: Optional[torch.Tensor] = None,
norm_out: torch.Tensor | None = None,
quant_out: torch.Tensor | None = None,
scale_out: torch.Tensor | None = None,
scale_factor: torch.Tensor | None = None,
) -> None:
pass
+17 -16
View File
@@ -4,8 +4,9 @@ import contextlib
import copy
import hashlib
import os
from collections.abc import Callable
from contextlib import ExitStack
from typing import Any, Callable, Optional
from typing import Any
from unittest.mock import patch
import torch
@@ -62,9 +63,9 @@ class CompilerInterface:
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
"""
Compile the graph with the given example inputs and compiler config,
with a runtime shape. If the `runtime_shape` is None, it means
@@ -97,7 +98,7 @@ class CompilerInterface:
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Callable:
"""
Load the compiled function from the handle.
@@ -191,9 +192,9 @@ class InductorStandaloneAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
compilation_counter.num_inductor_compiles += 1
current_config = {}
if compiler_config is not None:
@@ -229,7 +230,7 @@ class InductorStandaloneAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Callable:
assert isinstance(handle, tuple)
assert isinstance(handle[0], str)
@@ -293,9 +294,9 @@ class InductorAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
compilation_counter.num_inductor_compiles += 1
from torch._inductor.compile_fx import compile_fx
@@ -492,7 +493,7 @@ class InductorAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
graph_index: int,
runtime_shape: Optional[int] = None,
runtime_shape: int | None = None,
) -> Callable:
assert isinstance(handle, tuple)
assert isinstance(handle[0], str)
@@ -610,9 +611,9 @@ class EagerAdaptor(CompilerInterface):
graph: fx.GraphModule,
example_inputs: list[Any],
compiler_config: dict[str, Any],
runtime_shape: Optional[int] = None,
key: Optional[str] = None,
) -> tuple[Optional[Callable], Optional[Any]]:
runtime_shape: int | None = None,
key: str | None = None,
) -> tuple[Callable | None, Any | None]:
compilation_counter.num_eager_compiles += 1
# we don't need to compile the graph, just return the graph itself.
# It does not support caching, return None for the handle.
+6 -5
View File
@@ -2,8 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from collections.abc import Callable
from contextlib import ExitStack
from typing import Any, Callable, Optional
from typing import Any
from unittest.mock import patch
import torch
@@ -24,12 +25,12 @@ logger = init_logger(__name__)
@dataclasses.dataclass
class CUDAGraphEntry:
batch_descriptor: BatchDescriptor
cudagraph: Optional[torch.cuda.CUDAGraph] = None
output: Optional[Any] = None
cudagraph: torch.cuda.CUDAGraph | None = None
output: Any | None = None
# for cudagraph debugging, track the input addresses
# during capture, and check if they are the same during replay
input_addresses: Optional[list[int]] = None
input_addresses: list[int] | None = None
@dataclasses.dataclass
@@ -69,7 +70,7 @@ class CUDAGraphWrapper:
runnable: Callable,
vllm_config: VllmConfig,
runtime_mode: CUDAGraphMode,
cudagraph_options: Optional[CUDAGraphOptions] = None,
cudagraph_options: CUDAGraphOptions | None = None,
):
self.runnable = runnable
self.vllm_config = vllm_config
+12 -11
View File
@@ -6,7 +6,8 @@ import hashlib
import inspect
import os
import sys
from typing import Callable, Optional, TypeVar, Union, overload
from collections.abc import Callable
from typing import TypeVar, overload
from unittest.mock import patch
import torch
@@ -61,14 +62,14 @@ def _should_ignore_torch_compile(cls) -> bool:
@overload
def support_torch_compile(
*,
enable_if: Optional[Callable[[VllmConfig], bool]] = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
) -> Callable[[_T], _T]: ...
@overload
def support_torch_compile(
*,
dynamic_arg_dims: Optional[dict[str, Union[int, list[int]]]],
dynamic_arg_dims: dict[str, int | list[int]] | None,
) -> Callable[[_T], _T]: ...
@@ -77,11 +78,11 @@ def support_torch_compile(cls: _T) -> _T: ...
def support_torch_compile(
cls: Optional[_T] = None,
cls: _T | None = None,
*,
dynamic_arg_dims: Optional[dict[str, Union[int, list[int]]]] = None,
enable_if: Optional[Callable[[VllmConfig], bool]] = None,
) -> Union[Callable[[_T], _T], _T]:
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
enable_if: Callable[[VllmConfig], bool] | None = None,
) -> Callable[[_T], _T] | _T:
"""
A decorator to add support for compiling the forward method of a class.
@@ -147,9 +148,9 @@ def support_torch_compile(
for k, v in sig.parameters.items():
if v.annotation in [
torch.Tensor,
Optional[torch.Tensor],
torch.Tensor | None,
IntermediateTensors,
Optional[IntermediateTensors],
IntermediateTensors | None,
]:
inferred_dynamic_arg_dims[k] = 0
@@ -209,8 +210,8 @@ def _verify_source_unchanged(source_info, vllm_config) -> None:
def _support_torch_compile(
cls: _T,
dynamic_arg_dims: dict[str, Union[int, list[int]]],
enable_if: Optional[Callable[[VllmConfig], bool]] = None,
dynamic_arg_dims: dict[str, int | list[int]],
enable_if: Callable[[VllmConfig], bool] | None = None,
) -> _T:
"""
A decorator to add support for compiling the forward method of a class.
+5 -6
View File
@@ -3,7 +3,6 @@
import operator
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
@@ -150,7 +149,7 @@ class FixFunctionalizationPass(VllmInductorPass):
)
self.nodes_to_remove.clear()
def _remove(self, node_or_nodes: Union[torch.fx.Node, Iterable[torch.fx.Node]]):
def _remove(self, node_or_nodes: torch.fx.Node | Iterable[torch.fx.Node]):
"""
Stage a node (or nodes) for removal at the end of the pass.
"""
@@ -163,8 +162,8 @@ class FixFunctionalizationPass(VllmInductorPass):
self,
graph: torch.fx.Graph,
node: torch.fx.Node,
mutated_args: dict[int, Union[torch.fx.Node, str]],
args: Optional[tuple[Union[torch.fx.Node, str], ...]] = None,
mutated_args: dict[int, torch.fx.Node | str],
args: tuple[torch.fx.Node | str, ...] | None = None,
):
"""
De-functionalize a node by replacing it with a call to the original.
@@ -176,7 +175,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self._remove(node)
def replace_users_with_mutated_args(
self, node: torch.fx.Node, mutated_args: dict[int, Union[torch.fx.Node, str]]
self, node: torch.fx.Node, mutated_args: dict[int, torch.fx.Node | str]
):
"""
Replace all getitem users of the auto-functionalized node with the
@@ -207,7 +206,7 @@ class FixFunctionalizationPass(VllmInductorPass):
self,
graph: torch.fx.Graph,
node: torch.fx.Node,
args: Optional[tuple[Union[torch.fx.Node, str], ...]] = None,
args: tuple[torch.fx.Node | str, ...] | None = None,
):
"""
Insert a new defunctionalized node into the graph before node.
+3 -6
View File
@@ -3,7 +3,6 @@
import operator
from collections.abc import Iterable, Iterator
from typing import Optional
from torch import fx
from torch._higher_order_ops.auto_functionalize import auto_functionalized
@@ -19,9 +18,7 @@ def is_auto_func(node: fx.Node, op: OpOverload) -> bool:
# Returns the first specified node with the given op (if it exists)
def find_specified_fn_maybe(
nodes: Iterable[fx.Node], op: OpOverload
) -> Optional[fx.Node]:
def find_specified_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node | None:
for node in nodes:
if node.target == op:
return node
@@ -36,7 +33,7 @@ def find_specified_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node:
# Returns the first auto_functionalized node with the given op (if it exists)
def find_auto_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> Optional[fx.Node]:
def find_auto_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node | None:
for node in nodes:
if is_func(node, auto_functionalized) and node.args[0] == op: # noqa
return node
@@ -52,7 +49,7 @@ def find_auto_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node:
# Returns the getitem node that extracts the idx-th element from node
# (if it exists)
def find_getitem_maybe(node: fx.Node, idx: int) -> Optional[fx.Node]:
def find_getitem_maybe(node: fx.Node, idx: int) -> fx.Node | None:
for user in node.users:
if is_func(user, operator.getitem) and user.args[1] == idx:
return user
+7 -8
View File
@@ -6,8 +6,9 @@ import hashlib
import inspect
import json
import types
from collections.abc import Callable
from contextlib import contextmanager
from typing import Any, Callable, Optional, Union
from typing import Any
import torch
from torch import fx
@@ -27,7 +28,7 @@ _pass_context = None
class PassContext:
def __init__(self, runtime_shape: Optional[int]):
def __init__(self, runtime_shape: int | None):
self.runtime_shape = runtime_shape
@@ -38,7 +39,7 @@ def get_pass_context() -> PassContext:
@contextmanager
def pass_context(runtime_shape: Optional[int]):
def pass_context(runtime_shape: int | None):
"""A context manager that stores the current pass context,
usually it is a list of sizes to specialize.
"""
@@ -67,7 +68,7 @@ class InductorPass(CustomGraphPass):
return InductorPass.hash_source(self)
@staticmethod
def hash_source(*srcs: Union[str, Any]):
def hash_source(*srcs: str | Any):
"""
Utility method to hash the sources of functions or objects.
:param srcs: strings or objects to add to the hash.
@@ -95,7 +96,7 @@ class InductorPass(CustomGraphPass):
encoded = json.dumps(dict_, sort_keys=True).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def is_applicable_for_shape(self, shape: Optional[int]):
def is_applicable_for_shape(self, shape: int | None):
return True
@@ -105,9 +106,7 @@ class CallableInductorPass(InductorPass):
implementation of the UUID.
"""
def __init__(
self, callable: Callable[[fx.Graph], None], uuid: Optional[Any] = None
):
def __init__(self, callable: Callable[[fx.Graph], None], uuid: Any | None = None):
self.callable = callable
self._uuid = self.hash_source(callable) if uuid is None else uuid
+2 -5
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Union
import torch.fx
from torch import SymInt
@@ -105,9 +104,7 @@ class NoOpEliminationPass(VllmInductorPass):
logger.debug("Removed %s no-op reshapes and slices", count)
# ---------------------- Shape comparison helpers ----------------------
def dims_equivalent(
self, dim: Union[int, SymInt], i_dim: Union[int, SymInt]
) -> bool:
def dims_equivalent(self, dim: int | SymInt, i_dim: int | SymInt) -> bool:
"""
This function checks if two dimensions are equivalent.
:param dim: The dimension arg to reshape/slice
@@ -127,7 +124,7 @@ class NoOpEliminationPass(VllmInductorPass):
return False
def all_dims_equivalent(
self, dims: Iterable[Union[int, SymInt]], i_dims: Iterable[Union[int, SymInt]]
self, dims: Iterable[int | SymInt], i_dims: Iterable[int | SymInt]
) -> bool:
dims_ = list(dims)
i_dims_ = list(i_dims)
+2 -4
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import contextlib
from typing import TYPE_CHECKING
@@ -16,7 +14,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def resolve_defined_ops(op_names: list[str]) -> list[torch._ops.OpOverload]:
def resolve_defined_ops(op_names: list[str]) -> list["torch._ops.OpOverload"]:
"""Resolve operator names to OpOverload objects.
Skips operators that fail to resolve (e.g., operators not registered or
@@ -49,7 +47,7 @@ def resolve_defined_ops(op_names: list[str]) -> list[torch._ops.OpOverload]:
@contextlib.contextmanager
def inductor_partition_rule_context(overloads: list[torch._ops.OpOverload]):
def inductor_partition_rule_context(overloads: list["torch._ops.OpOverload"]):
"""Context manager to temporarily register Inductor partition rules.
Registers custom partition rules for specified operators, forcing the
+2 -1
View File
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from typing import Any, Callable
from collections.abc import Callable
from typing import Any
import torch.fx as fx
+3 -4
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
import torch._inductor.pattern_matcher as pm
@@ -27,7 +26,7 @@ class _RMSNormAndQuantOpHelper:
epsilon: float,
dtype: torch.dtype,
device: str,
quant_op: Optional[torch._ops.OpOverload] = None,
quant_op: torch._ops.OpOverload | None = None,
**kwargs,
):
self.epsilon = epsilon
@@ -110,7 +109,7 @@ class _SequenceParallelPatternHelper(_RMSNormAndQuantOpHelper):
epsilon: float,
dtype: torch.dtype,
device: str,
quant_op: Optional[torch._ops.OpOverload] = None,
quant_op: torch._ops.OpOverload | None = None,
**kwargs,
):
super().__init__(epsilon, dtype, device, quant_op=quant_op, **kwargs)
@@ -483,7 +482,7 @@ class SequenceParallelismPass(VllmPatternMatcherPass):
).register(self.patterns)
self.dump_patterns(config, self.patterns)
def is_applicable_for_shape(self, shape: Optional[int]) -> bool:
def is_applicable_for_shape(self, shape: int | None) -> bool:
tp_size = get_tensor_model_parallel_world_size()
return shape is not None and shape % tp_size == 0
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Any, Optional
from typing import Any
import torch
@@ -23,7 +23,7 @@ class Torch25CustomGraphPass(ABC): # noqa (redefinition)
"""
@abstractmethod
def uuid(self) -> Optional[Any]:
def uuid(self) -> Any | None:
"""
Return an ID to uniquely identify your custom pass implementation.
Return None to skip inductor code caching entirely.
+2 -2
View File
@@ -3,7 +3,7 @@
import functools
import operator
import time
from typing import ClassVar, Optional
from typing import ClassVar
import regex as re
import torch
@@ -24,7 +24,7 @@ class VllmInductorPass(InductorPass):
It provides timing, logging, and dumping utilities.
"""
dump_prefix: ClassVar[Optional[int]] = None
dump_prefix: ClassVar[int | None] = None
"""Keep track of pass index for debug dump ordering."""
def __init__(self, config: VllmConfig):
+2 -2
View File
@@ -4,9 +4,9 @@
import os
import sys
from abc import abstractmethod
from collections.abc import Callable
from contextlib import contextmanager
from types import CodeType
from typing import Callable, Optional
import torch
@@ -31,7 +31,7 @@ class TorchCompileWrapperWithCustomDispatcher:
"""
def __init__(
self, compiled_callable: Optional[Callable] = None, compilation_level: int = 0
self, compiled_callable: Callable | None = None, compilation_level: int = 0
):
vllm_config = get_current_vllm_config()
self.vllm_config = vllm_config