[MRV1] Scope breakable cudagraphs to the piecewise path only (#56312)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
dffb863c97
commit
cc5dd0a857
@@ -256,6 +256,10 @@ class BreakableCUDAGraphWrapper:
|
||||
* If runtime mode mismatch / NONE, run eagerly.
|
||||
* Otherwise, lazily capture per ``batch_descriptor`` and replay
|
||||
on subsequent invocations with the same descriptor.
|
||||
|
||||
When ``runtime_mode`` is given, only that mode is captured/replayed;
|
||||
other non-NONE modes fall through, so the wrapper can be nested under
|
||||
a ``CUDAGraphWrapper`` for another mode.
|
||||
"""
|
||||
|
||||
_all_instances: ClassVar[weakref.WeakSet[BreakableCUDAGraphWrapper]] = (
|
||||
@@ -271,15 +275,14 @@ class BreakableCUDAGraphWrapper:
|
||||
self,
|
||||
runnable: Callable[..., Any],
|
||||
vllm_config: VllmConfig,
|
||||
runtime_mode: CUDAGraphMode | None = None,
|
||||
) -> None:
|
||||
# Unlike the original CUDAGraphWrapper which strictly matches a
|
||||
# single runtime_mode, this wrapper captures whatever the
|
||||
# dispatcher emits (any non-NONE runtime_mode) -- breakable's
|
||||
# capture is identical for prefill and decode, so there's nothing
|
||||
# to dispatch on at the runtime_mode level. Entries are keyed by
|
||||
# BatchDescriptor which already encodes batch shape / uniformity.
|
||||
# runtime_mode=None matches any non-NONE dispatch mode; entries
|
||||
# are keyed by BatchDescriptor, which already encodes batch
|
||||
# shape / uniformity.
|
||||
self.runnable = runnable
|
||||
self.vllm_config = vllm_config
|
||||
self.runtime_mode = runtime_mode
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.graph_pool = current_platform.get_global_graph_pool()
|
||||
self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG"
|
||||
@@ -298,7 +301,9 @@ class BreakableCUDAGraphWrapper:
|
||||
raise AttributeError(key)
|
||||
|
||||
def unwrap(self) -> Callable[..., Any]:
|
||||
return self.runnable
|
||||
# Recurse through nested wrappers to reach the original runnable.
|
||||
runnable = self.runnable
|
||||
return runnable.unwrap() if hasattr(runnable, "unwrap") else runnable
|
||||
|
||||
@property
|
||||
def cudagraph_wrapper(self) -> BreakableCUDAGraphWrapper:
|
||||
@@ -317,11 +322,13 @@ class BreakableCUDAGraphWrapper:
|
||||
batch_descriptor = forward_context.batch_descriptor
|
||||
cudagraph_runtime_mode = forward_context.cudagraph_runtime_mode
|
||||
|
||||
# Capture whenever the dispatcher says "some cudagraph mode" --
|
||||
# breakable produces the same artifact regardless of PIECEWISE
|
||||
# vs FULL, so we match either. Entries are keyed by batch
|
||||
# descriptor, which already encodes prefill/decode distinctions.
|
||||
if cudagraph_runtime_mode == CUDAGraphMode.NONE:
|
||||
# By default (runtime_mode=None) match any non-NONE mode; a set
|
||||
# runtime_mode restricts capture so other modes can be handled by
|
||||
# an outer wrapper (e.g. CUDAGraphWrapper for FULL).
|
||||
if cudagraph_runtime_mode == CUDAGraphMode.NONE or (
|
||||
self.runtime_mode is not None
|
||||
and cudagraph_runtime_mode != self.runtime_mode
|
||||
):
|
||||
return self.runnable(*args, **kwargs)
|
||||
|
||||
assert batch_descriptor is not None
|
||||
|
||||
@@ -221,7 +221,9 @@ class CUDAGraphWrapper:
|
||||
|
||||
def unwrap(self) -> Callable[..., Any]:
|
||||
# in case we need to access the original runnable.
|
||||
return self.runnable
|
||||
runnable = self.runnable
|
||||
# Recurse through nested wrappers.
|
||||
return runnable.unwrap() if hasattr(runnable, "unwrap") else runnable
|
||||
|
||||
@property
|
||||
def cudagraph_wrapper(self) -> "CUDAGraphWrapper":
|
||||
|
||||
Reference in New Issue
Block a user