Signed-off-by: Lucas Kabela <lucaskabela@meta.com>
This commit is contained in:
@@ -10,7 +10,6 @@ from typing import Callable, Optional
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import (CompilationLevel, CUDAGraphMode,
|
||||
get_current_vllm_config)
|
||||
from vllm.logger import init_logger
|
||||
@@ -47,11 +46,10 @@ class TorchCompileWrapperWithCustomDispatcher:
|
||||
options = get_current_vllm_config(
|
||||
).compilation_config.inductor_compile_config
|
||||
|
||||
compiled_callable = torch.compile(
|
||||
self.forward,
|
||||
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
|
||||
backend=backend,
|
||||
options=options)
|
||||
compiled_callable = torch.compile(self.forward,
|
||||
fullgraph=True,
|
||||
backend=backend,
|
||||
options=options)
|
||||
|
||||
self.compiled_callable = compiled_callable
|
||||
self.original_code_object = self.__class__.forward.__code__
|
||||
|
||||
@@ -434,11 +434,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_FLASH_ATTN_VERSION":
|
||||
lambda: maybe_convert_int(os.environ.get("VLLM_FLASH_ATTN_VERSION", None)),
|
||||
|
||||
# Internal flag to enable Dynamo fullgraph capture
|
||||
"VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE":
|
||||
lambda: bool(
|
||||
os.environ.get("VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE", "1") != "0"),
|
||||
|
||||
# Feature flag to enable/disable Inductor standalone compile.
|
||||
# In torch <= 2.7 we ignore this flag; in torch >= 2.8 this is
|
||||
# enabled by default.
|
||||
|
||||
@@ -2602,9 +2602,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
backend = self.vllm_config.compilation_config.init_backend(
|
||||
self.vllm_config)
|
||||
compilation_counter.dynamo_as_is_count += 1
|
||||
self.model.compile(
|
||||
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
|
||||
backend=backend)
|
||||
self.model.compile(fullgraph=True, backend=backend)
|
||||
return
|
||||
# for other compilation levels, cudagraph behavior is controlled by
|
||||
# CudagraphWraper and CudagraphDispatcher of vllm.
|
||||
|
||||
@@ -18,7 +18,6 @@ import torch.distributed
|
||||
import torch.nn as nn
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.attention import AttentionMetadata, get_attn_backend
|
||||
from vllm.attention.backends.abstract import AttentionState
|
||||
from vllm.attention.backends.utils import CommonAttentionState
|
||||
@@ -1099,10 +1098,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
|
||||
backend = self.vllm_config.compilation_config.init_backend(
|
||||
self.vllm_config)
|
||||
compilation_counter.dynamo_as_is_count += 1
|
||||
self.model = torch.compile(
|
||||
self.model,
|
||||
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
|
||||
backend=backend)
|
||||
self.model = torch.compile(self.model,
|
||||
fullgraph=True,
|
||||
backend=backend)
|
||||
|
||||
def get_model(self) -> nn.Module:
|
||||
return self.model
|
||||
|
||||
Reference in New Issue
Block a user