[Bugfix] VLLM_V1 supports passing other compilation levels (#19340)
Signed-off-by: Richard Zou <zou3519@gmail.com>
This commit is contained in:
@@ -43,7 +43,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput
|
||||
from vllm.tasks import GenerationTask, PoolingTask, SupportedTask
|
||||
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
|
||||
GiB_bytes, LazyLoader, check_use_alibi, get_dtype_size,
|
||||
is_pin_memory_available, round_up)
|
||||
is_pin_memory_available, round_up, supports_dynamo)
|
||||
from vllm.v1.attention.backends.mamba_selectors import get_mamba_attn_backend
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
AttentionMetadataBuilder, CommonAttentionMetadata,
|
||||
@@ -1930,6 +1930,17 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
rank_mapping,
|
||||
)
|
||||
|
||||
if (
|
||||
self.vllm_config.compilation_config.level == \
|
||||
CompilationLevel.DYNAMO_AS_IS and supports_dynamo()
|
||||
):
|
||||
backend = self.vllm_config.compilation_config.init_backend(
|
||||
self.vllm_config)
|
||||
compilation_counter.dynamo_as_is_count += 1
|
||||
self.model.compile(
|
||||
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
|
||||
backend=backend)
|
||||
|
||||
def reload_weights(self) -> None:
|
||||
assert getattr(self, "model", None) is not None, \
|
||||
"Cannot reload weights before model is loaded."
|
||||
|
||||
Reference in New Issue
Block a user