[Perf] Set IR Op Priority Once at Worker Init (#42631)
Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.ir
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -87,6 +88,13 @@ class WorkerBase:
|
||||
self.device: torch.device | None = None
|
||||
self.model_runner: nn.Module | None = None
|
||||
|
||||
# IR op priority and torch-wrap state are constant for the worker's
|
||||
# lifetime.
|
||||
vllm_config.kernel_config.ir_op_priority.set_default()
|
||||
vllm.ir.set_default_torch_wrap(
|
||||
vllm_config.compilation_config.ir_enable_torch_wrap
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
|
||||
"""Get specifications for KV cache implementation."""
|
||||
raise NotImplementedError
|
||||
|
||||
Reference in New Issue
Block a user