Add ability to replace oot ops when using lora (#37181)
Signed-off-by: Kyuyeun Kim <kyuyeunk@google.com>
This commit is contained in:
@@ -22,10 +22,11 @@ op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
|
||||
op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
|
||||
|
||||
|
||||
def get_oot_class_by_name(class_name: str) -> type | None:
|
||||
def maybe_get_oot_by_class(class_type: type) -> type:
|
||||
class_name = class_type.__name__
|
||||
if class_name in op_registry_oot:
|
||||
return op_registry_oot[class_name]
|
||||
return None
|
||||
return class_type
|
||||
|
||||
|
||||
class PluggableLayer(nn.Module):
|
||||
|
||||
@@ -6,7 +6,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp, get_oot_class_by_name
|
||||
from vllm.model_executor.custom_op import CustomOp, maybe_get_oot_by_class
|
||||
from vllm.model_executor.models.vision import get_vit_attn_backend
|
||||
from vllm.utils.math_utils import round_up
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
@@ -125,7 +125,7 @@ class MMEncoderAttention(CustomOp):
|
||||
cu_seqlens: np.ndarray,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
if (oot_class := get_oot_class_by_name(cls.__name__)) is not None:
|
||||
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
|
||||
return oot_class.maybe_compute_seq_lens(attn_backend, cu_seqlens, device) # type: ignore[attr-defined]
|
||||
|
||||
if attn_backend != AttentionBackendEnum.FLASHINFER:
|
||||
@@ -149,7 +149,7 @@ class MMEncoderAttention(CustomOp):
|
||||
tp_size: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
if (oot_class := get_oot_class_by_name(cls.__name__)) is not None:
|
||||
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
|
||||
return oot_class.maybe_recompute_cu_seqlens( # type: ignore[attr-defined]
|
||||
attn_backend, cu_seqlens, hidden_size, tp_size, device
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user