Add ability to replace oot ops when using lora (#37181)

Signed-off-by: Kyuyeun Kim <kyuyeunk@google.com>
This commit is contained in:
Kyuyeun Kim
2026-03-16 18:04:15 -07:00
committed by GitHub
parent 6c1cfbad32
commit 0a0a1a198b
6 changed files with 16 additions and 11 deletions
+3 -2
View File
@@ -22,10 +22,11 @@ op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
def get_oot_class_by_name(class_name: str) -> type | None:
def maybe_get_oot_by_class(class_type: type) -> type:
class_name = class_type.__name__
if class_name in op_registry_oot:
return op_registry_oot[class_name]
return None
return class_type
class PluggableLayer(nn.Module):
@@ -6,7 +6,7 @@ import numpy as np
import torch
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp, get_oot_class_by_name
from vllm.model_executor.custom_op import CustomOp, maybe_get_oot_by_class
from vllm.model_executor.models.vision import get_vit_attn_backend
from vllm.utils.math_utils import round_up
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
@@ -125,7 +125,7 @@ class MMEncoderAttention(CustomOp):
cu_seqlens: np.ndarray,
device: torch.device,
) -> torch.Tensor | None:
if (oot_class := get_oot_class_by_name(cls.__name__)) is not None:
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
return oot_class.maybe_compute_seq_lens(attn_backend, cu_seqlens, device) # type: ignore[attr-defined]
if attn_backend != AttentionBackendEnum.FLASHINFER:
@@ -149,7 +149,7 @@ class MMEncoderAttention(CustomOp):
tp_size: int,
device: torch.device,
) -> torch.Tensor:
if (oot_class := get_oot_class_by_name(cls.__name__)) is not None:
if (oot_class := maybe_get_oot_by_class(cls)) is not cls:
return oot_class.maybe_recompute_cu_seqlens( # type: ignore[attr-defined]
attn_backend, cu_seqlens, hidden_size, tp_size, device
)