[Performance] Optimize Dots3 NOTE runtime (#53517)

Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
范裕达
2026-08-31 00:05:18 -07:00
committed by GitHub
co-authored by Jee Jee Li Jee Jee Li
parent 7292ee2791
commit da0b2d8b17
6 changed files with 94 additions and 85 deletions
+7 -21
View File
@@ -15,6 +15,7 @@ from transformers.modeling_utils import PreTrainedModel
from vllm.third_party.deep_gemm import per_block_cast_to_fp8
from .vision_attention import (
VisionRMSNorm,
VisionRotaryEmbedding,
VisionRotaryPositionEmbedding,
apply_vision_attention_residual,
@@ -107,23 +108,6 @@ class DotsMoEVitConfig(PretrainedConfig):
self.enable_fp8_moe = enable_fp8_moe
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
output = self._norm(x.float()).type_as(x)
return output * self.weight
def extra_repr(self) -> str:
return f"{tuple(self.weight.shape)}, eps={self.eps}"
def _norm(self, x: torch.Tensor) -> torch.Tensor:
return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
# ---- FFN modules ----
@@ -333,7 +317,7 @@ class DotsPatchEmbed(nn.Module):
kernel_size=(config.patch_size, config.patch_size),
stride=(config.patch_size, config.patch_size),
)
self.norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)
self.norm = VisionRMSNorm(config.embed_dim, eps=config.rms_norm_eps)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x.view(
@@ -359,8 +343,8 @@ class MoEVisionBlock(nn.Module):
)
self.attn = build_vision_attention(attn_impl, config, eager_fallback="eager_v2")
self._attn_uses_seqlens = attn_uses_seqlens(attn_impl)
self.norm_1 = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)
self.norm_2 = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)
self.norm_1 = VisionRMSNorm(config.embed_dim, eps=config.rms_norm_eps)
self.norm_2 = VisionRMSNorm(config.embed_dim, eps=config.rms_norm_eps)
is_moe = (
config.pyramid_num_routed
@@ -534,7 +518,9 @@ class DotsMoEVitModel(PreTrainedModel):
)
if config.post_norm:
self.post_trunk_norm = RMSNorm(config.embed_dim, eps=config.rms_norm_eps)
self.post_trunk_norm = VisionRMSNorm(
config.embed_dim, eps=config.rms_norm_eps
)
adapter_cls = _ADAPTER_CLASSES.get(config.adapter_type)
if adapter_cls is None: