[Model] Fix KV cache layout and optimize Dots3 NOTE Omni encoders (#53460)

Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
This commit is contained in:
范裕达
2026-08-23 10:02:35 -07:00
committed by GitHub
parent 6cd9713463
commit 185cada36b
5 changed files with 210 additions and 89 deletions
+22 -2
View File
@@ -3,6 +3,7 @@
"""vLLM composition layer for Dots3Note image and audio encoders."""
from collections.abc import Iterable
from contextlib import contextmanager
import torch
from torch import nn
@@ -41,6 +42,21 @@ from .model import Dots3NoteLanguageModelForCausalLM
from .vision import DotsMoEVitConfig, DotsMoEVitModel
@contextmanager
def _skip_linear_init():
"""Skip initialization for VE Linear weights loaded immediately afterward."""
original_reset_parameters = nn.Linear.reset_parameters
def _noop_reset_parameters(_linear: nn.Linear) -> None:
return None
nn.Linear.reset_parameters = _noop_reset_parameters
try:
yield
finally:
nn.Linear.reset_parameters = original_reset_parameters
@MULTIMODAL_REGISTRY.register_processor(
Dots3NoteMultiModalProcessor,
info=Dots3NoteProcessingInfo,
@@ -113,7 +129,10 @@ class Dots3NoteForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self.visual: DotsMoEVitModel | None
if image_enabled:
assert vision_config_dict is not None
self.visual = DotsMoEVitModel(DotsMoEVitConfig(**vision_config_dict))
with _skip_linear_init():
self.visual = DotsMoEVitModel(
DotsMoEVitConfig(**vision_config_dict)
)
else:
self.visual = None
self.audio_tower: Dots3NoteAudioModel | None
@@ -158,9 +177,10 @@ class Dots3NoteForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
) -> tuple[torch.Tensor, ...]:
if self.visual is None:
return ()
image_embeds = self.visual(pixel_values, image_grid_thw)
merge_size = self.visual.spatial_merge_size
sizes = (image_grid_thw.prod(-1) // merge_size**2).tolist()
meta = self.visual.prepare_meta(image_grid_thw)
image_embeds = self.visual(pixel_values, meta)
return image_embeds.split(sizes)
def _process_audio_input(