[Model] Fix KV cache layout and optimize Dots3 NOTE Omni encoders (#53460)
Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
"""vLLM composition layer for Dots3Note image and audio encoders."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from contextlib import contextmanager
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -41,6 +42,21 @@ from .model import Dots3NoteLanguageModelForCausalLM
|
||||
from .vision import DotsMoEVitConfig, DotsMoEVitModel
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _skip_linear_init():
|
||||
"""Skip initialization for VE Linear weights loaded immediately afterward."""
|
||||
original_reset_parameters = nn.Linear.reset_parameters
|
||||
|
||||
def _noop_reset_parameters(_linear: nn.Linear) -> None:
|
||||
return None
|
||||
|
||||
nn.Linear.reset_parameters = _noop_reset_parameters
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
nn.Linear.reset_parameters = original_reset_parameters
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
Dots3NoteMultiModalProcessor,
|
||||
info=Dots3NoteProcessingInfo,
|
||||
@@ -113,7 +129,10 @@ class Dots3NoteForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
self.visual: DotsMoEVitModel | None
|
||||
if image_enabled:
|
||||
assert vision_config_dict is not None
|
||||
self.visual = DotsMoEVitModel(DotsMoEVitConfig(**vision_config_dict))
|
||||
with _skip_linear_init():
|
||||
self.visual = DotsMoEVitModel(
|
||||
DotsMoEVitConfig(**vision_config_dict)
|
||||
)
|
||||
else:
|
||||
self.visual = None
|
||||
self.audio_tower: Dots3NoteAudioModel | None
|
||||
@@ -158,9 +177,10 @@ class Dots3NoteForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if self.visual is None:
|
||||
return ()
|
||||
image_embeds = self.visual(pixel_values, image_grid_thw)
|
||||
merge_size = self.visual.spatial_merge_size
|
||||
sizes = (image_grid_thw.prod(-1) // merge_size**2).tolist()
|
||||
meta = self.visual.prepare_meta(image_grid_thw)
|
||||
image_embeds = self.visual(pixel_values, meta)
|
||||
return image_embeds.split(sizes)
|
||||
|
||||
def _process_audio_input(
|
||||
|
||||
Reference in New Issue
Block a user