[Model] Fix KV cache layout and optimize Dots3 NOTE Omni encoders (#53460)

Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
This commit is contained in:
范裕达
2026-08-23 10:02:35 -07:00
committed by GitHub
parent 6cd9713463
commit 185cada36b
5 changed files with 210 additions and 89 deletions
+55 -61
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import math
from dataclasses import dataclass
from typing import Any
import torch
@@ -15,10 +16,11 @@ from vllm.third_party.deep_gemm import per_block_cast_to_fp8
from .vision_attention import (
VisionRotaryEmbedding,
VisionRotaryPositionEmbedding,
apply_vision_attention_residual,
attn_uses_seqlens,
build_vision_attention,
prepare_seqlens_for_attention,
prepare_rotary_pos_emb_vision,
resolve_attn_implementation,
)
from .vision_moe import note_vision_fused_moe_fp8
@@ -502,6 +504,15 @@ _ADAPTER_CLASSES: dict[str, type[nn.Module]] = {
}
@dataclass(frozen=True)
class VitForwardMeta:
cu_seqlens: torch.Tensor
max_seqlen: int
rotary_pos_emb: VisionRotaryPositionEmbedding
grid_thw_cpu: torch.Tensor
seqlens: list[int] | None = None
# ---- Full Model ----
@@ -562,7 +573,7 @@ class DotsMoEVitModel(PreTrainedModel):
block.norm_1 = torch.compile(block.norm_1, **compile_kwargs)
block.norm_2 = torch.compile(block.norm_2, **compile_kwargs)
def get_pos_ids_by_grid(self, grid_thw):
def get_pos_ids_by_grid(self, grid_thw_cpu: torch.Tensor):
# Mirrors ``cybertron`` ``AIMv2NativeModel.rot_pos_emb``: when ``pre_pixel_shuffle``
# is set, RoPE positions follow the qwen ``merge_size`` grouped layout (default 2x2);
# otherwise positions are flat row-major regardless of ``spatial_merge_size``.
@@ -573,10 +584,8 @@ class DotsMoEVitModel(PreTrainedModel):
else:
rope_merge_size = 1
pos_ids = []
for t, h, w in grid_thw:
hpos_ids = (
torch.arange(h, device=grid_thw.device).unsqueeze(1).expand(-1, w)
)
for t, h, w in grid_thw_cpu:
hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
hpos_ids = hpos_ids.reshape(
h // rope_merge_size,
rope_merge_size,
@@ -585,9 +594,7 @@ class DotsMoEVitModel(PreTrainedModel):
)
hpos_ids = hpos_ids.permute(0, 2, 1, 3).flatten()
wpos_ids = (
torch.arange(w, device=grid_thw.device).unsqueeze(0).expand(h, -1)
)
wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
wpos_ids = wpos_ids.reshape(
h // rope_merge_size,
rope_merge_size,
@@ -598,80 +605,67 @@ class DotsMoEVitModel(PreTrainedModel):
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
return pos_ids
def rot_pos_emb(self, grid_thw):
pos_ids = self.get_pos_ids_by_grid(grid_thw)
pos_ids = torch.cat(pos_ids, dim=0)
max_grid_size = grid_thw[:, 1:].max()
def rot_pos_emb(self, grid_thw_cpu: torch.Tensor):
pos_ids = torch.cat(self.get_pos_ids_by_grid(grid_thw_cpu), dim=0)
pos_ids = pos_ids.to(self.device, non_blocking=True)
max_grid_size = int(grid_thw_cpu[:, 1:].max())
rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
return rotary_pos_emb
return rotary_pos_emb_full[pos_ids].flatten(1)
def _build_cu_seqlens_from_grid(self, grid_thw: torch.Tensor):
cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(
dim=0,
dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,
)
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
# Same as (cu_seqlens[1:] - cu_seqlens[:-1]).max(); computed here to avoid D2H inside attention.
max_seqlen = int((grid_thw[:, 1] * grid_thw[:, 2]).max().item())
return cu_seqlens, max_seqlen
def prepare_meta(self, grid_thw_cpu: torch.Tensor) -> VitForwardMeta:
if grid_thw_cpu.device.type != "cpu":
raise ValueError("NOTE vision grid_thw must remain on CPU")
def _build_single_temporal_cu_seqlens_from_grid(self, grid_thw: torch.Tensor):
seq_lens = grid_thw[:, 1] * grid_thw[:, 2]
cu_seqlens = torch.empty(
(grid_thw.shape[0] + 1,), device=grid_thw.device, dtype=torch.int32
seq_per_frame = grid_thw_cpu[:, 1] * grid_thw_cpu[:, 2]
frame_lengths = torch.repeat_interleave(seq_per_frame, grid_thw_cpu[:, 0])
cu_seqlens_cpu = F.pad(
frame_lengths.cumsum(dim=0, dtype=torch.int32), (1, 0), value=0
)
resolved_attn = resolve_attn_implementation(
self.config.attn_implementation, eager_fallback="eager_v2"
)
return VitForwardMeta(
cu_seqlens=cu_seqlens_cpu.to(self.device, non_blocking=True),
max_seqlen=int(seq_per_frame.max()),
rotary_pos_emb=prepare_rotary_pos_emb_vision(
self.rot_pos_emb(grid_thw_cpu)
),
grid_thw_cpu=grid_thw_cpu,
seqlens=(
frame_lengths.tolist() if attn_uses_seqlens(resolved_attn) else None
),
)
cu_seqlens[0] = 0
torch.cumsum(seq_lens, dim=0, dtype=torch.int32, out=cu_seqlens[1:])
max_seqlen = int(seq_lens.max().item())
return cu_seqlens, max_seqlen
def forward(
self, hidden_states: torch.Tensor, grid_thw: torch.Tensor, bf16=True
self, hidden_states: torch.Tensor, meta: VitForwardMeta, bf16=True
) -> torch.Tensor:
grid_thw = grid_thw.to(device=self.device)
if bf16:
hidden_states = hidden_states.to(device=self.device, dtype=self.dtype)
hidden_states = self.patch_embed(hidden_states)
rotary_pos_emb = self.rot_pos_emb(grid_thw)
if grid_thw[:, 0].sum().item() == grid_thw.shape[0]:
cu_seqlens, max_seqlen = self._build_single_temporal_cu_seqlens_from_grid(
grid_thw
hidden_states = hidden_states.to(
device=self.device, dtype=self.dtype, non_blocking=True
)
else:
cu_seqlens, max_seqlen = self._build_cu_seqlens_from_grid(grid_thw)
seqlens = prepare_seqlens_for_attention(
self.config.attn_implementation,
cu_seqlens,
eager_fallback="eager_v2",
)
hidden_states = self.patch_embed(hidden_states)
for blk in self.blocks:
if self.gradient_checkpointing and self.training:
hidden_states = self._gradient_checkpointing_func(
blk.__call__,
hidden_states,
cu_seqlens,
rotary_pos_emb,
max_seqlen,
seqlens,
meta.cu_seqlens,
meta.rotary_pos_emb,
meta.max_seqlen,
meta.seqlens,
)
else:
hidden_states = blk(
hidden_states,
cu_seqlens,
rotary_pos_emb,
max_seqlen,
seqlens=seqlens,
meta.cu_seqlens,
meta.rotary_pos_emb,
meta.max_seqlen,
seqlens=meta.seqlens,
)
if self.config.post_norm:
hidden_states = self.post_trunk_norm(hidden_states)
hidden_states = self.adapter(hidden_states, grid_thw)
hidden_states = self.adapter(hidden_states, meta.grid_thw_cpu)
return hidden_states