Fix Kimi-K2.5 FlashInfer ViT metadata (#44493)

Signed-off-by: Kevin-XiongC <kevin_xiong1997@outlook.com>
This commit is contained in:
Kevin_Xiong
2026-06-04 08:14:35 +00:00
committed by GitHub
parent a6183563b6
commit 1bdc60ed53
2 changed files with 109 additions and 28 deletions
+1 -1
View File
@@ -235,7 +235,7 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo])
pixel_values=MultiModalFieldConfig.flat_from_sizes(
"vision_chunk", grid_sizes
),
grid_thws=MultiModalFieldConfig.batched("vision_chunk"),
grid_thws=MultiModalFieldConfig.batched("vision_chunk", keep_on_cpu=True),
)
def _call_hf_processor(
+108 -27
View File
@@ -154,9 +154,12 @@ class Learnable2DInterpPosEmbDivided_fixed(nn.Module):
def reset_parameters(self):
nn.init.normal_(self.weight)
def forward(self, x: torch.Tensor, grid_thws: torch.Tensor) -> torch.Tensor:
def forward(
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
) -> torch.Tensor:
pos_embs = []
for t, h, w in grid_thws.tolist():
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
for t, h, w in grid_thw_list:
assert t <= self.num_frames, f"t:{t} > self.num_frames:{self.num_frames}"
if (h, w) == self.weight.shape[:-1]:
pos_emb_2d = self.weight.flatten(end_dim=1)
@@ -218,7 +221,9 @@ class MoonVision3dPatchEmbed(nn.Module):
else:
raise NotImplementedError(f"Not support pos_emb_type: {pos_emb_type}")
def forward(self, x: torch.Tensor, grid_thws: torch.Tensor) -> torch.Tensor:
def forward(
self, x: torch.Tensor, grid_thws: torch.Tensor | list[list[int]]
) -> torch.Tensor:
x = self.proj(x).view(x.size(0), -1)
# apply positional embedding
x = self.pos_emb(x, grid_thws)
@@ -265,7 +270,7 @@ class Rope2DPosEmbRepeated(nn.Module):
return freqs_cis
def get_freqs_cis(
self, grid_thws: torch.Tensor, device: torch.device
self, grid_thws: torch.Tensor | list[list[int]], device: torch.device
) -> torch.Tensor:
"""
Args:
@@ -279,7 +284,7 @@ class Rope2DPosEmbRepeated(nn.Module):
"freqs_cis", self._precompute_freqs_cis(device), persistent=False
)
shapes = grid_thws.tolist()
shapes = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
assert all(
1 <= h <= self.max_height and 1 <= w <= self.max_width for t, h, w in shapes
), (
@@ -401,6 +406,8 @@ class MoonViTEncoderLayer(nn.Module):
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rope_freqs_cis: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
):
"""Compute self-attention with packed QKV.
@@ -422,13 +429,15 @@ class MoonViTEncoderLayer(nn.Module):
xq, xk = apply_rope(xq, xk, rope_freqs_cis)
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
if max_seqlen is None:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
attn_out = self.attn(
xq.unsqueeze(0),
xk.unsqueeze(0),
xv.unsqueeze(0),
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
attn_out = attn_out.reshape(
seq_length,
@@ -443,12 +452,18 @@ class MoonViTEncoderLayer(nn.Module):
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor,
rope_freqs_cis: torch.Tensor | None = None,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
):
residual = hidden_states
hidden_states = self.norm0(hidden_states)
hidden_states = self.attention_qkvpacked(
hidden_states, cu_seqlens, rope_freqs_cis
hidden_states,
cu_seqlens,
rope_freqs_cis,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
hidden_states = residual + hidden_states
@@ -493,27 +508,70 @@ class MoonViT3dEncoder(nn.Module):
)
self.final_layernorm = nn.LayerNorm(hidden_dim)
def prepare_encoder_metadata(
self,
grid_thw_list: list[list[int]],
*,
device: torch.device,
) -> dict[str, torch.Tensor | None]:
metadata: dict[str, torch.Tensor | None] = {}
metadata["rope_freqs_cis"] = self.rope_2d.get_freqs_cis(
grid_thw_list, device=device
)
grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
lengths = grid_thw_np[:, 0] * grid_thw_np[:, 1] * grid_thw_np[:, 2]
cu_seqlens = np.concatenate(
[np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)]
)
attn_backend = self.blocks[0].attn.attn_backend
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
attn_backend, cu_seqlens, device
)
metadata["max_seqlen"] = torch.tensor(
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens),
dtype=torch.int32,
)
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
attn_backend,
cu_seqlens,
self.blocks[0].hidden_dim,
self.blocks[0].tp_size,
device,
)
return metadata
def forward(
self,
hidden_states: torch.Tensor,
grid_thws: torch.Tensor,
grid_thws: torch.Tensor | list[list[int]],
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
rope_freqs_cis = self.rope_2d.get_freqs_cis(
grid_thws=grid_thws, device=hidden_states.device
)
lengths = torch.cat(
(
torch.zeros(1, dtype=grid_thws.dtype, device=grid_thws.device),
grid_thws[:, 0] * grid_thws[:, 1] * grid_thws[:, 2],
if encoder_metadata is None:
grid_thw_list = (
grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
)
encoder_metadata = self.prepare_encoder_metadata(
grid_thw_list, device=hidden_states.device
)
)
cu_seqlens = lengths.to(hidden_states.device).cumsum(dim=0, dtype=torch.int32)
rope_freqs_cis = encoder_metadata["rope_freqs_cis"]
cu_seqlens = encoder_metadata["cu_seqlens"]
max_seqlen = encoder_metadata["max_seqlen"]
sequence_lengths = encoder_metadata.get("sequence_lengths")
assert rope_freqs_cis is not None
assert cu_seqlens is not None
assert max_seqlen is not None
for block in self.blocks:
hidden_states = block(
hidden_states, cu_seqlens, rope_freqs_cis=rope_freqs_cis
hidden_states,
cu_seqlens,
rope_freqs_cis=rope_freqs_cis,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
hidden_states = self.final_layernorm(hidden_states)
@@ -523,16 +581,17 @@ class MoonViT3dEncoder(nn.Module):
def tpool_patch_merger(
x: torch.Tensor,
grid_thws: torch.Tensor,
grid_thws: torch.Tensor | list[list[int]],
merge_kernel_size: tuple[int, int] = (2, 2),
) -> list[torch.Tensor]:
"""Temporal pooling patch merger."""
kh, kw = merge_kernel_size
lengths = (grid_thws[:, 0] * grid_thws[:, 1] * grid_thws[:, 2]).tolist()
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
lengths = [t * h * w for t, h, w in grid_thw_list]
seqs = x.split(lengths, dim=0)
outputs = []
for seq, (t, h, w) in zip(seqs, grid_thws.tolist()):
for seq, (t, h, w) in zip(seqs, grid_thw_list):
nh, nw = h // kh, w // kw
# Reshape: (t*h*w, d) -> (t, nh, kh, nw, kw, d)
v = seq.view(t, nh, kh, nw, kw, -1)
@@ -589,7 +648,11 @@ class MoonViT3dPretrainedModel(nn.Module):
)
def forward(
self, pixel_values: torch.Tensor, grid_thws: torch.Tensor
self,
pixel_values: torch.Tensor,
grid_thws: torch.Tensor | list[list[int]],
*,
encoder_metadata: dict[str, torch.Tensor | None] | None = None,
) -> torch.Tensor:
"""
Args:
@@ -599,13 +662,23 @@ class MoonViT3dPretrainedModel(nn.Module):
Returns:
torch.Tensor: The output tokens.
"""
hidden_states = self.patch_embed(pixel_values, grid_thws)
hidden_states = self.encoder(hidden_states, grid_thws)
grid_thw_list = grid_thws if isinstance(grid_thws, list) else grid_thws.tolist()
if encoder_metadata is None:
encoder_metadata = self.encoder.prepare_encoder_metadata(
grid_thw_list, device=pixel_values.device
)
hidden_states = self.patch_embed(pixel_values, grid_thw_list)
hidden_states = self.encoder(
hidden_states,
grid_thw_list,
encoder_metadata=encoder_metadata,
)
if (
self.merge_type == "sd2_tpool"
): # spatial downsampling 2x with temporal pooling all
hidden_states = tpool_patch_merger(
hidden_states, grid_thws, merge_kernel_size=self.merge_kernel_size
hidden_states, grid_thw_list, merge_kernel_size=self.merge_kernel_size
)
else:
raise NotImplementedError(f"Not support {self.merge_type}")
@@ -649,7 +722,15 @@ def vision_tower_forward(
rope_type="rope_2d",
)
else:
vt_outputs = vision_tower(pixel_values, grid_thw)
grid_thw_list = grid_thw.tolist()
encoder_metadata = vision_tower.encoder.prepare_encoder_metadata(
grid_thw_list, device=pixel_values.device
)
vt_outputs = vision_tower(
pixel_values,
grid_thw_list,
encoder_metadata=encoder_metadata,
)
tensors = mm_projector_forward(mm_projector, list(vt_outputs))
return list(tensors)