[MM][CG][BugFix] Fix Ernie-4.5-VL encoder CG postprocess for multi-path outputs (#51461)
Signed-off-by: qyYue1389 <yueqiuyang1389@gmail.com>
This commit is contained in:
@@ -1721,7 +1721,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest,
|
||||
@@ -1731,6 +1731,8 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
# The graph output is the raw ViT features (pre-merge), padded to the
|
||||
# token budget. Run the resampler eagerly on the valid portion using
|
||||
# the actual batch grid_thw, then scatter the post-merge embeddings.
|
||||
# Ernie only uses the single "default" encoder path.
|
||||
output = outputs["default"]
|
||||
grid_thw = batch_mm_kwargs["image_grid_thw"].to(output.device)
|
||||
num_valid = int((grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).sum())
|
||||
image_embeds = self.resampler_model(output[:num_valid], grid_thw)
|
||||
|
||||
Reference in New Issue
Block a user