Fix GLM-4.1V video placeholder token ID handling. (#49484)
Signed-off-by: aarushjain29 <Aarushi.Jain2@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
7e51939e25
commit
c064fa52b6
@@ -93,7 +93,6 @@ def test_processor_override(
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
hf_processor_mm_kwargs = {"fps": fps}
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
@@ -112,8 +111,8 @@ def test_processor_override(
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
video_token_id = tokenizer.convert_tokens_to_ids(hf_processor.video_token)
|
||||
video_tok_count = processed_inputs["prompt_token_ids"].count(video_token_id)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
assert grid_t == expected_grid_t
|
||||
|
||||
@@ -1403,6 +1403,11 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
||||
timestamps_list = full_second_idxs[::2]
|
||||
return list(timestamps_list)
|
||||
|
||||
def _get_video_frame_embed_token_id(self, hf_processor: object) -> int:
|
||||
if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA:
|
||||
return hf_processor.image_token_id
|
||||
return hf_processor.video_token_id
|
||||
|
||||
def _construct_video_placeholder(
|
||||
self,
|
||||
video_array: np.ndarray,
|
||||
@@ -1440,13 +1445,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
||||
num_tokens_per_frame = int(H * W) // merge_length
|
||||
placeholder = []
|
||||
placeholder.append(bov_token_id)
|
||||
# Glm46VProcessor uses image_token_id for video frame embeddings;
|
||||
# Glm4vProcessor uses video_token_id.
|
||||
frame_embed_token_id = (
|
||||
hf_processor.video_token_id
|
||||
if isinstance(hf_processor, Glm4vProcessor) or not TRANSFORMERS_WITH_GA
|
||||
else hf_processor.image_token_id
|
||||
)
|
||||
frame_embed_token_id = self._get_video_frame_embed_token_id(hf_processor)
|
||||
for frame_idx in frames_idx_token:
|
||||
placeholder.append(boi_token_id)
|
||||
placeholder.extend([frame_embed_token_id] * num_tokens_per_frame)
|
||||
@@ -1604,11 +1603,6 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
|
||||
# Glm46VProcessor and GLMGA handle image/video placeholders together
|
||||
# via the direct path. Only Glm4vProcessor (GLM-4.1V) needs the
|
||||
# split-video path because it uses image_token_id as the video
|
||||
# placeholder. The direct path requires transformers >= 5.5.0
|
||||
# (Glm46VProcessor / GlmgaVideoProcessor support).
|
||||
use_direct_path = (
|
||||
not isinstance(processor, Glm4vProcessor) and TRANSFORMERS_WITH_GA
|
||||
)
|
||||
@@ -1630,6 +1624,8 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
):
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
|
||||
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
|
||||
for item in mm_data.pop("videos", []):
|
||||
video_array, metadata = item
|
||||
|
||||
@@ -1650,9 +1646,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
input_ids = video_outputs.pop("input_ids")
|
||||
input_ids[input_ids == processor.image_token_id] = (
|
||||
processor.video_token_id
|
||||
)
|
||||
if swap_video_frame_tokens:
|
||||
input_ids[input_ids == processor.image_token_id] = (
|
||||
processor.video_token_id
|
||||
)
|
||||
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace(
|
||||
"<|begin_of_video|><|video|><|end_of_video|>",
|
||||
@@ -1668,6 +1665,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
)
|
||||
else:
|
||||
video_outputs = dict()
|
||||
swap_video_frame_tokens = False
|
||||
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
@@ -1675,6 +1673,10 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
if swap_video_frame_tokens:
|
||||
input_ids = processed_outputs["input_ids"]
|
||||
input_ids[input_ids == processor.video_token_id] = processor.image_token_id
|
||||
processed_outputs["input_ids"] = input_ids
|
||||
combined_outputs = dict(
|
||||
processed_outputs,
|
||||
**video_outputs,
|
||||
@@ -1701,7 +1703,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
|
||||
merge_length = image_processor.merge_size**2
|
||||
|
||||
def get_image_replacement_glm4v(item_idx: int):
|
||||
def get_image_replacement(item_idx: int):
|
||||
out_item = out_mm_kwargs["image"][item_idx]
|
||||
grid_thw = out_item["image_grid_thw"].data
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
@@ -1709,7 +1711,7 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
num_tokens = int(grid_thw.prod()) // merge_length
|
||||
return [hf_processor.image_token_id] * num_tokens
|
||||
|
||||
def get_video_replacement_glm4v(item_idx: int):
|
||||
def get_video_replacement(item_idx: int):
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
grid_thw = out_item["video_grid_thw"].data
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
@@ -1720,39 +1722,19 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
placeholder,
|
||||
embed_token_id=hf_processor.video_token_id,
|
||||
embed_token_id=self.info._get_video_frame_embed_token_id(hf_processor),
|
||||
)
|
||||
|
||||
def get_video_replacement_glm46v(item_idx: int):
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
grid_thw = out_item["video_grid_thw"].data
|
||||
assert isinstance(grid_thw, torch.Tensor)
|
||||
|
||||
video, metadata = mm_items["video"][item_idx]
|
||||
placeholder = self.info._construct_video_placeholder(
|
||||
video, metadata, grid_thw
|
||||
)
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
placeholder,
|
||||
embed_token_id=hf_processor.image_token_id,
|
||||
)
|
||||
|
||||
is_glm46v = not isinstance(hf_processor, Glm4vProcessor)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=hf_processor.image_token,
|
||||
replacement=get_image_replacement_glm4v,
|
||||
replacement=get_image_replacement,
|
||||
),
|
||||
PromptReplacement(
|
||||
modality="video",
|
||||
target="<|begin_of_video|><|video|><|end_of_video|>",
|
||||
replacement=(
|
||||
get_video_replacement_glm46v
|
||||
if is_glm46v and TRANSFORMERS_WITH_GA
|
||||
else get_video_replacement_glm4v
|
||||
),
|
||||
replacement=get_video_replacement,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user