[Bugfix][V1] Fix molmo text-only inputs (#11676)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -1081,45 +1081,25 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
|
||||
else:
|
||||
out = processor.process(None, image, tokens=inputs["prompt_token_ids"])
|
||||
|
||||
# If there is no image, return directly.
|
||||
if image is None:
|
||||
new_prompt_token_ids = out["input_ids"].tolist()
|
||||
prompt = inputs.get("prompt")
|
||||
if prompt is None:
|
||||
prompt = tokenizer.decode(new_prompt_token_ids)
|
||||
return token_inputs(
|
||||
prompt_token_ids=new_prompt_token_ids,
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
image_processor = processor.image_processor
|
||||
max_total_crops = 1 + image_processor.max_crops
|
||||
if image is not None:
|
||||
images, image_input_idx, image_masks = pad_images(
|
||||
max_total_crops,
|
||||
out["images"],
|
||||
out["image_input_idx"],
|
||||
out.get("image_masks"),
|
||||
)
|
||||
else:
|
||||
base_image_input_size = image_processor.base_image_input_size
|
||||
image_patch_size = image_processor.image_patch_size
|
||||
image_num_patch = (
|
||||
base_image_input_size[0] // image_patch_size,
|
||||
base_image_input_size[1] // image_patch_size,
|
||||
)
|
||||
n_pixels = image_patch_size * image_patch_size * 3
|
||||
n_patches = image_num_patch[0] * image_num_patch[1]
|
||||
|
||||
image_length_w = image_processor.image_token_length_w
|
||||
image_length_h = image_processor.image_token_length_h
|
||||
tokens_per_image = image_length_w * image_length_h
|
||||
images = torch.full(
|
||||
(max_total_crops, n_patches, n_pixels),
|
||||
-1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
image_input_idx = torch.full(
|
||||
(max_total_crops, tokens_per_image),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if image_processor.image_padding_mask:
|
||||
image_masks = torch.full(
|
||||
(max_total_crops, n_patches),
|
||||
-1,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
images, image_input_idx, image_masks = pad_images(
|
||||
max_total_crops,
|
||||
out["images"],
|
||||
out["image_input_idx"],
|
||||
out.get("image_masks"),
|
||||
)
|
||||
image_data = dict(
|
||||
images=images,
|
||||
image_input_idx=image_input_idx,
|
||||
@@ -1143,11 +1123,9 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
|
||||
offset = i
|
||||
size += 1
|
||||
image_data["image_start_end"] = (offset, offset + size)
|
||||
|
||||
prompt = inputs.get("prompt")
|
||||
if prompt is None:
|
||||
prompt = tokenizer.decode(new_prompt_token_ids)
|
||||
|
||||
return token_inputs(
|
||||
prompt_token_ids=new_prompt_token_ids,
|
||||
prompt=prompt,
|
||||
|
||||
Reference in New Issue
Block a user