From 412414d8e081e7057768714d37670728dc8cb363 Mon Sep 17 00:00:00 2001 From: Tiezhen WANG <38108242+xianbaoqian@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:59:08 +0800 Subject: [PATCH] Remove PersimmonForCausalLM and FuyuForCausalLM model architectures (#48096) Signed-off-by: Xianbao QIAN Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Cyrus Leung Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Cyrus Leung --- docs/configuration/conserving_memory.md | 2 +- docs/contributing/model/multimodal.md | 415 +++++------------- docs/models/supported_models.md | 2 - .../multimodal/vision_language_offline.py | 19 - tests/distributed/test_pipeline_parallel.py | 2 - .../multimodal/generation/test_common.py | 14 - .../generation/vlm_utils/model_utils.py | 9 - tests/models/registry.py | 2 - tests/renderers/test_hf.py | 1 - vllm/model_executor/models/fuyu.py | 368 ---------------- vllm/model_executor/models/persimmon.py | 373 ---------------- vllm/model_executor/models/registry.py | 4 +- .../chat_templates/registry.py | 1 - .../chat_templates/template_fuyu.jinja | 3 - 14 files changed, 120 insertions(+), 1095 deletions(-) delete mode 100644 vllm/model_executor/models/fuyu.py delete mode 100644 vllm/model_executor/models/persimmon.py delete mode 100644 vllm/transformers_utils/chat_templates/template_fuyu.jinja diff --git a/docs/configuration/conserving_memory.md b/docs/configuration/conserving_memory.md index 2c098118db..96a903bc31 100644 --- a/docs/configuration/conserving_memory.md +++ b/docs/configuration/conserving_memory.md @@ -42,7 +42,7 @@ and the maximum batch size (`max_num_seqs` option). ```python from vllm import LLM -llm = LLM(model="adept/fuyu-8b", max_model_len=2048, max_num_seqs=2) +llm = LLM(model="Qwen/Qwen2.5-VL-3B-Instruct", max_model_len=2048, max_num_seqs=2) ``` ## Reduce CUDA Graphs diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index b48258d539..33d89db75d 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -324,154 +324,44 @@ Assuming that the memory usage increases with the number of tokens, the dummy in return image_token * num_images ``` -=== "No input placeholders: Fuyu" +=== "No input placeholders: PaliGemma" - Looking at the code of HF's `FuyuForCausalLM`: - - ??? code - - ```python - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/modeling_fuyu.py#L311-L322 - if image_patches is not None and past_key_values is None: - patch_embeddings = [ - self.vision_embed_tokens(patch.to(self.vision_embed_tokens.weight.dtype)) - .squeeze(0) - .to(inputs_embeds.device) - for patch in image_patches - ] - inputs_embeds = self.gather_continuous_embeddings( - word_embeddings=inputs_embeds, - continuous_embeddings=patch_embeddings, - image_patch_input_indices=image_patches_indices, - ) - ``` - - The number of placeholder feature tokens for the `i`th item in the batch is `patch_embeddings[i].shape[0]`, - which is the same as `image_patches[i].shape[0]`, i.e. `num_total_patches`. - - Unlike LLaVA, Fuyu does not define the number of patches inside the modeling file. Where can we get more information? - Considering that the model input comes from the output of `FuyuProcessor`, let's **look at the preprocessing files**. - - The image outputs are obtained by calling `FuyuImageProcessor.preprocess` and then - `FuyuImageProcessor.preprocess_with_tokenizer_info` inside `FuyuProcessor`. - - In `FuyuImageProcessor.preprocess`, the images are resized and padded to the target `FuyuImageProcessor.size`, - returning the dimensions after resizing (but before padding) as metadata. - - ??? code - - ```python - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/processing_fuyu.py#L541-L544 - image_encoding = self.image_processor.preprocess(images, **output_kwargs["images_kwargs"]) - batch_images = image_encoding["images"] - image_unpadded_heights = image_encoding["image_unpadded_heights"] - image_unpadded_widths = image_encoding["image_unpadded_widths"] - - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L480-L - if do_resize: - batch_images = [ - [self.resize(image, size=size, input_data_format=input_data_format) for image in images] - for images in batch_images - ] - - image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images] - image_unpadded_heights = [[image_size[0]] for image_size in image_sizes] - image_unpadded_widths = [[image_size[1]] for image_size in image_sizes] - - if do_pad: - batch_images = [ - [ - self.pad_image( - image, - size=size, - mode=padding_mode, - constant_values=padding_value, - input_data_format=input_data_format, - ) - for image in images - ] - for images in batch_images - ] - ``` - - In `FuyuImageProcessor.preprocess_with_tokenizer_info`, the images are split into patches based on this metadata: - - ??? code - - ```python - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/processing_fuyu.py#L417-L425 - model_image_input = self.image_processor.preprocess_with_tokenizer_info( - image_input=tensor_batch_images, - image_present=image_present, - image_unpadded_h=image_unpadded_heights, - image_unpadded_w=image_unpadded_widths, - image_placeholder_id=image_placeholder_id, - image_newline_id=image_newline_id, - variable_sized=True, - ) - - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L638-L658 - image_height, image_width = image.shape[1], image.shape[2] - if variable_sized: # variable_sized=True - new_h = min( - image_height, - math.ceil(image_unpadded_h[batch_index, subseq_index] / patch_height) * patch_height, - ) - new_w = min( - image_width, - math.ceil(image_unpadded_w[batch_index, subseq_index] / patch_width) * patch_width, - ) - image = image[:, :new_h, :new_w] - image_height, image_width = new_h, new_w - - num_patches = self.get_num_patches(image_height=image_height, image_width=image_width) - tensor_of_image_ids = torch.full( - [num_patches], image_placeholder_id, dtype=torch.int32, device=image_input.device - ) - patches = self.patchify_image(image=image.unsqueeze(0)).squeeze(0) - assert num_patches == patches.shape[0] - ``` - - The number of patches is in turn defined by `FuyuImageProcessor.get_num_patches`: - - ??? code - - ```python - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L552-L562 - patch_size = patch_size if patch_size is not None else self.patch_size - patch_height, patch_width = self.patch_size["height"], self.patch_size["width"] - - if image_height % patch_height != 0: - raise ValueError(f"{image_height=} must be divisible by {patch_height}") - if image_width % patch_width != 0: - raise ValueError(f"{image_width=} must be divisible by {patch_width}") - - num_patches_per_dim_h = image_height // patch_height - num_patches_per_dim_w = image_width // patch_width - num_patches = num_patches_per_dim_h * num_patches_per_dim_w - ``` - - These image patches correspond to placeholder tokens (`|SPEAKER|`). So, we just need to maximize the number of image patches. Since input images are first resized - to fit within `image_processor.size`, we can maximize the number of image patches by inputting an image with size equal to `image_processor.size`. - - ```python - def get_image_size_with_most_features(self) -> ImageSize: - image_processor = self.get_image_processor() - return ImageSize( - width=image_processor.size["width"], - height=image_processor.size["height"], - ) - ``` - - Fuyu does not expect image placeholders in the inputs to HF processor, so - the dummy prompt text is empty regardless of the number of images. + Unlike LLaVA, PaliGemma's HF processor does not expect image placeholder + tokens in the input prompt; the placeholder feature tokens are instead + inserted afterwards (see [Prompt updates](#prompt-updates)). So the dummy + prompt text is empty regardless of the number of images: ```python def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: return "" ``` - For the multimodal image profiling data, the logic is very similar to LLaVA: + PaliGemma resizes every image to a square of `vision_config.image_size`, so + the number of placeholder feature tokens per image is fixed at + `(image_size // patch_size) ** 2`. This is computed by the SigLIP vision + encoder that PaliGemma uses: + + ??? code + + ```python + # vllm/model_executor/models/siglip.py + class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]): + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + ) -> int: + return self.get_patch_grid_length() ** 2 + + def get_patch_grid_length(self) -> int: + image_size, patch_size = self.get_image_size(), self.get_patch_size() + return image_size // patch_size + ``` + + Since the number of image tokens doesn't depend on the input image dimensions, + we can simply use a dummy image of the model's expected input size for the + multimodal profiling data: ??? code @@ -482,16 +372,18 @@ Assuming that the memory usage increases with the number of tokens, the dummy in mm_counts: Mapping[str, int], mm_options: Mapping[str, BaseDummyOptions], ) -> MultiModalDataDict: - target_width, target_height = \ - self.info.get_image_size_with_most_features() + hf_config = self.info.get_hf_config() + vision_config = hf_config.vision_config + max_image_size = vision_config.image_size + num_images = mm_counts.get("image", 0) image_overrides = mm_options.get("image") return { "image": self._get_dummy_images( - width=target_width, - height=target_height, + width=max_image_size, + height=max_image_size, num_images=num_images, overrides=image_overrides, ) @@ -545,28 +437,15 @@ return a schema of the tensors outputted by the HF processor that are related to Our [actual code](../../../vllm/model_executor/models/llava.py) additionally supports pre-computed image embeddings, which can be passed to be model via the `image_embeds` argument. -=== "With postprocessing: Fuyu" +=== "With postprocessing: Mistral3" - The `image_patches` output of `FuyuImageProcessor.preprocess_with_tokenizer_info` concatenates - the patches from each image belonging to an item in the batch: + The `pixel_values` output of Mistral3's HF processor pads every image in the + batch to a common size, so that they can be stacked into a single tensor. - ```python - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L673-L679 - image_input_ids.append(tensor_of_image_ids) - image_patches.append(patches) - else: - image_input_ids.append(torch.tensor([], dtype=torch.int32, device=image_input.device)) - - batch_image_input_ids.append(image_input_ids) - batch_image_patches.append(image_patches) - ``` - - The shape of `image_patches` outputted by `FuyuImageProcessor` is therefore - `(1, num_images, num_patches, patch_width * patch_height * num_channels)`. - - In order to support the use of - [MultiModalFieldConfig.batched][vllm.multimodal.inputs.MultiModalFieldConfig.batched] - like in LLaVA, we remove the extra batch dimension by overriding + To use [MultiModalFieldConfig.batched][vllm.multimodal.inputs.MultiModalFieldConfig.batched] + like in LLaVA, each image's features must be independent of the others (which + is also required for prefix caching to work correctly). So, we un-pad each image + back to its own size by overriding [BaseMultiModalProcessor._call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]: ??? code @@ -586,33 +465,27 @@ return a schema of the tensors outputted by the HF processor that are related to tok_kwargs=tok_kwargs, ) - image_patches = processed_outputs.get("image_patches") - if image_patches is not None: - images = mm_data["images"] - assert isinstance(images, list) + pixel_values = processed_outputs.get("pixel_values") + if pixel_values is not None: + # Avoid padding since we need the output for each image to be + # independent of other images for the cache to work correctly + image_sizes = processed_outputs["image_sizes"] + assert len(pixel_values) == len(image_sizes) - # Original output: (1, num_images, Pn, Px * Py * C) - # New output: (num_images, Pn, Px * Py * C) - assert (isinstance(image_patches, list) - and len(image_patches) == 1) - assert (isinstance(image_patches[0], torch.Tensor) - and len(image_patches[0]) == len(images)) - - processed_outputs["image_patches"] = image_patches[0] + processed_outputs["pixel_values"] = [ + p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes) + ] return processed_outputs ``` - !!! note - Our [actual code](../../../vllm/model_executor/models/fuyu.py) has special handling - for text-only inputs to prevent unnecessary warnings from HF processor. - !!! note The `_call_hf_processor` method specifies both `mm_kwargs` and `tok_kwargs` for processing. `mm_kwargs` is used to both initialize and call the huggingface processor, whereas `tok_kwargs` is only used to call the huggingface processor. - This lets us override [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: + Since `pixel_values` is now a list with one tensor per image, we can override + [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: ```python def _get_mm_fields_config( @@ -620,9 +493,15 @@ return a schema of the tensors outputted by the HF processor that are related to hf_inputs: BatchFeature, hf_processor_mm_kwargs: Mapping[str, object], ) -> Mapping[str, MultiModalFieldConfig]: - return dict(image_patches=MultiModalFieldConfig.batched("image")) + return dict( + pixel_values=MultiModalFieldConfig.batched("image"), + image_embeds=MultiModalFieldConfig.batched("image"), + ) ``` + !!! note + See our [actual code](../../../vllm/model_executor/models/mistral3.py) for the full implementation. + ### Prompt updates Override [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates] to @@ -678,121 +557,53 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies ] ``` -=== "Handling additional tokens: Fuyu" +=== "Handling additional tokens: PaliGemma" - Recall the layout of feature tokens from Step 2: - - ``` - |SPEAKER||SPEAKER|...|SPEAKER||NEWLINE| - |SPEAKER||SPEAKER|...|SPEAKER||NEWLINE| - ... - |SPEAKER||SPEAKER|...|SPEAKER||NEWLINE| - ``` - - We define a helper function to return `ncols` and `nrows` directly: + PaliGemma's HF processor inserts, after the prompt's leading `` token, a + run of image tokens followed by a second `` token that marks the start of + the text prompt. We start by building the run of image tokens, one per + placeholder feature token: ??? code ```python - def get_image_feature_grid_size( - self, - *, - image_width: int, - image_height: int, - ) -> tuple[int, int]: - image_processor = self.get_image_processor() - target_width = image_processor.size["width"] - target_height = image_processor.size["height"] - patch_width = image_processor.patch_size["width"] - patch_height = image_processor.patch_size["height"] - - if not (image_width <= target_width and image_height <= target_height): - height_scale_factor = target_height / image_height - width_scale_factor = target_width / image_width - optimal_scale_factor = min(height_scale_factor, width_scale_factor) - - image_height = int(image_height * optimal_scale_factor) - image_width = int(image_width * optimal_scale_factor) - - ncols = math.ceil(image_width / patch_width) - nrows = math.ceil(image_height / patch_height) - return ncols, nrows - ``` - - Based on this, we can initially define our replacement tokens as: - - ??? code - - ```python - def get_replacement(item_idx: int): - images = mm_items.get_items("image", ImageProcessorItems) - image_size = images.get_image_size(item_idx) - - ncols, nrows = self.info.get_image_feature_grid_size( - image_width=image_size.width, - image_height=image_size.height, + def get_insertion(item_idx: int): + images = mm_items.get_items( + "image", (ImageEmbeddingItems, ImageProcessorItems) ) - # `_IMAGE_TOKEN_ID` corresponds to `|SPEAKER|` - # `_NEWLINE_TOKEN_ID` corresponds to `|NEWLINE|` - return ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows + if isinstance(images, ImageEmbeddingItems): + num_image_tokens = images.get_feature_size(item_idx) + else: + image_size = images.get_image_size(item_idx) + num_image_tokens = self.info.get_num_image_tokens( + image_width=image_size.width, + image_height=image_size.height, + ) + + image_tokens = [image_token_id] * num_image_tokens + ... ``` - However, this is not entirely correct. After `FuyuImageProcessor.preprocess_with_tokenizer_info` is called, - a BOS token (``) is also added to the prompt: + The trailing `` token is an additional token that must **not** receive a + vision embedding. To assign the vision embeddings to only the image tokens, + instead of returning the token ids directly you can return an instance of + [PromptUpdateDetails][vllm.multimodal.processing.PromptUpdateDetails] and mark + the embedding tokens with `embed_token_id`: ??? code ```python - # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/processing_fuyu.py#L417-L435 - model_image_input = self.image_processor.preprocess_with_tokenizer_info( - image_input=tensor_batch_images, - image_present=image_present, - image_unpadded_h=image_unpadded_heights, - image_unpadded_w=image_unpadded_widths, - image_placeholder_id=image_placeholder_id, - image_newline_id=image_newline_id, - variable_sized=True, - ) - prompt_tokens, prompts_length = _tokenize_prompts_with_image_and_batch( - tokenizer=self.tokenizer, - prompts=prompts, - scale_factors=scale_factors, - max_tokens_to_generate=self.max_tokens_to_generate, - max_position_embeddings=self.max_position_embeddings, - add_BOS=True, - add_beginning_of_answer_token=True, + return PromptUpdateDetails.select_token_id( + image_tokens + [bos_token_id], + embed_token_id=image_token_id, ) ``` - To assign the vision embeddings to only the image tokens, instead of a string - you can return an instance of [PromptUpdateDetails][vllm.multimodal.processing.PromptUpdateDetails]: - - ??? code - - ```python - hf_config = self.info.get_hf_config() - bos_token_id = hf_config.bos_token_id # `` - assert isinstance(bos_token_id, int) - - def get_replacement_fuyu(item_idx: int): - images = mm_items.get_items("image", ImageProcessorItems) - image_size = images.get_image_size(item_idx) - - ncols, nrows = self.info.get_image_feature_grid_size( - image_width=image_size.width, - image_height=image_size.height, - ) - image_tokens = ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows - - return PromptUpdateDetails.select_token_id( - image_tokens + [bos_token_id], - embed_token_id=_IMAGE_TOKEN_ID, - ) - ``` - - Finally, noticing that the HF processor removes the `|ENDOFTEXT|` token from the tokenized prompt, - we can search for it to conduct the replacement at the start of the string: + Putting it together, we override [_get_prompt_updates][vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates]. + Since these tokens are inserted (rather than replacing an existing placeholder) + after the prompt's leading ``, we use [PromptInsertion][vllm.multimodal.processing.PromptInsertion] + with a prefix target: ??? code @@ -804,33 +615,41 @@ Each [PromptUpdate][vllm.multimodal.processing.PromptUpdate] instance specifies out_mm_kwargs: MultiModalKwargsItems, ) -> Sequence[PromptUpdate]: hf_config = self.info.get_hf_config() - bos_token_id = hf_config.bos_token_id - assert isinstance(bos_token_id, int) + image_token_id = hf_config.image_token_index tokenizer = self.info.get_tokenizer() - eot_token_id = tokenizer.bos_token_id - assert isinstance(eot_token_id, int) - def get_replacement_fuyu(item_idx: int): - images = mm_items.get_items("image", ImageProcessorItems) - image_size = images.get_image_size(item_idx) + bos_token_id = tokenizer.bos_token_id + assert isinstance(bos_token_id, int) - ncols, nrows = self.info.get_image_feature_grid_size( - image_width=image_size.width, - image_height=image_size.height, + def get_insertion(item_idx: int): + images = mm_items.get_items( + "image", (ImageEmbeddingItems, ImageProcessorItems) ) - image_tokens = ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows + + if isinstance(images, ImageEmbeddingItems): + num_image_tokens = images.get_feature_size(item_idx) + else: + image_size = images.get_image_size(item_idx) + num_image_tokens = self.info.get_num_image_tokens( + image_width=image_size.width, + image_height=image_size.height, + ) + + image_tokens = [image_token_id] * num_image_tokens return PromptUpdateDetails.select_token_id( image_tokens + [bos_token_id], - embed_token_id=_IMAGE_TOKEN_ID, + embed_token_id=image_token_id, ) return [ - PromptReplacement( + PromptInsertion( modality="image", - target=[eot_token_id], - replacement=get_replacement_fuyu, + target=PromptIndexTargets.prefix( + [bos_token_id] if tokenizer.add_bos_token else [] + ), + insertion=get_insertion, ) ] ``` @@ -873,7 +692,7 @@ Examples: Examples: - Chameleon (appends `sep_token`): [vllm/model_executor/models/chameleon.py](../../../vllm/model_executor/models/chameleon.py) -- Fuyu (appends `boa_token`): [vllm/model_executor/models/fuyu.py](../../../vllm/model_executor/models/fuyu.py) +- Molmo2 (prepends `bos_token`): [vllm/model_executor/models/molmo2.py](../../../vllm/model_executor/models/molmo2.py) - Molmo (applies chat template which is not defined elsewhere): [vllm/model_executor/models/molmo.py](../../../vllm/model_executor/models/molmo.py) ### Custom HF processor diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 4e44a577db..c3c2d3df05 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -436,7 +436,6 @@ th { | `PhiForCausalLM` | Phi | `microsoft/phi-1_5`, `microsoft/phi-2`, etc. | ✅︎ | ✅︎ | | `Phi3ForCausalLM` | Phi-4, Phi-3 | `microsoft/Phi-4-mini-instruct`, `microsoft/Phi-4`, `microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, `microsoft/Phi-3-medium-128k-instruct`, etc. | ✅︎ | ✅︎ | | `PhiMoEForCausalLM` | Phi-3.5-MoE | `microsoft/Phi-3.5-MoE-instruct`, etc. | ✅︎ | ✅︎ | -| `PersimmonForCausalLM` | Persimmon | `adept/persimmon-8b-base`, `adept/persimmon-8b-chat`, etc. | | ✅︎ | | `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | ✅ | ✅︎ | | `Plamo3ForCausalLM` | PLaMo3 | `pfnet/plamo-3-nict-2b-base`, `pfnet/plamo-3-nict-8b-base`, etc. | ✅ | ✅︎ | | `Qwen2ForCausalLM` | QwQ, Qwen2 | `Qwen/QwQ-32B-Preview`, `Qwen/Qwen2-7B-Instruct`, `Qwen/Qwen2-7B`, etc. | ✅︎ | ✅︎ | @@ -520,7 +519,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + IE+ | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ | | `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I+/ V+ | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ | | `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + IE+ | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ | -| `FuyuForCausalLM` | Fuyu | T + I | `adept/fuyu-8b`, etc. | | ✅︎ | | `Gemma3ForConditionalGeneration` | Gemma 3 | T + IE+ | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ | | `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | | `Gemma4ForConditionalGeneration` | Gemma 4 | T + I+ + V + A* | `google/gemma-4-E2B-it`, etc. | | ✅︎ | diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index a1eb6cc538..bddf6388ae 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -448,24 +448,6 @@ def run_exaone4_5(questions: list[str], modality: str) -> ModelRequestData: ) -# Fuyu -def run_fuyu(questions: list[str], modality: str) -> ModelRequestData: - assert modality == "image" - - prompts = [f"{question}\n" for question in questions] - engine_args = EngineArgs( - model="adept/fuyu-8b", - max_model_len=2048, - max_num_seqs=2, - limit_mm_per_prompt={modality: 1}, - ) - - return ModelRequestData( - engine_args=engine_args, - prompts=prompts, - ) - - # Gemma 3 def run_gemma3(questions: list[str], modality: str) -> ModelRequestData: assert modality == "image" @@ -2318,7 +2300,6 @@ model_example_map = { "eagle2_5": run_eagle2_5, "ernie45_vl": run_ernie45_vl, "exaone4_5": run_exaone4_5, - "fuyu": run_fuyu, "gemma3": run_gemma3, "gemma3n": run_gemma3n, "glm4v": run_glm4v, diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py index 762a95fb98..fd4a1d0f57 100644 --- a/tests/distributed/test_pipeline_parallel.py +++ b/tests/distributed/test_pipeline_parallel.py @@ -139,7 +139,6 @@ TEXT_GENERATION_MODELS = { "allenai/OLMoE-1B-7B-0924-Instruct": PPTestSettings.fast(), "facebook/opt-iml-max-1.3b": PPTestSettings.fast(), "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(), - "adept/persimmon-8b-chat": PPTestSettings.fast(), "microsoft/phi-2": PPTestSettings.fast(), "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(), "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.detailed( @@ -168,7 +167,6 @@ MULTIMODAL_MODELS = { # [Decoder-only] "Salesforce/blip2-opt-6.7b": PPTestSettings.fast(), "facebook/chameleon-7b": PPTestSettings.fast(), - "adept/fuyu-8b": PPTestSettings.fast(), "zai-org/glm-4v-9b": PPTestSettings.fast(), "OpenGVLab/InternVL3-1B": PPTestSettings.fast(), "llava-hf/llava-1.5-7b-hf": PPTestSettings.fast(), diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index 6c619739c6..fb5c518038 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -388,20 +388,6 @@ VLM_TEST_SETTINGS = { stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"], image_size_factors=[(1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)], ), - "fuyu": VLMTestInfo( - models=["adept/fuyu-8b"], - test_type=VLMTestType.IMAGE, - prompt_formatter=lambda img_prompt: f"{img_prompt}\n", - img_idx_to_prompt=lambda idx: "", - max_model_len=2048, - max_num_seqs=2, - auto_cls=AutoModelForImageTextToText, - use_tokenizer_eos=True, - vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output, - num_logprobs=10, - image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)], - marks=[large_gpu_mark(min_gb=32)], - ), "gemma3": VLMTestInfo( models=["google/gemma-3-4b-it"], test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 72151b9ff9..7bee4615aa 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -52,15 +52,6 @@ def blip2_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutp return hf_output_ids, hf_output_str, out_logprobs -def fuyu_vllm_to_hf_output(vllm_output: RunnerOutput, model: str) -> RunnerOutput: - """Sanitize vllm output [fuyu models] to be comparable with hf output.""" - output_ids, output_str, out_logprobs = vllm_output - - hf_output_str = output_str.lstrip() + "|ENDOFTEXT|" - - return output_ids, hf_output_str, out_logprobs - - def qwen_vllm_to_hf_output( vllm_output: RunnerOutput, model: str ) -> tuple[list[int], str, SampleLogprobs | None]: diff --git a/tests/models/registry.py b/tests/models/registry.py index e7e11e8763..e04f647725 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -466,7 +466,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "bharatgenai/Param2-17B-A2.4B-Thinking", trust_remote_code=True, ), - "PersimmonForCausalLM": _HfExamplesInfo("adept/persimmon-8b-chat"), "PhiForCausalLM": _HfExamplesInfo("microsoft/phi-2"), "Phi3ForCausalLM": _HfExamplesInfo("microsoft/Phi-3-mini-4k-instruct"), "PhiMoEForCausalLM": _HfExamplesInfo( @@ -866,7 +865,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "FunAudioChatForConditionalGeneration": _HfExamplesInfo( "funaudiochat", is_available_online=False ), - "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"), "Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"), "DiffusionGemmaForBlockDiffusion": _HfExamplesInfo( "google/diffusiongemma-26B-A4B-it", diff --git a/tests/renderers/test_hf.py b/tests/renderers/test_hf.py index 0ccbbaf21a..f611bac02b 100644 --- a/tests/renderers/test_hf.py +++ b/tests/renderers/test_hf.py @@ -426,7 +426,6 @@ def test_resolve_content_format_hf_defined(model, expected_format): ("Salesforce/blip2-opt-2.7b", "string"), ("facebook/chameleon-7b", "string"), ("deepseek-ai/deepseek-vl2-tiny", "string"), - ("adept/fuyu-8b", "string"), ("google/paligemma-3b-mix-224", "string"), ], ) diff --git a/vllm/model_executor/models/fuyu.py b/vllm/model_executor/models/fuyu.py deleted file mode 100644 index 3c0b911c3a..0000000000 --- a/vllm/model_executor/models/fuyu.py +++ /dev/null @@ -1,368 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/fuyu/modeling_fuyu.py -# Copyright 2023 The vLLM team. -# Copyright 2023 HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""PyTorch Fuyu model.""" - -import math -from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal - -import torch -import torch.nn as nn -from transformers import BatchFeature, FuyuConfig, FuyuImageProcessor, FuyuProcessor - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.inputs import MultiModalDataDict -from vllm.model_executor.layers.linear import ColumnParallelLinear -from vllm.model_executor.models.persimmon import PersimmonForCausalLM -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - MultiModalFieldConfig, - MultiModalKwargsItems, -) -from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems -from vllm.multimodal.processing import ( - BaseDummyInputsBuilder, - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, -) -from vllm.sequence import IntermediateTensors -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP -from .utils import AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix - -# Cannot find the following 2 numbers from hf config. -_IMAGE_TOKEN_ID = 71011 -_NEWLINE_TOKEN_ID = 71019 - - -class FuyuImagePatchInputs(TensorSchema): - """ - Dimensions: - - bn: Batch size * number of images - - bnp: Batch size * number of images * number of patches - - fn: patch_size_x * patch_size_y * num_channels - """ - - type: Literal["image_patches"] = "image_patches" - - image_patches_flat: Annotated[torch.Tensor, TensorShape("bnp", "fn")] - - patches_per_image: Annotated[list[int], TensorShape("bn")] - """ - The number of total patches for each image in the batch. - - This is used to split the embeddings which has the first two dimensions - flattened just like `image_patches_flat`. - """ - - -class FuyuProcessingInfo(BaseProcessingInfo): - def get_hf_config(self): - return self.ctx.get_hf_config(FuyuConfig) - - def get_hf_processor(self, **kwargs: object): - return self.ctx.get_hf_processor(FuyuProcessor, **kwargs) - - def get_image_processor(self, **kwargs: object) -> FuyuImageProcessor: - return self.get_hf_processor(**kwargs).image_processor - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"image": 1} - - def get_image_feature_grid_size( - self, - *, - image_width: int, - image_height: int, - ) -> tuple[int, int]: - image_processor = self.get_image_processor() - target_width = image_processor.size["width"] - target_height = image_processor.size["height"] - patch_width = image_processor.patch_size["width"] - patch_height = image_processor.patch_size["height"] - - if not (image_width <= target_width and image_height <= target_height): - height_scale_factor = target_height / image_height - width_scale_factor = target_width / image_width - optimal_scale_factor = min(height_scale_factor, width_scale_factor) - - image_height = int(image_height * optimal_scale_factor) - image_width = int(image_width * optimal_scale_factor) - - ncols = math.ceil(image_width / patch_width) - nrows = math.ceil(image_height / patch_height) - return ncols, nrows - - def get_num_image_tokens( - self, - *, - image_width: int, - image_height: int, - ) -> int: - ncols, nrows = self.get_image_feature_grid_size( - image_width=image_width, - image_height=image_height, - ) - - return ncols * nrows - - def get_image_size_with_most_features(self) -> ImageSize: - image_processor = self.get_image_processor() - return ImageSize( - width=image_processor.size["width"], height=image_processor.size["height"] - ) - - -class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - return "" - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], - ) -> MultiModalDataDict: - target_width, target_height = self.info.get_image_size_with_most_features() - num_images = mm_counts.get("image", 0) - - image_overrides = mm_options.get("image") - - return { - "image": self._get_dummy_images( - width=target_width, - height=target_height, - num_images=num_images, - overrides=image_overrides, - ) - } - - -class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], - ) -> BatchFeature: - if not mm_data: - # Avoid warning from HF logger for text-only input - prompt_ids = self.info.get_tokenizer().encode(prompt) - prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids) - return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt") - - processed_outputs = super()._call_hf_processor( - prompt=prompt, - mm_data=mm_data, - mm_kwargs=mm_kwargs, - tok_kwargs=tok_kwargs, - ) - - image_patches = processed_outputs["image_patches"] - processed_outputs["image_patches"] = flatten_bn(image_patches) - processed_outputs["patches_per_image"] = torch.tensor( - [len(p) for p in image_patches] - ) - - return processed_outputs - - def _apply_hf_processor_tokens_only( - self, - prompt_tokens: list[int], - ) -> list[int]: - # HF processor adds boa_token_id - tokenizer = self.info.get_tokenizer() - vocab = tokenizer.get_vocab() - - boa_token_id = vocab["<0x04>"] - if prompt_tokens[-1] != boa_token_id: - prompt_tokens.append(boa_token_id) - - return prompt_tokens - - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - patches_per_image = hf_inputs.get("patches_per_image", torch.empty(0)) - - return dict( - image_patches=MultiModalFieldConfig.flat_from_sizes( - "image", patches_per_image - ), - patches_per_image=MultiModalFieldConfig.batched("image"), - ) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - hf_config = self.info.get_hf_config() - bos_token_id = hf_config.bos_token_id - assert isinstance(bos_token_id, int) - - tokenizer = self.info.get_tokenizer() - eot_token_id = tokenizer.bos_token_id - assert isinstance(eot_token_id, int) - - def get_replacement_fuyu(item_idx: int): - images = mm_items.get_items("image", ImageProcessorItems) - image_size = images.get_image_size(item_idx) - - ncols, nrows = self.info.get_image_feature_grid_size( - image_width=image_size.width, - image_height=image_size.height, - ) - image_tokens = ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows - - return PromptUpdateDetails.select_token_id( - image_tokens + [bos_token_id], - embed_token_id=_IMAGE_TOKEN_ID, - ) - - return [ - PromptReplacement( - modality="image", - target=[eot_token_id], - replacement=get_replacement_fuyu, - ) - ] - - -@MULTIMODAL_REGISTRY.register_processor( - FuyuMultiModalProcessor, - info=FuyuProcessingInfo, - dummy_inputs=FuyuDummyInputsBuilder, -) -class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP): - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={ - "model.vision_embed_tokens.": "vision_embed_tokens.", - "model.language_model.": "language_model.model.", - "lm_head.": "language_model.lm_head.", - } - ) - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return None - - raise ValueError("Only image modality is supported") - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config - self.config = config - self.multimodal_config = multimodal_config - - self.vocab_size = config.text_config.vocab_size - self.image_token_id = _IMAGE_TOKEN_ID - self.image_feature_size = config.patch_size**2 * config.num_channels - - with self._mark_tower_model(vllm_config, "image"): - self.vision_embed_tokens = ColumnParallelLinear( - self.image_feature_size, - config.hidden_size, - quant_config=quant_config, - gather_output=True, - ) - - with self._mark_language_model(vllm_config): - self.language_model = PersimmonForCausalLM( - vllm_config=vllm_config.with_hf_config(config.text_config), - prefix=maybe_prefix(prefix, "language_model"), - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> FuyuImagePatchInputs | None: - image_patches = kwargs.pop("image_patches", None) - patches_per_image = kwargs.pop("patches_per_image", None) - - if image_patches is None: - return None - - return FuyuImagePatchInputs( - type="image_patches", - image_patches_flat=image_patches, - patches_per_image=patches_per_image, - resolve_bindings={"fn": self.image_feature_size}, - ) - - def _process_image_input( - self, image_input: FuyuImagePatchInputs - ) -> MultiModalEmbeddings: - image_patches_flat = image_input["image_patches_flat"] - patches_per_image = image_input["patches_per_image"] - - vision_embeddings_flat, _ = self.vision_embed_tokens(image_patches_flat) - - return vision_embeddings_flat.split(patches_per_image.tolist(), dim=0) - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - image_input = self._parse_and_validate_image_input(**kwargs) - if image_input is None: - return [] - - return self._process_image_input(image_input) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ): - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.language_model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - return self.language_model.compute_logits(hidden_states) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) diff --git a/vllm/model_executor/models/persimmon.py b/vllm/model_executor/models/persimmon.py deleted file mode 100644 index a03a785577..0000000000 --- a/vllm/model_executor/models/persimmon.py +++ /dev/null @@ -1,373 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# adapted from https://github.com/huggingface/transformers/blob/v4.39.3/src/transformers/models/persimmon/modeling_persimmon.py -# Copyright 2023 The vLLM team. -# Copyright 2023 EleutherAI and the HuggingFace Inc. team. All rights reserved. -# -# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX -# and OPT implementations in this library. It has been modified from its -# original forms to accommodate minor architectural differences compared -# to GPT-NeoX and OPT used by the Meta AI team that trained the model. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Inference-only persimmon model compatible with HuggingFace weights.""" - -from collections.abc import Iterable -from itertools import islice - -import torch -from torch import nn -from transformers import PersimmonConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, VllmConfig -from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.linear import ( - ColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear, -) -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from vllm.sequence import IntermediateTensors - -from .interfaces import SupportsPP -from .utils import ( - AutoWeightsLoader, - is_pp_missing_parameter, - make_empty_intermediate_tensors_factory, - make_layers, - maybe_prefix, -) - - -class PersimmonMLP(nn.Module): - def __init__( - self, - config: PersimmonConfig, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.dense_h_to_4h = ColumnParallelLinear( - config.hidden_size, - config.intermediate_size, - quant_config=quant_config, - prefix=f"{prefix}.dense_h_to_4h", - ) - self.dense_4h_to_h = RowParallelLinear( - config.intermediate_size, - config.hidden_size, - quant_config=quant_config, - prefix=f"{prefix}.dense_4h_to_h", - ) - self.act = get_act_fn(config.hidden_act) - - def forward(self, hidden_states) -> torch.Tensor: - hidden_states, _ = self.dense_h_to_4h(hidden_states) - hidden_states = self.act(hidden_states) - hidden_states, _ = self.dense_4h_to_h(hidden_states) - return hidden_states - - -class PersimmonAttention(nn.Module): - def __init__( - self, - config: PersimmonConfig, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.config = config - tensor_parallel_world_size = get_tensor_model_parallel_world_size() - - self.hidden_size = config.hidden_size - self.total_num_heads = config.num_attention_heads - self.num_heads = self.total_num_heads // tensor_parallel_world_size - self.head_dim = self.hidden_size // self.total_num_heads - self.max_position_embeddings = config.max_position_embeddings - self.is_causal = True - - assert (self.head_dim * self.total_num_heads) == self.hidden_size - assert self.total_num_heads % tensor_parallel_world_size == 0 - - self.query_key_value = QKVParallelLinear( - self.hidden_size, - self.head_dim, - self.total_num_heads, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.query_key_value", - ) - self.dense = RowParallelLinear( - self.total_num_heads * self.head_dim, - self.hidden_size, - bias=True, - quant_config=quant_config, - prefix=f"{prefix}.dense", - ) - self.is_qk_layernorm = config.qk_layernorm - - if self.is_qk_layernorm: - self.q_layernorm = nn.LayerNorm(self.head_dim) - self.k_layernorm = nn.LayerNorm(self.head_dim) - - self.rotary_emb = get_rope( - self.head_dim, - max_position=self.max_position_embeddings, - rope_parameters=config.rope_parameters, - ) - self.scaling = self.head_dim**-0.5 - self.attn = Attention( - self.num_heads, - self.head_dim, - scale=self.scaling, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn", - ) - - def _split_heads(self, x: torch.Tensor) -> torch.Tensor: - # [seq_length, hidden_size] -> [seq_length, num_heads, head_dim] - seq_length = x.shape[0] - return x.view(seq_length, self.num_heads, self.head_dim) - - def _merge_heads(self, x: torch.Tensor) -> torch.Tensor: - # [seq_length, num_heads, head_dim] -> [seq_length, hidden_size] - seq_length = x.shape[0] - return x.view(seq_length, self.num_heads * self.head_dim) - - def forward( - self, - position_ids: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - # [seq_length, 3 x hidden_size] - qkv, _ = self.query_key_value(hidden_states) - q, k, v = qkv.chunk(chunks=3, dim=-1) - - if self.is_qk_layernorm: - # [seq_length, num_heads, head_dim] - q = self._split_heads(q) - k = self._split_heads(k) - - q = self.q_layernorm(q) - k = self.k_layernorm(k) - - q = self._merge_heads(q) - k = self._merge_heads(k) - - q, k = self.rotary_emb(position_ids, q, k) - attn_output = self.attn(q, k, v) - output, _ = self.dense(attn_output) - return output - - -class PersimmonDecoderLayer(nn.Module): - def __init__( - self, - config: PersimmonConfig, - cache_config: CacheConfig | None = None, - quant_config: QuantizationConfig | None = None, - prefix: str = "", - ): - super().__init__() - self.hidden_size = config.hidden_size - self.self_attn = PersimmonAttention( - config=config, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.self_attn", - ) - self.mlp = PersimmonMLP( - config, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) - self.input_layernorm = nn.LayerNorm( - config.hidden_size, eps=config.layer_norm_eps - ) - self.post_attention_layernorm = nn.LayerNorm( - config.hidden_size, eps=config.layer_norm_eps - ) - - def forward( - self, - position_ids: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - residual = hidden_states - - hidden_states = self.input_layernorm(hidden_states) - - # Self Attention - hidden_states = self.self_attn( - position_ids=position_ids, - hidden_states=hidden_states, - ) - hidden_states = residual + hidden_states - - # Fully Connected - residual = hidden_states - hidden_states = self.post_attention_layernorm(hidden_states) - hidden_states = self.mlp(hidden_states) - - hidden_states = hidden_states + residual - - outputs = hidden_states - return outputs - - -@support_torch_compile -class PersimmonModel(nn.Module): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - - config = vllm_config.model_config.hf_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - - self.vocab_size = config.vocab_size - self.config = config - self.embed_tokens = VocabParallelEmbedding( - config.vocab_size, config.hidden_size - ) - self.start_layer, self.end_layer, self.layers = make_layers( - config.num_hidden_layers, - lambda prefix: PersimmonDecoderLayer( - config, cache_config, quant_config, prefix=prefix - ), - prefix=f"{prefix}.layers", - ) - self.final_layernorm = nn.LayerNorm( - config.hidden_size, eps=config.layer_norm_eps - ) - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states"], config.hidden_size - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - if get_pp_group().is_first_rank: - if inputs_embeds is not None: - hidden_states = inputs_embeds - else: - hidden_states = self.embed_input_ids(input_ids) - else: - assert intermediate_tensors is not None - hidden_states = intermediate_tensors["hidden_states"] - for layer in islice(self.layers, self.start_layer, self.end_layer): - hidden_states = layer(positions, hidden_states) - if not get_pp_group().is_last_rank: - return IntermediateTensors({"hidden_states": hidden_states}) - hidden_states = self.final_layernorm(hidden_states) - return hidden_states - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - - if "query_key_value" in name: - # copy from vllm/model_executor/models/bloom.py - # NOTE: Persimmon's fused QKV's output_dim has the shape of - # (num_heads * 3 * head_size), while the - # required shape is (3 * num_heads * head_size). - # Thus, we need weight conversion. - output_dim = getattr(param, "output_dim", None) - num_heads = self.config.num_attention_heads - if output_dim is not None: - loaded_weight_shape = loaded_weight.shape - loaded_weight = loaded_weight.view( - loaded_weight_shape[:output_dim] - + (num_heads, 3, -1) - + loaded_weight_shape[output_dim + 1 :] - ) - loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1) - loaded_weight = loaded_weight.reshape(loaded_weight_shape) - - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - -class PersimmonForCausalLM(nn.Module, SupportsPP): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config = vllm_config.model_config.hf_config - self.config = config - self.vocab_size = config.vocab_size - self.model = PersimmonModel( - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") - ) - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - bias=False, - prefix=maybe_prefix(prefix, "lm_head"), - ) - self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - ): - hidden_states = self.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - logits = self.logits_processor(self.lm_head, hidden_states) - return logits - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 6031996a1d..8d5879cc3d 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -180,7 +180,6 @@ _TEXT_GENERATION_MODELS = { "PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"), "PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"), "Param2MoEForCausalLM": ("param2moe", "Param2MoEForCausalLM"), - "PersimmonForCausalLM": ("persimmon", "PersimmonForCausalLM"), "PhiForCausalLM": ("phi", "PhiForCausalLM"), "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"), "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"), @@ -377,7 +376,6 @@ _MULTIMODAL_MODELS = { "funaudiochat", "FunAudioChatForConditionalGeneration", ), - "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"), "Gemma3ForConditionalGeneration": ("gemma3_mm", "Gemma3ForConditionalGeneration"), "Gemma3nForConditionalGeneration": ( "gemma3n_mm", @@ -738,6 +736,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = { "MusicFlamingoForConditionalGeneration": "0.24.0", "AyaVisionForConditionalGeneration": "0.24.0", "TeleChatForCausalLM": "0.25.0", + "PersimmonForCausalLM": "0.25.0", + "FuyuForCausalLM": "0.25.0", } _OOT_SUPPORTED_MODELS = { diff --git a/vllm/transformers_utils/chat_templates/registry.py b/vllm/transformers_utils/chat_templates/registry.py index ed74474290..9af4217be6 100644 --- a/vllm/transformers_utils/chat_templates/registry.py +++ b/vllm/transformers_utils/chat_templates/registry.py @@ -31,7 +31,6 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = { "deepseek_ocr2": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja", "unlimited-ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja", "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja", - "fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja", "minicpmv": _get_minicpmv_chat_template_fallback, "minicpmv4_6": _get_minicpmv_chat_template_fallback, "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja", diff --git a/vllm/transformers_utils/chat_templates/template_fuyu.jinja b/vllm/transformers_utils/chat_templates/template_fuyu.jinja deleted file mode 100644 index ec337d0c64..0000000000 --- a/vllm/transformers_utils/chat_templates/template_fuyu.jinja +++ /dev/null @@ -1,3 +0,0 @@ -{%- for message in messages -%} - {{- message['content'] + '\n' -}} -{%- endfor -%}