Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Multi-modal processors may be defined in this directory for the following
|
||||
"""Multi-modal processors may be defined in this directory for the following
|
||||
reasons:
|
||||
|
||||
- There is no processing file defined by HF Hub or Transformers library.
|
||||
|
||||
@@ -18,8 +18,7 @@ class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-
|
||||
|
||||
|
||||
class BagelProcessor(ProcessorMixin):
|
||||
"""
|
||||
Constructs a BAGEL processor which wraps a
|
||||
"""Constructs a BAGEL processor which wraps a
|
||||
SigLIP image processor and a Qwen2 tokenizer.
|
||||
"""
|
||||
|
||||
@@ -36,9 +35,7 @@ class BagelProcessor(ProcessorMixin):
|
||||
images: ImageInput = None,
|
||||
**kwargs: Unpack[BagelProcessorKwargs],
|
||||
):
|
||||
"""
|
||||
Main method to prepare for the model one or several sequences(s) and image(s).
|
||||
"""
|
||||
"""Prepare one or more sequences and images for the model."""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
BagelProcessorKwargs,
|
||||
tokenizer_init_kwargs=self.tokenizer.init_kwargs,
|
||||
@@ -62,15 +59,11 @@ class BagelProcessor(ProcessorMixin):
|
||||
return BatchFeature(data={**pixel_values, **text_inputs})
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's batch_decode.
|
||||
"""
|
||||
"""Forward all arguments to `Qwen2TokenizerFast.batch_decode`."""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's decode.
|
||||
"""
|
||||
"""This method forwards all its arguments to Qwen2TokenizerFast's decode."""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
@property
|
||||
|
||||
@@ -228,7 +228,7 @@ class FilterbankFeatures(nn.Module):
|
||||
return self.fb
|
||||
|
||||
def splice_frames(self, x, frame_splicing):
|
||||
"""Stacks frames together across feature dim
|
||||
"""Stacks frames together across feature dim.
|
||||
|
||||
input is batch_size, feature_dim, num_frames
|
||||
output is batch_size, feature_dim*frame_splicing, num_frames
|
||||
@@ -537,7 +537,8 @@ class CohereASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
|
||||
class CohereASRProcessor(ProcessorMixin):
|
||||
"""HF-compatible processor combining CohereASRFeatureExtractor and a
|
||||
tokenizer."""
|
||||
tokenizer.
|
||||
"""
|
||||
|
||||
feature_extractor_class = "CohereASRFeatureExtractor"
|
||||
tokenizer_class = "AutoTokenizer"
|
||||
|
||||
@@ -70,8 +70,7 @@ class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor):
|
||||
resample: PILImageResampling | InterpolationMode | int | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Rescales the image so that the following conditions are met:
|
||||
"""Rescales the image so that the following conditions are met:
|
||||
|
||||
1. Both dimensions (height and width) are divisible by 'factor'.
|
||||
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
|
||||
@@ -322,6 +321,7 @@ class Cosmos3EdgeProcessor(Qwen3VLProcessor):
|
||||
normalize, and patch video inputs.
|
||||
chat_template: An optional Jinja template used to render chat messages
|
||||
into a tokenizable string.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer", "video_processor"]
|
||||
|
||||
@@ -234,9 +234,7 @@ class DeepseekOCRProcessor(ProcessorMixin):
|
||||
images: list[Image.Image],
|
||||
crop_mode: bool = CROP_MODE,
|
||||
):
|
||||
"""
|
||||
|
||||
Args:
|
||||
"""Args:
|
||||
prompt (str): the formatted prompt;
|
||||
images (List[ImageType]): the list of images;
|
||||
crop_mode (bool): if True, then crop the image;
|
||||
@@ -248,8 +246,8 @@ class DeepseekOCRProcessor(ProcessorMixin):
|
||||
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
|
||||
- image_id (int): the id of the image token
|
||||
- num_image_tokens (List[int]): the number of image tokens
|
||||
"""
|
||||
|
||||
"""
|
||||
if prompt is None or images is None:
|
||||
raise ValueError("prompt and images must be used at the same time.")
|
||||
|
||||
@@ -309,7 +307,6 @@ class DeepseekOCRProcessor(ProcessorMixin):
|
||||
cropping: bool = True,
|
||||
):
|
||||
"""Tokenize text with <image> tags."""
|
||||
|
||||
num_image_tags = conversation.count(self.image_token)
|
||||
if num_image_tags != len(images):
|
||||
raise ValueError(
|
||||
|
||||
@@ -186,9 +186,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
|
||||
inference_mode: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""
|
||||
|
||||
Args:
|
||||
"""Args:
|
||||
prompt (str): the formatted prompt;
|
||||
images (list[ImageType]): the list of images;
|
||||
inference_mode (bool): if True, then remove the last eos token;
|
||||
@@ -201,8 +199,8 @@ class DeepseekVLV2Processor(ProcessorMixin):
|
||||
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
|
||||
- image_id (int): the id of the image token
|
||||
- num_image_tokens (list[int]): the number of image tokens
|
||||
"""
|
||||
|
||||
"""
|
||||
if prompt is None or images is None:
|
||||
raise ValueError("prompt and images must be used at the same time.")
|
||||
|
||||
@@ -276,9 +274,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
|
||||
inference_mode: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""
|
||||
|
||||
Args:
|
||||
"""Args:
|
||||
text (str): the formatted prompt;
|
||||
images (list[ImageType]): the list of images;
|
||||
inference_mode (bool): if True, then remove the last eos token;
|
||||
@@ -290,8 +286,8 @@ class DeepseekVLV2Processor(ProcessorMixin):
|
||||
- images (torch.FloatTensor): [n_images, 3, H, W]
|
||||
- image_id (int): the id of the image token
|
||||
- num_image_tokens (list[int]): the number of image tokens
|
||||
"""
|
||||
|
||||
"""
|
||||
prepare = self.process_one(
|
||||
prompt=text,
|
||||
images=images,
|
||||
|
||||
@@ -68,8 +68,7 @@ class KaldifeatFbank:
|
||||
|
||||
|
||||
class FireRedASR2FeatureExtractor(SequenceFeatureExtractor):
|
||||
r"""
|
||||
Constructs a FireRedASR2 feature extractor.
|
||||
r"""Constructs a FireRedASR2 feature extractor.
|
||||
|
||||
This feature extractor inherits from [`~feature_extraction_sequence_
|
||||
utils.SequenceFeatureExtractor`] which contains most of the main
|
||||
@@ -99,6 +98,7 @@ class FireRedASR2FeatureExtractor(SequenceFeatureExtractor):
|
||||
Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
|
||||
the high log_mel_fbank values for signals with hard-zero sections,
|
||||
when VAD cutoff is present in the signal.
|
||||
|
||||
"""
|
||||
|
||||
model_input_names = ["input_features"]
|
||||
@@ -230,9 +230,8 @@ class FireRedASR2FeatureExtractor(SequenceFeatureExtractor):
|
||||
|
||||
|
||||
class FireRedASR2Processor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor and
|
||||
a FireRedASR2 tokenizer into a single processor.
|
||||
r"""Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor
|
||||
and a FireRedASR2 tokenizer into a single processor.
|
||||
|
||||
[`FireRedASR2Processor`] offers all the functionalities of
|
||||
[`FireRedASR2FeatureExtractor`] and [`Qwen2Tokenizer`]. See the
|
||||
@@ -246,6 +245,7 @@ class FireRedASR2Processor(ProcessorMixin):
|
||||
tokenizer (`Qwen2Tokenizer`):
|
||||
An instance of [`Qwen2Tokenizer`]. The tokenizer is a required
|
||||
input.
|
||||
|
||||
"""
|
||||
|
||||
feature_extractor_class = "FireRedASR2FeatureExtractor"
|
||||
@@ -271,8 +271,7 @@ class FireRedASR2Processor(ProcessorMixin):
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""
|
||||
Forwards the `audio` argument to FireRedASR2FeatureExtractor's
|
||||
"""Forwards the `audio` argument to FireRedASR2FeatureExtractor's
|
||||
[`~FireRedASR2FeatureExtractor.__call__`] and the `text` argument to
|
||||
[`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the
|
||||
above two methods for more information.
|
||||
|
||||
@@ -21,10 +21,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def apply_cmvn(inputs, cmvn): # noqa
|
||||
"""
|
||||
Apply CMVN with mvn data
|
||||
"""
|
||||
|
||||
"""Apply CMVN with mvn data."""
|
||||
device = inputs.device
|
||||
# dtype = inputs.dtype
|
||||
frame, dim = inputs.shape
|
||||
@@ -219,8 +216,7 @@ class WavFrontend(nn.Module):
|
||||
|
||||
|
||||
class FunASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
r"""
|
||||
Constructs a FunASR feature extractor.
|
||||
r"""Constructs a FunASR feature extractor.
|
||||
|
||||
This feature extractor inherits from [`~feature_extraction_sequence_
|
||||
utils.SequenceFeatureExtractor`] which contains most of the main
|
||||
@@ -255,6 +251,7 @@ class FunASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces
|
||||
the high log_mel_fbank values for signals with hard-zero sections,
|
||||
when VAD cutoff is present in the signal.
|
||||
|
||||
"""
|
||||
|
||||
model_input_names = ["input_features"]
|
||||
@@ -369,8 +366,7 @@ class FunASRFeatureExtractor(SequenceFeatureExtractor):
|
||||
|
||||
|
||||
class FunASRProcessor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs a FunASR processor which wraps a FunASR feature extractor and
|
||||
r"""Constructs a FunASR processor which wraps a FunASR feature extractor and
|
||||
a FunASR tokenizer into a single processor.
|
||||
|
||||
[`FunASRProcessor`] offers all the functionalities of
|
||||
@@ -385,6 +381,7 @@ class FunASRProcessor(ProcessorMixin):
|
||||
tokenizer (`Qwen2Tokenizer`):
|
||||
An instance of [`Qwen2Tokenizer`]. The tokenizer is a required
|
||||
input.
|
||||
|
||||
"""
|
||||
|
||||
feature_extractor_class = "FunASRFeatureExtractor"
|
||||
@@ -410,8 +407,7 @@ class FunASRProcessor(ProcessorMixin):
|
||||
)
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""
|
||||
Forwards the `audio` argument to FunASRFeatureExtractor's
|
||||
"""Forwards the `audio` argument to FunASRFeatureExtractor's
|
||||
[`~FunASRFeatureExtractor.__call__`] and the `text` argument to
|
||||
[`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the
|
||||
above two methods for more information.
|
||||
|
||||
@@ -10,8 +10,7 @@ from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
|
||||
class GLM4VImageProcessorFast(BaseImageProcessorFast):
|
||||
"""
|
||||
Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
|
||||
"""Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
|
||||
to HF Transformers.
|
||||
"""
|
||||
|
||||
|
||||
@@ -261,7 +261,8 @@ def _resize_or_pad(
|
||||
allow_upscale: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""Resize onto the aligned canvas, or keep the aspect ratio and
|
||||
zero-pad the right/bottom sides (``resize_mode="pad"``)."""
|
||||
zero-pad the right/bottom sides (``resize_mode="pad"``).
|
||||
"""
|
||||
height, width = stacked_images.shape[-2:]
|
||||
|
||||
if resize_mode == "resize":
|
||||
@@ -307,7 +308,8 @@ def _pixel_budget(
|
||||
) -> tuple[int, int]:
|
||||
"""(min_pixels, max_pixels) from the token bounds of
|
||||
``processor_config.json``; one vision token covers
|
||||
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels."""
|
||||
``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels.
|
||||
"""
|
||||
if min_image_tokens is None or max_image_tokens is None:
|
||||
raise ValueError(
|
||||
"min_image_tokens and max_image_tokens must be provided by "
|
||||
|
||||
@@ -358,8 +358,7 @@ class InternVLVideoProcessor(BaseVideoProcessor):
|
||||
|
||||
|
||||
class InternVLProcessor(ProcessorMixin):
|
||||
"""
|
||||
This model doesn't define its own HF processor,
|
||||
"""This model doesn't define its own HF processor,
|
||||
so we implement our own one here.
|
||||
|
||||
The code to insert image tokens is based on:
|
||||
|
||||
@@ -91,6 +91,7 @@ def get_image_size_for_max_num_patches(
|
||||
`tuple[int, int]`: Height and width (in pixels) that are multiples of
|
||||
`patch_size * pixel_shuffle_scale` and respect both the maximum and
|
||||
optional minimum patch-count constraints.
|
||||
|
||||
"""
|
||||
|
||||
def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale):
|
||||
@@ -177,9 +178,11 @@ def prepare_image_tensor(
|
||||
The tensor is converted to floating point if needed.
|
||||
scale (`float`, *optional*, defaults to `VISION_SCALE`):
|
||||
Scalar multiplier applied before normalization.
|
||||
|
||||
Returns:
|
||||
`torch.Tensor`: Normalized tensor with the same shape as the input and
|
||||
dtype `torch.float32`.
|
||||
|
||||
"""
|
||||
if not torch.is_floating_point(image):
|
||||
image = image.float()
|
||||
@@ -209,6 +212,7 @@ def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor:
|
||||
|
||||
Raises:
|
||||
ValueError: If `height` or `width` is not divisible by `patch_size`.
|
||||
|
||||
"""
|
||||
num_images, height, width, channels = image.shape
|
||||
if height % patch_size or width % patch_size:
|
||||
@@ -265,6 +269,7 @@ def process_vision_for_patches(
|
||||
/ patch_size, channels * patch_size**2)` and `dims_virtual` encodes
|
||||
effective `(images, height, width)` dimensions after optional pixel
|
||||
shuffling.
|
||||
|
||||
"""
|
||||
# Add batch dim if single image
|
||||
if images.dim() == 3:
|
||||
|
||||
@@ -28,8 +28,7 @@ class KimiK25Processor(ProcessorMixin):
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
text: The text to be field to the model.
|
||||
vision_chunks: List of `VisionChunk` items to be processed.
|
||||
For image: `VisionChunkImage` with
|
||||
@@ -44,6 +43,7 @@ class KimiK25Processor(ProcessorMixin):
|
||||
Returned when `vision_chunks` is not `None`.
|
||||
- **grid_thws** -- list of image 3D grid in LLM.
|
||||
Returned when `vision_chunks` is not `None`.
|
||||
|
||||
"""
|
||||
if vision_chunks is not None:
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
|
||||
@@ -1071,6 +1071,8 @@ class MiMoOmniProcessor(ProcessorMixin):
|
||||
tuples (pre-decoded).
|
||||
audio: list of ``str`` (path/url/base64), ``bytes``,
|
||||
``(waveform_1D, sample_rate)`` tuples, or ``np.ndarray``.
|
||||
video_audio: audio track extracted from ``videos``, in the same
|
||||
formats accepted by ``audio``.
|
||||
return_tensors: Passed to :class:`BatchFeature`.
|
||||
|
||||
Returns:
|
||||
@@ -1079,6 +1081,7 @@ class MiMoOmniProcessor(ProcessorMixin):
|
||||
- ``pixel_values`` + ``image_grid_thw``
|
||||
- ``pixel_values_videos`` + ``video_grid_thw`` + ``second_per_grid_ts``
|
||||
- ``audio_features``
|
||||
|
||||
"""
|
||||
if isinstance(text, list):
|
||||
text = text[0] if len(text) == 1 else "\n".join(text)
|
||||
|
||||
@@ -14,9 +14,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Processor class for MiniCPMO.
|
||||
"""
|
||||
"""Processor class for MiniCPMO."""
|
||||
|
||||
import math
|
||||
from typing import Literal, TypeAlias
|
||||
@@ -35,8 +33,7 @@ MiniCPMOBatchFeature: TypeAlias = BatchFeature
|
||||
|
||||
|
||||
class MiniCPMOProcessor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs a MiniCPMV processor which wraps a MiniCPMV image
|
||||
r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image
|
||||
processor and a MiniCPMV tokenizer into a single processor.
|
||||
|
||||
[`MiniCPMVProcessor`] offers all the functionalities of
|
||||
@@ -49,6 +46,7 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
The image processor is a required input.
|
||||
tokenizer ([`LlamaTokenizerWrapper`], *optional*):
|
||||
The tokenizer is a required input.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "feature_extractor", "tokenizer"]
|
||||
@@ -253,8 +251,7 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode
|
||||
# with CLIP->Llama
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.batch_decode`]. Please refer to the
|
||||
docstring of this method for more information.
|
||||
"""
|
||||
@@ -274,8 +271,7 @@ class MiniCPMOProcessor(ProcessorMixin):
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode
|
||||
# with CLIP->Llama
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.decode`]. Please refer to the docstring
|
||||
of this method for more information.
|
||||
"""
|
||||
@@ -551,15 +547,11 @@ class MelSpectrogramFeatures(torch.nn.Module):
|
||||
)
|
||||
|
||||
def __call__(self, audio: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
audio: Tensor([num_channels, num_samples])
|
||||
"""
|
||||
"""audio: Tensor([num_channels, num_samples])."""
|
||||
return super().__call__(audio)
|
||||
|
||||
def forward(self, audio: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
audio: Tensor([num_channels, num_samples])
|
||||
"""
|
||||
"""audio: Tensor([num_channels, num_samples])."""
|
||||
mel: torch.Tensor = self.mel_spec(audio)
|
||||
features = torch.log(torch.clip(mel, min=1e-5))
|
||||
return features
|
||||
|
||||
@@ -13,9 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Processor class for MiniCPMV.
|
||||
"""
|
||||
"""Processor class for MiniCPMV."""
|
||||
|
||||
from typing import TypeAlias
|
||||
|
||||
@@ -36,8 +34,7 @@ MiniCPMVBatchFeature: TypeAlias = BatchFeature
|
||||
|
||||
|
||||
class MiniCPMVProcessor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs a MiniCPMV processor which wraps a MiniCPMV image
|
||||
r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image
|
||||
processor and a MiniCPMV tokenizer into a single processor.
|
||||
|
||||
[`MiniCPMVProcessor`] offers all the functionalities of
|
||||
@@ -50,6 +47,7 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
The image processor is a required input.
|
||||
tokenizer ([`LlamaTokenizerWrapper`], *optional*):
|
||||
The tokenizer is a required input.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
@@ -93,8 +91,7 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor
|
||||
# .batch_decode with CLIP->Llama
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.batch_decode`]. Please refer to the
|
||||
docstring of this method for more information.
|
||||
"""
|
||||
@@ -126,8 +123,7 @@ class MiniCPMVProcessor(ProcessorMixin):
|
||||
# Copied from transformers.models.clip.processing_clip.CLIPProcessor
|
||||
# .decode with CLIP->Llama
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to LlamaTokenizerFast's
|
||||
"""This method forwards all its arguments to LlamaTokenizerFast's
|
||||
[`~PythonBackend.decode`]. Please refer to the docstring
|
||||
of this method for more information.
|
||||
"""
|
||||
|
||||
@@ -67,8 +67,7 @@ def select_tiling(
|
||||
|
||||
|
||||
class Moondream3Processor(ProcessorMixin):
|
||||
"""
|
||||
Constructs a Moondream3 processor which handles image preprocessing
|
||||
"""Constructs a Moondream3 processor which handles image preprocessing
|
||||
and tokenization for the Moondream3 multimodal model.
|
||||
|
||||
Args:
|
||||
@@ -78,6 +77,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
max_crops: Maximum number of crops per image.
|
||||
overlap_margin: Margin for overlapping crops in patches.
|
||||
patch_size: Size of each patch.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["tokenizer"]
|
||||
@@ -189,8 +189,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
pretrained_model_name_or_path,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Load the processor, using a separate tokenizer repo.
|
||||
"""Load the processor, using a separate tokenizer repo.
|
||||
|
||||
The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1'
|
||||
instead of having tokenizer files in the model repo.
|
||||
@@ -294,8 +293,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[Moondream3ProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Process images and text for Moondream3 model.
|
||||
"""Process images and text for Moondream3 model.
|
||||
|
||||
Args:
|
||||
images: Input images (PIL Image, numpy array, or list thereof).
|
||||
@@ -304,6 +302,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
|
||||
Returns:
|
||||
BatchFeature with processed inputs.
|
||||
|
||||
"""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Moondream3ProcessorKwargs,
|
||||
@@ -428,8 +427,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
convert_to_rgb: bool = True,
|
||||
return_tensors: str = "pt",
|
||||
) -> tuple[torch.Tensor, tuple[int, int]]:
|
||||
"""
|
||||
Preprocess an image using overlap-and-resize cropping strategy.
|
||||
"""Preprocess an image using overlap-and-resize cropping strategy.
|
||||
|
||||
Args:
|
||||
image: Input PIL image, numpy array, or torch tensor.
|
||||
@@ -442,6 +440,7 @@ class Moondream3Processor(ProcessorMixin):
|
||||
|
||||
Returns:
|
||||
Tuple of (pixel_values tensor, tiling tuple).
|
||||
|
||||
"""
|
||||
image = self._to_pil_image(image)
|
||||
if convert_to_rgb:
|
||||
|
||||
@@ -292,8 +292,7 @@ class DynamicResolutionImageTiler:
|
||||
self,
|
||||
target_num_tokens_post_shuffle: int,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
TODO: optimize this so it squeezes closer to target number of tokens.
|
||||
"""TODO: optimize this so it squeezes closer to target number of tokens.
|
||||
Calculate image dimensions that produce approximately `target` tokens after
|
||||
pixel_shuffle.
|
||||
|
||||
@@ -318,6 +317,7 @@ class DynamicResolutionImageTiler:
|
||||
... height // PATCH_SIZE
|
||||
... ) // 2**2 == 8100 # tokens post-shuffle
|
||||
>>> assert tiler._get_num_embeddings(width=width, height=height) == 8100
|
||||
|
||||
"""
|
||||
side_pixels = (
|
||||
math.isqrt(target_num_tokens_post_shuffle)
|
||||
@@ -386,6 +386,7 @@ class DynamicResolutionImageTiler:
|
||||
num_tokens_available: Number of tokens available for this media
|
||||
Returns:
|
||||
DynamicResolutionParams for the media
|
||||
|
||||
"""
|
||||
current_num_tokens_available = num_tokens_available
|
||||
assert isinstance(media, Image.Image), (
|
||||
@@ -473,6 +474,7 @@ class DynamicResolutionImageTiler:
|
||||
num_tokens_available: Total number of tokens available across all media
|
||||
Returns:
|
||||
List of ImageTilingParams for each media item
|
||||
|
||||
"""
|
||||
num_tokens_available = (
|
||||
num_tokens_available
|
||||
@@ -571,8 +573,7 @@ class DynamicResolutionImageTiler:
|
||||
|
||||
|
||||
class BaseNanoNemotronVLProcessor(ABC):
|
||||
"""
|
||||
This model doesn't define its own HF processor,
|
||||
"""This model doesn't define its own HF processor,
|
||||
so we implement our own one here.
|
||||
|
||||
The code to insert image tokens is based on:
|
||||
@@ -761,8 +762,7 @@ class BaseNanoNemotronVLProcessor(ABC):
|
||||
|
||||
|
||||
class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
"""
|
||||
HF Processor with extended video processing logic.
|
||||
"""HF Processor with extended video processing logic.
|
||||
Code for video processing is adapted from video example:
|
||||
https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
|
||||
"""
|
||||
@@ -1125,8 +1125,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
|
||||
img_context_token_ids: list[int],
|
||||
video_temporal_patch_size: int = 1,
|
||||
) -> PromptUpdateDetails:
|
||||
"""
|
||||
Build prompt replacement for a video.
|
||||
"""Build prompt replacement for a video.
|
||||
The replacement returned is not actually used to replace the placeholder
|
||||
tokens - it's just used to make sure we allocate the correct number
|
||||
of tokens.
|
||||
|
||||
@@ -191,8 +191,7 @@ class LlamaNemotronNanoVLImageProcessor(InternVLImageProcessor):
|
||||
|
||||
|
||||
class LlamaNemotronNanoVLProcessor(InternVLProcessor):
|
||||
"""
|
||||
This model doesn't define its own HF processor,
|
||||
"""This model doesn't define its own HF processor,
|
||||
so we implement our own one here.
|
||||
|
||||
The image processor is given by:
|
||||
@@ -287,8 +286,7 @@ class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor):
|
||||
|
||||
|
||||
class LlamaNemotronVLEmbedProcessor(InternVLProcessor):
|
||||
"""
|
||||
Processor for LlamaNemotronVL embedding model.
|
||||
"""Processor for LlamaNemotronVL embedding model.
|
||||
|
||||
Inherits from NemotronVLProcessor and specializes it for embedding tasks:
|
||||
- Uses SigLIP transform with normalization instead of base transform
|
||||
|
||||
@@ -50,10 +50,10 @@ class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-a
|
||||
|
||||
|
||||
class OvisProcessor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor.
|
||||
r"""Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor.
|
||||
[`OvisProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the
|
||||
[`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] for more information.
|
||||
|
||||
Args:
|
||||
image_processor ([`Qwen2VLImageProcessor`], *optional*):
|
||||
The image processor is a required input.
|
||||
@@ -61,6 +61,7 @@ class OvisProcessor(ProcessorMixin):
|
||||
The tokenizer is a required input.
|
||||
chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
|
||||
in a chat into a tokenizable string.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
@@ -107,12 +108,12 @@ class OvisProcessor(ProcessorMixin):
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[OvisProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
||||
"""Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
|
||||
and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode
|
||||
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to
|
||||
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`.
|
||||
Args:
|
||||
|
||||
Args:
|
||||
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
|
||||
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
|
||||
tensor. Both channels-first and channels-last formats are supported.
|
||||
@@ -129,7 +130,8 @@ class OvisProcessor(ProcessorMixin):
|
||||
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
||||
- `'np'`: Return NumPy `np.ndarray` objects.
|
||||
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
||||
Returns:
|
||||
|
||||
Returns:
|
||||
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
||||
- **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
|
||||
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
|
||||
@@ -140,8 +142,8 @@ class OvisProcessor(ProcessorMixin):
|
||||
- **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`.
|
||||
- **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`.
|
||||
- **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`.
|
||||
"""
|
||||
|
||||
"""
|
||||
max_partition = kwargs.pop("max_partition", 9)
|
||||
covering_threshold = kwargs.pop("covering_threshold", 0.9)
|
||||
|
||||
@@ -416,28 +418,28 @@ class OvisProcessor(ProcessorMixin):
|
||||
return torch.tensor(pixel_values), image_placeholders, torch.tensor(grid)
|
||||
|
||||
def batch_decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please
|
||||
"""This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please
|
||||
refer to the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.batch_decode(*args, **kwargs)
|
||||
|
||||
def decode(self, *args, **kwargs):
|
||||
"""
|
||||
This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to
|
||||
"""This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to
|
||||
the docstring of this method for more information.
|
||||
"""
|
||||
return self.tokenizer.decode(*args, **kwargs)
|
||||
|
||||
def post_process_image_text_to_text(self, generated_outputs):
|
||||
"""
|
||||
Post-process the output of the model to decode the text.
|
||||
"""Post-process the output of the model to decode the text.
|
||||
|
||||
Args:
|
||||
generated_outputs (`torch.Tensor` or `np.ndarray`):
|
||||
The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
|
||||
or `(sequence_length,)`.
|
||||
|
||||
Returns:
|
||||
`list[str]`: The decoded text.
|
||||
|
||||
"""
|
||||
return self.tokenizer.batch_decode(
|
||||
generated_outputs,
|
||||
|
||||
@@ -33,13 +33,13 @@ class Ovis2_5ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[cal
|
||||
|
||||
|
||||
class Ovis2_5Processor(ProcessorMixin):
|
||||
r"""
|
||||
Constructs an Ovis processor which wraps an Ovis image processor
|
||||
r"""Constructs an Ovis processor which wraps an Ovis image processor
|
||||
and a Qwen2 tokenizer into a single processor.
|
||||
[`OvisProcessor`] offers all the functionalities of
|
||||
[`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`].
|
||||
See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`]
|
||||
for more information.
|
||||
|
||||
Args:
|
||||
image_processor ([`Qwen2VLImageProcessor`], *optional*):
|
||||
The image processor is a required input.
|
||||
@@ -48,6 +48,7 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
chat_template (`str`, *optional*): A Jinja template which will
|
||||
be used to convert lists of messages in a chat into
|
||||
a tokenizable string.
|
||||
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
@@ -110,43 +111,39 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
| list[PreTokenizedInput] = None,
|
||||
**kwargs: Unpack[Ovis2_5ProcessorKwargs],
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Main method to prepare for the model one or several sequences(s)
|
||||
"""Main method to prepare for the model one or several sequences(s)
|
||||
and image(s). This method forwards the `text`and `kwargs` arguments
|
||||
to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text`
|
||||
is not `None` to encode the text. To prepare the vision inputs,
|
||||
this method forwards the `vision_infos` and `kwrags` arguments to
|
||||
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`]
|
||||
if `vision_infos` is not `None`.
|
||||
Args:
|
||||
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`,
|
||||
`list[PIL.Image.Image]`, `list[np.ndarray]`,
|
||||
`list[torch.Tensor]`):
|
||||
The image or batch of images to be prepared.
|
||||
Each image can be a PIL image, NumPy array or PyTorch
|
||||
tensor. Both channels-first and channels-last formats
|
||||
are supported.
|
||||
text (`str`, `list[str]`, `list[list[str]]`):
|
||||
The sequence or batch of sequences to be encoded.
|
||||
Each sequence can be a string or a list of strings
|
||||
(pretokenized string). If the sequences are provided as
|
||||
list of strings (pretokenized), you must set
|
||||
`is_split_into_words=True` (to lift the ambiguity with
|
||||
a batch of sequences).
|
||||
videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`,
|
||||
`list[torch.Tensor]`):
|
||||
The image or batch of videos to be prepared. Each video
|
||||
can be a 4D NumPy array or PyTorch tensor, or a nested
|
||||
list of 3D frames. Both channels-first and channels-last
|
||||
formats are supported.
|
||||
return_tensors (`str` or [`~utils.TensorType`], *optional*):
|
||||
If set, will return tensors of a particular framework.
|
||||
Acceptable values are:
|
||||
- `'tf'`: Return TensorFlow `tf.constant` objects.
|
||||
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
||||
- `'np'`: Return NumPy `np.ndarray` objects.
|
||||
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
||||
Returns:
|
||||
|
||||
Args:
|
||||
images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, or a list):
|
||||
The image or batch of images to be prepared. Each image can
|
||||
be a PIL image, NumPy array or PyTorch tensor. Both
|
||||
channels-first and channels-last formats are supported.
|
||||
videos (`np.ndarray`, `torch.Tensor`, or a list of either):
|
||||
The video or batch of videos to be prepared. Each video can
|
||||
be a 4D NumPy array or PyTorch tensor, or a nested list of
|
||||
3D frames. Both channels-first and channels-last formats are
|
||||
supported.
|
||||
text (`str`, `list[str]`, `list[list[str]]`):
|
||||
The sequence or batch of sequences to be encoded. Each
|
||||
sequence can be a string or a list of strings (pretokenized
|
||||
string). If the sequences are provided as list of strings
|
||||
(pretokenized), you must set `is_split_into_words=True` (to
|
||||
lift the ambiguity with a batch of sequences).
|
||||
return_tensors (`str` or [`~utils.TensorType`], *optional*):
|
||||
If set, will return tensors of a particular framework.
|
||||
Acceptable values are:
|
||||
- `'tf'`: Return TensorFlow `tf.constant` objects.
|
||||
- `'pt'`: Return PyTorch `torch.Tensor` objects.
|
||||
- `'np'`: Return NumPy `np.ndarray` objects.
|
||||
- `'jax'`: Return JAX `jnp.ndarray` objects.
|
||||
|
||||
Returns:
|
||||
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
|
||||
- **input_ids** -- list of token ids to be fed to a model.
|
||||
Returned when `text` is not `None`.
|
||||
@@ -164,6 +161,7 @@ class Ovis2_5Processor(ProcessorMixin):
|
||||
when `videos` is not `None`.
|
||||
- **second_per_grid_ts** -- list of video seconds per time grid.
|
||||
Returned when `videos` is not `None`.
|
||||
|
||||
"""
|
||||
output_kwargs = self._merge_kwargs(
|
||||
Ovis2_5ProcessorKwargs,
|
||||
|
||||
@@ -11,8 +11,7 @@ from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonImageProcessor(ImageProcessingMixin):
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
"""Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
|
||||
"""
|
||||
|
||||
|
||||
@@ -18,8 +18,7 @@ from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
|
||||
class MistralCommonFeatureExtractor(SequenceFeatureExtractor):
|
||||
"""
|
||||
Provide a HF-compatible interface for
|
||||
"""Provide a HF-compatible interface for
|
||||
`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
|
||||
"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user