Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -42,7 +42,8 @@ def _assert_close_ulps(
|
||||
compare against the reference's local ulp spacing. ``pre_act`` (the
|
||||
reference pre-activation) adds derivative-propagated slack: near
|
||||
``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by
|
||||
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps."""
|
||||
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps.
|
||||
"""
|
||||
g, r = got.float(), ref.float()
|
||||
tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol)
|
||||
if pre_act is not None:
|
||||
|
||||
@@ -123,7 +123,8 @@ def _check_writes_only_its_own_shard(
|
||||
device: torch.device, tp_size: int, rank: int
|
||||
) -> None:
|
||||
"""Two ranks that swapped both weight rows and write offsets still sum to
|
||||
the right total, so inspect each slice before the final collective."""
|
||||
the right total, so inspect each slice before the final collective.
|
||||
"""
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
@@ -130,7 +130,8 @@ def test_disabled_reproduces_the_plain_residual_sum(
|
||||
|
||||
def test_taps_the_consumer_layer_when_one_follows(recorder, monkeypatch):
|
||||
"""The value the next layer reads is the mixture against *its* weights,
|
||||
so the tap has to reach forward rather than use the current layer's."""
|
||||
so the tap has to reach forward rather than use the current layer's.
|
||||
"""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
|
||||
_call(_stub_model(enabled=True), 1, torch.zeros(2), None, torch.zeros(2))
|
||||
@@ -181,7 +182,8 @@ def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
|
||||
):
|
||||
"""The kernel writes an applied delta back into the prefix in place, which
|
||||
would double-add it into the live residual stream, so the pending output
|
||||
has to arrive already summed into the prefix with `delta` left None."""
|
||||
has to arrive already summed into the prefix with `delta` left None.
|
||||
"""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
prefix_sum = torch.tensor([1.0, 2.0])
|
||||
pending = torch.tensor([0.5, 0.25])
|
||||
|
||||
@@ -727,7 +727,8 @@ def test_cudagraph_capture_metadata_avoids_device_to_host_copy(
|
||||
"""`build_for_cudagraph_capture` is not capture-only: a DP rank with nothing
|
||||
scheduled re-stages its FULL-graph metadata through it on every dummy step,
|
||||
so it must not synchronize the device. The host-side draft counts have to
|
||||
come from `query_start_loc_cpu` and match the device-derived values."""
|
||||
come from `query_start_loc_cpu` and match the device-derived values.
|
||||
"""
|
||||
num_speculative_tokens = 2
|
||||
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
@@ -60,7 +60,8 @@ def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
|
||||
"""KV-shared layers have q_proj instead of a packed qkv_proj; their
|
||||
redundant K/V tensors in original checkpoints have no parameter and are
|
||||
skipped rather than failing the load."""
|
||||
skipped rather than failing the load.
|
||||
"""
|
||||
model = torch.nn.Module()
|
||||
model.config = SimpleNamespace(num_experts=0)
|
||||
model.start_layer, model.end_layer = 0, 2
|
||||
@@ -92,7 +93,8 @@ def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma3n_kv_shared_layer_mapper() -> None:
|
||||
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
|
||||
q_proj and drop the redundant K/V tensors original checkpoints ship."""
|
||||
q_proj and drop the redundant K/V tensors original checkpoints ship.
|
||||
"""
|
||||
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
|
||||
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
|
||||
weights = [
|
||||
|
||||
@@ -164,8 +164,7 @@ def test_chunked_prefill_with_parallel_sampling(
|
||||
max_tokens: int,
|
||||
conv_state_layout: str,
|
||||
) -> None:
|
||||
"""
|
||||
Tests chunked prefill in conjunction with n > 1.
|
||||
"""Tests chunked prefill in conjunction with n > 1.
|
||||
|
||||
In this case, prefill is populated with decoding tokens and
|
||||
we test that it doesn't fail.
|
||||
@@ -199,8 +198,7 @@ def test_mamba_cache_cg_padding(
|
||||
max_tokens: int,
|
||||
conv_state_layout: str,
|
||||
) -> None:
|
||||
"""
|
||||
This test is for verifying that mamba cache is padded to CG captured
|
||||
"""This test is for verifying that mamba cache is padded to CG captured
|
||||
batch size. If it's not, a torch RuntimeError will be raised because
|
||||
tensor dimensions aren't compatible.
|
||||
"""
|
||||
@@ -236,8 +234,7 @@ def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
|
||||
example_prompts,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""
|
||||
This test is for verifying that the hybrid inner state management doesn't
|
||||
"""This test is for verifying that the hybrid inner state management doesn't
|
||||
collapse in case where the number of incoming requests and
|
||||
finished_requests_ids is larger than the maximum mamba block capacity.
|
||||
|
||||
@@ -263,8 +260,7 @@ def test_state_cleanup(
|
||||
example_prompts,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""
|
||||
This test is for verifying that the Hybrid state is cleaned up between
|
||||
"""This test is for verifying that the Hybrid state is cleaned up between
|
||||
steps.
|
||||
|
||||
If it's not cleaned, an error would be expected.
|
||||
|
||||
@@ -274,7 +274,8 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
|
||||
|
||||
def test_mistral_function_call_nested_json():
|
||||
"""Ensure that the function-name regex captures the entire outermost
|
||||
JSON block, including nested braces."""
|
||||
JSON block, including nested braces.
|
||||
"""
|
||||
|
||||
# Create a minimal stub tokenizer that provides the few attributes the
|
||||
# parser accesses (`version` and `get_vocab`).
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for max_tokens_per_doc and max_tokens_per_query.
|
||||
"""
|
||||
"""Tests for max_tokens_per_doc and max_tokens_per_query."""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
@@ -29,7 +29,8 @@ from vllm.v1.worker.gpu.states import RequestState
|
||||
@torch.inference_mode
|
||||
def test_splade_pooler_matches_reference_formula(B, T, H, V):
|
||||
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
|
||||
log1p(relu(logits)) -> max over sequence length (after masking)."""
|
||||
log1p(relu(logits)) -> max over sequence length (after masking).
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Prepare [B] sequences of shape [T, H]
|
||||
|
||||
@@ -23,11 +23,9 @@ NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
|
||||
@pytest.mark.parametrize("dtype", ["float16"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
|
||||
"""
|
||||
This is a simple test to check if interleaved and non-interleaved prompts
|
||||
"""This is a simple test to check if interleaved and non-interleaved prompts
|
||||
give the same result.
|
||||
"""
|
||||
|
||||
image_cherry = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
image_stop = convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB")
|
||||
images = [image_cherry, image_stop]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Create a reduced-layer version of the Maverick model for testing purposes.
|
||||
"""Create a reduced-layer version of the Maverick model for testing purposes.
|
||||
|
||||
This script creates a new model with fewer layers by:
|
||||
1. Loading the original Maverick model configuration
|
||||
@@ -39,7 +38,6 @@ def run_maverick_serving(model: str):
|
||||
"""Test Llama-4-Maverick model with vLLM LLM class using CLI equivalent
|
||||
options with reduced layers.
|
||||
"""
|
||||
|
||||
try:
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
|
||||
|
||||
@@ -71,8 +69,7 @@ def run_maverick_serving(model: str):
|
||||
|
||||
|
||||
def get_rope_layers_config(model_path: str) -> list[int]:
|
||||
"""
|
||||
Get the interleaved RoPE configuration from HuggingFace config
|
||||
"""Get the interleaved RoPE configuration from HuggingFace config.
|
||||
|
||||
Args:
|
||||
model_path: Path to the local directory containing the reduced
|
||||
@@ -81,6 +78,7 @@ def get_rope_layers_config(model_path: str) -> list[int]:
|
||||
Returns:
|
||||
List of 0 or 1 indicating whether each layer uses RoPE and local attn
|
||||
0 indicates that RoPE is not used while 1 indicates that RoPE is used.
|
||||
|
||||
"""
|
||||
config_path = Path(model_path) / "config.json"
|
||||
model_config = json.loads(config_path.read_text())
|
||||
@@ -98,8 +96,7 @@ def create_reduced_maverick_model(
|
||||
vision_layers: int = 2,
|
||||
force_recreate: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
Create a reduced-layer version of the Maverick model.
|
||||
"""Create a reduced-layer version of the Maverick model.
|
||||
|
||||
Args:
|
||||
original_model_name: Name of the original Maverick model
|
||||
@@ -111,8 +108,8 @@ def create_reduced_maverick_model(
|
||||
|
||||
Returns:
|
||||
Path to the created reduced model directory
|
||||
"""
|
||||
|
||||
"""
|
||||
print(
|
||||
f"Creating reduced Maverick model with {text_layers} text layers and "
|
||||
f"{vision_layers} vision layers..."
|
||||
@@ -178,7 +175,6 @@ def create_reduced_config(
|
||||
original_config: Any, text_layers: int, num_experts: int, vision_layers: int
|
||||
) -> dict[str, Any]:
|
||||
"""Create a reduced configuration based on the original."""
|
||||
|
||||
# Convert config to dictionary
|
||||
config_dict = original_config.to_dict()
|
||||
|
||||
@@ -220,7 +216,6 @@ def create_reduced_config(
|
||||
|
||||
def copy_tokenizer_files(original_model_name: str, output_path: Path) -> None:
|
||||
"""Copy tokenizer files from the original model."""
|
||||
|
||||
try:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
original_model_name, trust_remote_code=True
|
||||
@@ -233,7 +228,6 @@ def copy_tokenizer_files(original_model_name: str, output_path: Path) -> None:
|
||||
|
||||
def create_preprocessor_config(original_config: Any, output_path: Path) -> None:
|
||||
"""Create preprocessor_config.json for multimodal model."""
|
||||
|
||||
# Try to load the original preprocessor config
|
||||
try:
|
||||
processor = AutoProcessor.from_pretrained(
|
||||
@@ -253,7 +247,6 @@ def create_reduced_safetensors(
|
||||
original_config: Any, reduced_config: dict[str, Any], output_path: Path
|
||||
) -> None:
|
||||
"""Create safetensors files with weights for the reduced model."""
|
||||
|
||||
print("Generating synthetic weights for reduced model...")
|
||||
|
||||
text_config = reduced_config["text_config"]
|
||||
@@ -276,7 +269,6 @@ def create_reduced_safetensors(
|
||||
|
||||
def create_text_model_weights(text_config: dict[str, Any]) -> dict[str, torch.Tensor]:
|
||||
"""Create synthetic weights for the text model with MoE structure."""
|
||||
|
||||
weights = {}
|
||||
|
||||
vocab_size = text_config["vocab_size"]
|
||||
@@ -405,7 +397,6 @@ def create_vision_model_weights(
|
||||
vision_config: dict[str, Any],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Create synthetic weights for the vision model."""
|
||||
|
||||
weights = {}
|
||||
|
||||
hidden_size = vision_config["hidden_size"]
|
||||
@@ -473,8 +464,7 @@ def create_vision_model_weights(
|
||||
def create_shared_weights(
|
||||
text_config: dict[str, Any], vision_config: dict[str, Any]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Create weights for shared components (vision-language connector)"""
|
||||
|
||||
"""Create weights for shared components (vision-language connector)."""
|
||||
weights = {}
|
||||
|
||||
text_hidden_size = text_config["hidden_size"]
|
||||
@@ -492,7 +482,6 @@ def save_weights_to_safetensors(
|
||||
weights: dict[str, torch.Tensor], output_path: Path
|
||||
) -> None:
|
||||
"""Save weights to safetensors files and create index."""
|
||||
|
||||
# Determine how to shard the weights
|
||||
max_shard_size = 5 * 1024 * 1024 * 1024 # 5GB per shard
|
||||
|
||||
@@ -655,7 +644,6 @@ def test_dummy_maverick(
|
||||
|
||||
def main():
|
||||
"""Main function to create and test the reduced model."""
|
||||
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
|
||||
@@ -33,7 +33,8 @@ NUM_TOKENS = 3
|
||||
|
||||
class _StubGate(torch.nn.Module):
|
||||
"""Router stand-in, so the test does not depend on platform-specific
|
||||
``ReplicatedLinear`` weight post-processing."""
|
||||
``ReplicatedLinear`` weight post-processing.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size: int, num_experts: int):
|
||||
super().__init__()
|
||||
|
||||
@@ -31,8 +31,7 @@ def mask_bbox_tokens(
|
||||
output: tuple[list[int], str, SampleLogprobs],
|
||||
tokenizer: TokenizerLike,
|
||||
) -> tuple[list[int], str, SampleLogprobs]:
|
||||
"""
|
||||
Always pass check_logprobs_close check for bounding box tokens
|
||||
"""Always pass check_logprobs_close check for bounding box tokens
|
||||
because it is reasonable for them to differ slightly.
|
||||
"""
|
||||
ignore_pattern = r"<[xy]_[\d.]+>"
|
||||
|
||||
@@ -97,7 +97,7 @@ def batch_make_image_embeddings(
|
||||
processor,
|
||||
llm: VllmRunner,
|
||||
) -> list[Qwen2VLPromptImageEmbeddingInput]:
|
||||
"""batched image embeddings for Qwen2-VL
|
||||
"""Batched image embeddings for Qwen2-VL.
|
||||
|
||||
This will infer all images' embeddings in a single batch,
|
||||
and split the result according to input batches.
|
||||
@@ -108,7 +108,6 @@ def batch_make_image_embeddings(
|
||||
|
||||
returns: `list[Qwen2VLPromptImageEmbeddingInput]`
|
||||
"""
|
||||
|
||||
image_batches_: list[Any] = image_batches[:]
|
||||
|
||||
# convert single-image batches to multiple-image batches
|
||||
@@ -183,7 +182,7 @@ def batch_make_image_embeddings(
|
||||
def batch_make_video_embeddings(
|
||||
video_batches: PromptVideoInput, processor, llm: VllmRunner
|
||||
) -> list[Qwen2VLPromptVideoEmbeddingInput]:
|
||||
"""batched video embeddings for Qwen2-VL
|
||||
"""Batched video embeddings for Qwen2-VL.
|
||||
|
||||
A NDArray represents a single video's all frames.
|
||||
|
||||
@@ -194,7 +193,6 @@ def batch_make_video_embeddings(
|
||||
- Single-video batches: `list[NDArray]`
|
||||
- Multiple-video batches: `list[list[NDArray]]`
|
||||
"""
|
||||
|
||||
video_batches_: list[Any] = video_batches[:]
|
||||
|
||||
for idx in range(len(video_batches_)):
|
||||
|
||||
@@ -81,7 +81,8 @@ def test_transformers_audio_generation(
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
|
||||
deadlock and allows worker processes to terminate cleanly, and release
|
||||
GPU memory between test runs until the issue is fixed."""
|
||||
GPU memory between test runs until the issue is fixed.
|
||||
"""
|
||||
# TODO: Remove monkeypatch once
|
||||
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
|
||||
disable_envs_cache()
|
||||
|
||||
@@ -204,7 +204,6 @@ def test_variable_length_audio_batching(
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_serving(client, audio_assets: AudioTestAssets):
|
||||
"""Exercises online serving with/without chunked prefill enabled."""
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
|
||||
@@ -78,7 +78,6 @@ def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
|
||||
|
||||
Steps run sequentially so each releases the GPU before the next starts.
|
||||
"""
|
||||
|
||||
question = f"What's happening in these {len(audio_assets)} audio clips?"
|
||||
max_tokens = 10
|
||||
audio_data = [asset.audio_and_sample_rate for asset in audio_assets]
|
||||
|
||||
@@ -294,7 +294,8 @@ def sample_frames_with_video_metadata(
|
||||
|
||||
def apply_image_size_scaling(image, size: float | tuple[int, int], size_type: SizeType):
|
||||
"""Applies a size scaler to one image; this can be an image size factor,
|
||||
which scales the image while maintaining the aspect ratio"""
|
||||
which scales the image while maintaining the aspect ratio
|
||||
"""
|
||||
# Special case for embeddings; if it's a tensor, it's only valid if we
|
||||
# are considering size factors at constant scale, i.e., we just clone
|
||||
# the tensor
|
||||
|
||||
@@ -156,6 +156,7 @@ def get_wrapped_test_sizes(
|
||||
Args:
|
||||
test_info: Test configuration to be expanded.
|
||||
test_type: The type of test being filtered for.
|
||||
|
||||
"""
|
||||
# If it is an embedding test, we always use the EMBEDDING_SIZE_FACTORS
|
||||
if test_type == VLMTestType.EMBEDDING:
|
||||
|
||||
@@ -200,7 +200,7 @@ def process_runner_outputs(
|
||||
|
||||
|
||||
def process_outputs(output_processor, model, outputs_per_image):
|
||||
"""Applies a model specific post-processor function to a runner's output"""
|
||||
"""Applies a model specific post-processor function to a runner's output."""
|
||||
return [
|
||||
[output_processor(res, model) for res in outputs]
|
||||
for outputs in outputs_per_image
|
||||
|
||||
@@ -22,6 +22,7 @@ def multi_image_multi_aspect_ratio_inputs(formatter: Callable[[str], str]):
|
||||
|
||||
Args:
|
||||
formatter: model-specific prompt formatter.
|
||||
|
||||
"""
|
||||
stop_sign = IMAGE_ASSETS[0].pil_image
|
||||
cherry_blossom = IMAGE_ASSETS[1].pil_image
|
||||
@@ -65,6 +66,7 @@ def multi_video_multi_aspect_ratio_inputs(
|
||||
|
||||
Args:
|
||||
formatter: model-specific prompt formatter.
|
||||
|
||||
"""
|
||||
video = sample_frames_from_video(VIDEO_ASSETS[0].np_ndarrays, num_frames)
|
||||
# Apply the selected formatter to the base prompts
|
||||
|
||||
@@ -252,6 +252,7 @@ def qwen_prompt_path_encoder(
|
||||
tmp_path: Tempdir for test under consideration.
|
||||
prompt: Prompt with image placeholders.
|
||||
assets: list of image assets whose len equals the num placeholders.
|
||||
|
||||
"""
|
||||
# Ensure that the number of placeholders matches the number of assets;
|
||||
# If this is not true, the test is probably written incorrectly.
|
||||
@@ -507,15 +508,12 @@ def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
2) Ensure IsaacModel.forward returns hidden_states
|
||||
for compatibility with hidden_states_to_seq_logprobs()
|
||||
"""
|
||||
|
||||
from perceptron.tensorstream import TextType
|
||||
from perceptron.tensorstream.ops import compute_mrope_pos_tensor, modality_mask
|
||||
from transformers.modeling_outputs import BaseModelOutputWithPast
|
||||
|
||||
def compute_position_ids_input_ids(input_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Create 3D positional indices for token input.
|
||||
"""
|
||||
"""Create 3D positional indices for token input."""
|
||||
batch_size, seq_length = input_ids.shape
|
||||
position_ids = torch.arange(seq_length, device=input_ids.device)
|
||||
position_ids = position_ids.view(1, -1).expand(batch_size, -1)
|
||||
@@ -570,8 +568,7 @@ def isaac_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
cache_position=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Forward pass with MRoPE position embeddings.
|
||||
"""Forward pass with MRoPE position embeddings.
|
||||
Computes position embeddings once and passes them through all layers.
|
||||
"""
|
||||
output_hidden_states = (
|
||||
@@ -1184,7 +1181,6 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
|
||||
HfRunner.generate calls batch_decode on the full sequence (prompt +
|
||||
generated).
|
||||
"""
|
||||
|
||||
import io
|
||||
|
||||
import pybase64 as base64
|
||||
|
||||
@@ -87,7 +87,7 @@ def _run_test(
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""SET PYTHONPATH"""
|
||||
"""SET PYTHONPATH."""
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
|
||||
@@ -108,8 +108,7 @@ def _normalize_image(image_val: str) -> str:
|
||||
def create_score_multimodal_param(
|
||||
content_parts: list[dict],
|
||||
) -> list[ScoreMultiModalParam]:
|
||||
"""
|
||||
Create a ScoreMultiModalParam from a list of content dictionaries.
|
||||
"""Create a ScoreMultiModalParam from a list of content dictionaries.
|
||||
|
||||
Each dict supports the following formats:
|
||||
- Text: {'text': 'content'}
|
||||
@@ -257,7 +256,7 @@ def test_model_text_image(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Visual Documents Reranking"""
|
||||
"""Visual Documents Reranking."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
@@ -280,7 +279,7 @@ def test_model_text_text(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Textual Documents Reranking"""
|
||||
"""Textual Documents Reranking."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
@@ -303,7 +302,7 @@ def test_model_image_text(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Image Querying for Textual Documents"""
|
||||
"""Image Querying for Textual Documents."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
@@ -326,7 +325,7 @@ def test_model_image_image(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Image Querying for Image Documents"""
|
||||
"""Image Querying for Image Documents."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
@@ -349,7 +348,7 @@ def test_model_text_mixed_documents(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Text Query for Mixed Text and Image Documents"""
|
||||
"""Text Query for Mixed Text and Image Documents."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for the LlamaNemotronVL model family:
|
||||
"""Tests for the LlamaNemotronVL model family:
|
||||
- nvidia/llama-nemotron-embed-vl-1b-v2 (LlamaNemotronVLForCausalLM / embed)
|
||||
- nvidia/llama-nemotron-rerank-vl-1b-v2
|
||||
(LlamaNemotronVLForSequenceClassification / rerank)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
|
||||
"""Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
|
||||
caching.
|
||||
|
||||
Tests the use_audio_in_video feature where audio is extracted from video and
|
||||
@@ -50,8 +49,7 @@ def create_mm_data(num_videos: int) -> dict[str, list]:
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
@pytest.mark.parametrize("num_videos", [1, 2])
|
||||
def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
|
||||
"""
|
||||
Regression test for https://github.com/vllm-project/vllm/pull/36800
|
||||
"""Regression test for https://github.com/vllm-project/vllm/pull/36800.
|
||||
|
||||
MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
|
||||
on a cache hit, so mm_kwargs["video"] items become None on the second call.
|
||||
@@ -117,8 +115,7 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
|
||||
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
||||
"""
|
||||
A video with no audio track, combined with `use_audio_in_video=True`,
|
||||
"""A video with no audio track, combined with `use_audio_in_video=True`,
|
||||
must raise a clear `ValueError` naming the mismatch, rather than an
|
||||
opaque `StopIteration`-derived error from the underlying HF processor.
|
||||
"""
|
||||
|
||||
@@ -33,9 +33,7 @@ from ...registry import (
|
||||
|
||||
|
||||
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
"""
|
||||
Add metadata to video mm_data
|
||||
"""
|
||||
"""Add metadata to video mm_data."""
|
||||
|
||||
def create_metadata(frames: np.ndarray):
|
||||
num_frames = len(frames)
|
||||
@@ -61,8 +59,7 @@ def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
|
||||
|
||||
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
"""
|
||||
Patch the multimodal data for GLM-ASR model.
|
||||
"""Patch the multimodal data for GLM-ASR model.
|
||||
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
|
||||
"""
|
||||
if "audio" in mm_data:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
|
||||
"""Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
|
||||
|
||||
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
|
||||
images that are small enough to not require cropping produce an empty
|
||||
@@ -109,7 +108,8 @@ class TestDeepseekOCREmptyImagesCrop:
|
||||
|
||||
def test_mismatched_image_size_raises(self, processor):
|
||||
"""Deliberately wrong image_size binding should still be caught
|
||||
by TensorSchema validation."""
|
||||
by TensorSchema validation.
|
||||
"""
|
||||
small_image = Image.new("RGB", (100, 100), color="green")
|
||||
|
||||
result = processor(
|
||||
@@ -143,7 +143,8 @@ class TestDeepseekOCRInputValidation:
|
||||
def test_mismatched_image_count_raises_value_error(self, processor):
|
||||
"""Two ``<image>`` tokens with one image must raise ValueError, not
|
||||
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
|
||||
drop the middle text segment)."""
|
||||
drop the middle text segment).
|
||||
"""
|
||||
image = Image.new("RGB", (100, 100), color="red")
|
||||
with pytest.raises(ValueError, match="does not match number of images"):
|
||||
processor(prompt="<image> and <image>", images=[image])
|
||||
|
||||
@@ -152,8 +152,7 @@ def test_video_loader_consistency(
|
||||
fps: int,
|
||||
backend: str,
|
||||
):
|
||||
"""
|
||||
Ensure dynamic video loader (pre-sampled by loader) and normal video
|
||||
"""Ensure dynamic video loader (pre-sampled by loader) and normal video
|
||||
loader (post-sampled by processor) produce same video processing outputs.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
|
||||
@@ -116,8 +116,7 @@ def _test_image_prompt_replacements(
|
||||
num_imgs: int,
|
||||
image_sizes: list[ImageSize],
|
||||
) -> None:
|
||||
"""
|
||||
Ensure LlavaNextMultiModalProcessor
|
||||
"""Ensure LlavaNextMultiModalProcessor
|
||||
handles prompt replacement properly for input images.
|
||||
"""
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
@@ -16,7 +16,8 @@ from ...utils import build_model_context
|
||||
class _StubModel:
|
||||
"""Carries only the attributes the token-count method reads from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
the full `nn.Module` (vision tower, language model, etc.).
|
||||
"""
|
||||
|
||||
patch_grid_length: int
|
||||
pooled_grid_length: int
|
||||
@@ -31,7 +32,8 @@ get_mm_lora_token_counts = (
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the LoRA token counts must preserve the
|
||||
frame-level scaling exactly."""
|
||||
frame-level scaling exactly.
|
||||
"""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
|
||||
|
||||
@@ -122,8 +122,7 @@ def _test_image_prompt_replacements(
|
||||
num_imgs: int,
|
||||
image_sizes: list[ImageSize],
|
||||
) -> None:
|
||||
"""
|
||||
Ensure LlavaOnevisionMultiModalProcessor
|
||||
"""Ensure LlavaOnevisionMultiModalProcessor
|
||||
handles prompt replacement properly for input images.
|
||||
"""
|
||||
failed_size_excs = list[tuple[ImageSize, Exception]]()
|
||||
|
||||
@@ -70,7 +70,8 @@ def test_dummy_video_num_frames_override_honors_min_of_two(
|
||||
):
|
||||
"""A ``num_frames`` override below 2 must be ignored (the model needs at
|
||||
least 2 frames), matching the "cannot be less than 2, will be ignored"
|
||||
warning."""
|
||||
warning.
|
||||
"""
|
||||
builder = object.__new__(Molmo2DummyInputsBuilder)
|
||||
builder.info = SimpleNamespace(
|
||||
get_hf_processor=lambda: SimpleNamespace(
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
|
||||
"""Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
|
||||
correctly assigned to audio/image/video token positions.
|
||||
|
||||
Regression test for: https://github.com/vllm-project/vllm/issues/34506
|
||||
@@ -40,8 +39,7 @@ def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tens
|
||||
def make_token_seq(
|
||||
audio_n: int, image_n: int, video_n: int, text_prefix: int = 3, text_sep: int = 2
|
||||
):
|
||||
"""
|
||||
Build a flat token sequence:
|
||||
"""Build a flat token sequence:
|
||||
[text_prefix] [AUDIO * audio_n] [text_sep] [IMAGE * image_n]
|
||||
[text_sep] [VIDEO * video_n] [text_sep]
|
||||
Returns (input_ids tensor, is_multimodal mask, positions dict).
|
||||
@@ -67,9 +65,8 @@ def make_token_seq(
|
||||
def make_interleaved_seq(
|
||||
video_chunks: list[int], audio_chunks: list[int], text_prefix: int = 2
|
||||
):
|
||||
"""
|
||||
Build an interleaved sequence like use_audio_in_video:
|
||||
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
|
||||
"""Build an interleaved sequence like use_audio_in_video:
|
||||
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
|
||||
"""
|
||||
tokens = [TEXT_TOKEN_ID] * text_prefix
|
||||
for v, a in zip(video_chunks, audio_chunks):
|
||||
@@ -140,8 +137,7 @@ class TestCheckInterleavedAudioVideo:
|
||||
)
|
||||
|
||||
def test_batched_non_interleaved_no_false_positive(self):
|
||||
"""
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/35394.
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/35394.
|
||||
|
||||
5 identical non-interleaved mixed-modality requests batched together:
|
||||
each has [audio][image][video] in separate blocks with text between them.
|
||||
@@ -172,8 +168,7 @@ class TestCheckInterleavedAudioVideo:
|
||||
|
||||
|
||||
def make_mock_model(hidden: int = 8):
|
||||
"""
|
||||
Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
|
||||
"""Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
|
||||
that has enough structure to run embed_input_ids.
|
||||
"""
|
||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
@@ -236,8 +231,7 @@ def make_mock_model(hidden: int = 8):
|
||||
def build_mm_embeds(
|
||||
audio_n, image_n, video_n, hidden, audio_val=10.0, image_val=20.0, video_val=30.0
|
||||
):
|
||||
"""
|
||||
Build multimodal_embeddings list in position order (audio, image, video).
|
||||
"""Build multimodal_embeddings list in position order (audio, image, video).
|
||||
Each embedding is filled with a distinct constant so we can verify placement.
|
||||
"""
|
||||
embs = []
|
||||
@@ -252,8 +246,7 @@ def build_mm_embeds(
|
||||
|
||||
class TestEmbedInputIds:
|
||||
def _run(self, audio_n, image_n, video_n, hidden=8):
|
||||
"""
|
||||
Run embed_input_ids for a non-interleaved mixed-modality sequence.
|
||||
"""Run embed_input_ids for a non-interleaved mixed-modality sequence.
|
||||
Returns (result_embeds, input_ids, is_multimodal).
|
||||
"""
|
||||
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
|
||||
@@ -288,8 +281,7 @@ class TestEmbedInputIds:
|
||||
)
|
||||
|
||||
def test_mixed_modalities_audio_goes_to_audio_pos(self):
|
||||
"""
|
||||
Regression test for GitHub issue #34506:
|
||||
"""Regression test for GitHub issue #34506:
|
||||
With audio + image + video (non-interleaved), audio positions must
|
||||
receive audio embeddings (not image or video embeddings).
|
||||
"""
|
||||
@@ -329,8 +321,7 @@ class TestEmbedInputIds:
|
||||
)
|
||||
|
||||
def test_interleaved_use_audio_in_video(self):
|
||||
"""
|
||||
Interleaved (use_audio_in_video): video chunks interleaved with audio.
|
||||
"""Interleaved (use_audio_in_video): video chunks interleaved with audio.
|
||||
Video embeddings must go to video positions, audio to audio positions.
|
||||
"""
|
||||
hidden = 8
|
||||
|
||||
@@ -232,7 +232,6 @@ def test_mm_device_do_normalize(
|
||||
num_imgs: int,
|
||||
):
|
||||
"""Ensure that enable mm_device_do_normalize yields the correct result."""
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
|
||||
@@ -25,8 +25,7 @@ def test_processor_with_audio_sample_rate(
|
||||
audio_sample_rate: int,
|
||||
audio_duration_sec: float,
|
||||
) -> None:
|
||||
"""
|
||||
Test that vLLM's processor generates expected outputs with audio_sample_rate.
|
||||
"""Test that vLLM's processor generates expected outputs with audio_sample_rate.
|
||||
|
||||
This validates that the processor correctly handles audio_sample_rate
|
||||
passed via hf_processor_mm_kwargs and generates audio tokens.
|
||||
@@ -70,8 +69,7 @@ def test_processor_with_audio_sample_rate(
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen3-Omni-30B-A3B-Instruct"])
|
||||
def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
"""
|
||||
Test that longer audio generates more tokens than shorter audio.
|
||||
"""Test that longer audio generates more tokens than shorter audio.
|
||||
|
||||
This validates that audio_sample_rate is being used correctly by checking
|
||||
that audio duration affects token count as expected.
|
||||
|
||||
@@ -31,7 +31,8 @@ def _build_video_mm_data(
|
||||
|
||||
``total_num_frames`` is set equal to the ndarray frame count so
|
||||
that HF's ``sample_frames`` indices stay within bounds of the
|
||||
actual tensor that is passed."""
|
||||
actual tensor that is passed.
|
||||
"""
|
||||
video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": original_fps,
|
||||
@@ -237,7 +238,8 @@ def _build_video_embeds_mm_data(
|
||||
grid_thw: tuple[int, int, int] = (2, 4, 4),
|
||||
) -> dict[str, Any]:
|
||||
"""Create an embeds-only video item as an EPD consumer receives it:
|
||||
pre-computed embeddings plus the metadata published by the encoder."""
|
||||
pre-computed embeddings plus the metadata published by the encoder.
|
||||
"""
|
||||
import torch
|
||||
|
||||
t, h, w = grid_thw
|
||||
@@ -256,7 +258,8 @@ def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
"""Embeds-only video input must size the placeholder range from the
|
||||
grid and the real timestamps published by the encoder (EC consumer
|
||||
path); synthesized or missing timestamps would change the token count
|
||||
and break embedding merging downstream."""
|
||||
and break embedding merging downstream.
|
||||
"""
|
||||
# `build_model_context` forces enable_mm_embeds off, so build the
|
||||
# config directly; keep the same online-availability skip behavior.
|
||||
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
|
||||
@@ -301,7 +304,8 @@ def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
|
||||
"""Timestamps are required metadata for video embeds: they size the
|
||||
placeholder range, so omitting them must fail loudly at parse time
|
||||
instead of silently producing a wrong prompt."""
|
||||
instead of silently producing a wrong prompt.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
|
||||
@@ -116,7 +116,8 @@ def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
and multi-modal item by the Transformers modelling backend.
|
||||
"""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
|
||||
@@ -83,7 +83,8 @@ def _process_two_images(processor_cls, separator: str):
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_multiple_inputs(processor_cls):
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
and multi-modal item by the Transformers modelling backend.
|
||||
"""
|
||||
result = _process_two_images(processor_cls, separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
@@ -102,7 +103,8 @@ def test_image_adjacent_inputs(processor_cls):
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||
item's data dependent on what it was processed with and so uncacheable."""
|
||||
item's data dependent on what it was processed with and so uncacheable.
|
||||
"""
|
||||
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||
image_token = mm_processor.info.get_hf_processor().image_token
|
||||
|
||||
@@ -144,7 +146,8 @@ def _process_one_gemma3_image(processor_cls):
|
||||
@offsets_only
|
||||
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||
inside the placeholder range but masked out of it."""
|
||||
inside the placeholder range but masked out of it.
|
||||
"""
|
||||
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
@@ -156,7 +159,8 @@ def test_legacy_placeholders_hold_only_image_tokens():
|
||||
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||
marker in the unexpanded prompt, not the token the expansion repeats.
|
||||
"""
|
||||
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
@@ -172,7 +176,8 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||
tokenize it before splicing in the expansion."""
|
||||
tokenize it before splicing in the expansion.
|
||||
"""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
)
|
||||
@@ -192,7 +197,8 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
@offsets_only
|
||||
def test_missing_replacement_offsets_names_the_processor():
|
||||
"""A processor that reports no replacement offsets cannot be served, which must
|
||||
be said plainly rather than surfacing later as a field config mismatch."""
|
||||
be said plainly rather than surfacing later as a field config mismatch.
|
||||
"""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||
@@ -276,7 +282,8 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||
processor produces itself, which is what the legacy path returns."""
|
||||
processor produces itself, which is what the legacy path returns.
|
||||
"""
|
||||
prompt_ids = []
|
||||
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
@@ -298,7 +305,8 @@ def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
def test_nested_image_fields_split_per_image(processor_cls):
|
||||
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||
belonging to each image one dimension further in. Slicing the batch dimension
|
||||
instead handed the first image every row and the second an empty tensor."""
|
||||
instead handed the first image every row and the second an empty tensor.
|
||||
"""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
)
|
||||
|
||||
@@ -29,7 +29,8 @@ PROCESSOR_CLASSES = [
|
||||
|
||||
def create_processor(model_id: str, processor_cls):
|
||||
"""Build a processor directly, because the registry only ever builds the one the
|
||||
installed transformers version selects, leaving the other path untested."""
|
||||
installed transformers version selects, leaving the other path untested.
|
||||
"""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
@@ -40,7 +41,8 @@ def create_processor(model_id: str, processor_cls):
|
||||
|
||||
def create_cached_processor(model_id: str, processor_cls):
|
||||
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||
cache back so a test can check it was actually used."""
|
||||
cache back so a test can check it was actually used.
|
||||
"""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
ctx = InputProcessingContext(
|
||||
|
||||
@@ -18,7 +18,8 @@ from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGenerat
|
||||
def test_get_num_audio_tokens_streaming_estimate(audio_duration_s, expected_tokens):
|
||||
"""The duration-based estimate must convert ``window_stride`` (seconds) to a
|
||||
sample hop and divide by the encoder subsampling factor. Values are pinned to
|
||||
concrete numbers so a rounding regression is actually caught."""
|
||||
concrete numbers so a rounding regression is actually caught.
|
||||
"""
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
preprocessor={"window_stride": 0.01, "sample_rate": 16000},
|
||||
|
||||
@@ -78,7 +78,8 @@ def test_cosmos3_modelopt_quantizer_weights_mapper():
|
||||
"""ModelOpt/Diffusers FP8 checkpoints ship native fake-quant buffers
|
||||
(``*_quantizer._amax`` / ``._scale``) alongside the vLLM-consumable
|
||||
``weight_scale`` / ``input_scale`` sidecars. vLLM must drop the former
|
||||
(it has no parameter for them) while keeping the latter."""
|
||||
(it has no parameter for them) while keeping the latter.
|
||||
"""
|
||||
from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration
|
||||
|
||||
mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper
|
||||
@@ -215,7 +216,7 @@ def test_bailing_vl_mapper_handles_module_and_parameter_names():
|
||||
|
||||
|
||||
def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Create weights from safetensors checkpoint metadata"""
|
||||
"""Create weights from safetensors checkpoint metadata."""
|
||||
metadata = try_get_safetensors_metadata(repo)
|
||||
weight_names = list(metadata.weight_map.keys())
|
||||
with torch.device("meta"):
|
||||
@@ -223,9 +224,7 @@ def create_repo_dummy_weights(repo: str) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
|
||||
|
||||
def create_dummy_base_model(repo: str, model_arch: str) -> PreTrainedModel:
|
||||
"""
|
||||
Create weights from a dummy meta deserialized hf base model with name conversion
|
||||
"""
|
||||
"""Create weights from a dummy meta-deserialized HF base model, with renaming."""
|
||||
config = AutoConfig.from_pretrained(repo)
|
||||
with torch.device("meta"):
|
||||
model = AutoModel.from_config(config)
|
||||
@@ -233,9 +232,7 @@ def create_dummy_base_model(repo: str, model_arch: str) -> PreTrainedModel:
|
||||
|
||||
|
||||
def create_dummy_model(repo: str, model_arch: str) -> PreTrainedModel:
|
||||
"""
|
||||
Create weights from a dummy meta deserialized hf model with name conversion
|
||||
"""
|
||||
"""Create weights from a dummy meta deserialized hf model with name conversion."""
|
||||
model_cls: PreTrainedModel = getattr(transformers, model_arch)
|
||||
config = AutoConfig.from_pretrained(repo)
|
||||
with torch.device("meta"):
|
||||
|
||||
@@ -81,7 +81,8 @@ class _FakeTensor:
|
||||
"""Sentinel stand-in for torch.Tensor in load_weights tests. Supports the
|
||||
.detach().clone() chain used by load_weights for buffered mm weights;
|
||||
both methods return self so identity (and the existing equality
|
||||
assertions) are preserved through cloning."""
|
||||
assertions) are preserved through cloning.
|
||||
"""
|
||||
|
||||
def detach(self):
|
||||
return self
|
||||
@@ -165,7 +166,8 @@ def _make_mm_items_with_video_bytes(
|
||||
|
||||
def test_extract_audio_from_videos_passes_max_duration():
|
||||
"""_extract_audio_from_videos must forward VLLM_MAX_AUDIO_DECODE_DURATION_S
|
||||
to load_audio_pyav so decompression-bomb audio is rejected."""
|
||||
to load_audio_pyav so decompression-bomb audio is rejected.
|
||||
"""
|
||||
dummy_audio = (np.zeros(16000, dtype=np.float32), 16000.0)
|
||||
mm_items = _make_mm_items_with_video_bytes(b"\x00" * 64)
|
||||
|
||||
@@ -185,7 +187,8 @@ def test_extract_audio_from_videos_passes_max_duration():
|
||||
|
||||
def test_extract_audio_from_videos_rejects_oversized_audio():
|
||||
"""When load_audio_pyav raises due to duration limit the video is
|
||||
marked as having no audio instead of crashing the server."""
|
||||
marked as having no audio instead of crashing the server.
|
||||
"""
|
||||
mm_items = _make_mm_items_with_video_bytes(b"\x00" * 64)
|
||||
|
||||
processor = object.__new__(NanoNemotronVLMultiModalProcessor)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
End-to-end accuracy test for GPT-OSS model quantization.
|
||||
"""End-to-end accuracy test for GPT-OSS model quantization.
|
||||
|
||||
Config:
|
||||
Task: gsm8k_platinum
|
||||
@@ -170,8 +169,7 @@ def test_aiter_mxfp4_moe_ignores_padded_rows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dist_init,
|
||||
) -> None:
|
||||
"""
|
||||
Garbage in cudagraph padding rows must not reach the unpadded outputs.
|
||||
"""Garbage in cudagraph padding rows must not reach the unpadded outputs.
|
||||
|
||||
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
|
||||
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
|
||||
|
||||
@@ -144,7 +144,8 @@ def test_qwen4_exp_mtp_override_sets_draft_config(
|
||||
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
|
||||
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
|
||||
rest of the architecture is PP-capable, so the refusal must be conditional
|
||||
-- and must land before the engine spends time loading weights."""
|
||||
-- and must land before the engine spends time loading weights.
|
||||
"""
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
|
||||
@@ -127,8 +127,7 @@ class _HfExamplesInfo:
|
||||
check_min_version: bool = True,
|
||||
check_max_version: bool = True,
|
||||
) -> str | None:
|
||||
"""
|
||||
If the installed transformers version does not meet the requirements,
|
||||
"""If the installed transformers version does not meet the requirements,
|
||||
perform the given action.
|
||||
"""
|
||||
if (
|
||||
@@ -185,9 +184,7 @@ class _HfExamplesInfo:
|
||||
*,
|
||||
on_fail: Literal["error", "skip"],
|
||||
) -> None:
|
||||
"""
|
||||
If the model is not available online, perform the given action.
|
||||
"""
|
||||
"""If the model is not available online, perform the given action."""
|
||||
if not self.is_available_online:
|
||||
msg = "Model is not available online"
|
||||
|
||||
|
||||
@@ -103,7 +103,8 @@ def test_fi_backend_strings_are_registered_mega_moe_backends():
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
|
||||
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
|
||||
instead of silently falling through to the generic FusedMoE path."""
|
||||
instead of silently falling through to the generic FusedMoE path.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
|
||||
|
||||
@@ -118,8 +119,9 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
|
||||
)
|
||||
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
|
||||
"""vLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
|
||||
models validate their own constraints at construction time."""
|
||||
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
|
||||
models validate their own constraints at construction time.
|
||||
"""
|
||||
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
|
||||
|
||||
|
||||
@@ -132,7 +134,8 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
"""Every mega backend must qualify for sequence-parallel MoE at
|
||||
TP>1/EP: the predicate once matched only the native backend string,
|
||||
which silently ran the fi backends full-batch with an all-reduce on
|
||||
every rank — 0.42-0.65x native e2e at TP8."""
|
||||
every rank — 0.42-0.65x native e2e at TP8.
|
||||
"""
|
||||
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
@@ -176,7 +179,8 @@ def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
|
||||
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
|
||||
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
|
||||
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
|
||||
device it already bound via BootstrapConfig.device."""
|
||||
device it already bound via BootstrapConfig.device.
|
||||
"""
|
||||
import vllm.utils.flashinfer_moe_ep as mod
|
||||
|
||||
pg = object()
|
||||
@@ -241,7 +245,8 @@ def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
|
||||
|
||||
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
|
||||
"""One 32-element scale group per row: low nibble is the even element,
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group."""
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group.
|
||||
"""
|
||||
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
|
||||
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
|
||||
|
||||
|
||||
@@ -783,7 +783,8 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
|
||||
def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
|
||||
"""The loader invokes the model-level PWAL hook for every load format,
|
||||
so it must finalize megamoe + mhc broadcast weights to cover dummy
|
||||
load, which skips load_weights()."""
|
||||
load, which skips load_weights().
|
||||
"""
|
||||
calls = []
|
||||
stub = SimpleNamespace(
|
||||
model=SimpleNamespace(
|
||||
@@ -800,7 +801,8 @@ def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
|
||||
def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
|
||||
"""MTP/DSpark drafters load as their own top-level models, so each needs
|
||||
its own PWAL hook now that the megamoe forward no longer finalizes
|
||||
weights lazily on first use."""
|
||||
weights lazily on first use.
|
||||
"""
|
||||
calls = []
|
||||
mtp = SimpleNamespace(finalize_mega_moe_weights=lambda: calls.append("mtp"))
|
||||
DeepSeekV4MTP.process_weights_after_loading(mtp)
|
||||
|
||||
@@ -215,7 +215,8 @@ def test_v41_dspark_loads_linear_scales(
|
||||
):
|
||||
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
|
||||
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.nvidia import dspark
|
||||
|
||||
mxfp8 = scale_dtype != torch.float32
|
||||
|
||||
@@ -64,7 +64,6 @@ def can_initialize(
|
||||
The spawn process causes the _initialize_kv_caches_v1 function below to
|
||||
become ineffective.
|
||||
"""
|
||||
|
||||
model_info = EXAMPLE_MODELS.get_hf_info(model_arch)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
@@ -199,13 +198,13 @@ def can_initialize(
|
||||
|
||||
@pytest.mark.parametrize("model_arch", MINIMAL_MODEL_ARCH_LIST)
|
||||
def test_can_initialize_small_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test initializing small subset of supported models"""
|
||||
"""Test initializing small subset of supported models."""
|
||||
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_arch", OTHER_MODEL_ARCH_LIST)
|
||||
def test_can_initialize_large_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test initializing large subset of supported models
|
||||
"""Test initializing large subset of supported models.
|
||||
|
||||
This test covers the complement of the tests covered in the "small subset"
|
||||
test.
|
||||
|
||||
@@ -73,7 +73,8 @@ def test_mtp_override_extracts_n_predict_from_multimodal_wrapper(
|
||||
model_type: str, expected_arch: str
|
||||
) -> None:
|
||||
"""Verify that multimodal wrapper checkpoints with mtp_num_hidden_layers
|
||||
in text_config resolve n_predict and architecture correctly."""
|
||||
in text_config resolve n_predict and architecture correctly.
|
||||
"""
|
||||
cfg = SpeculativeConfig.hf_config_override(
|
||||
_multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
|
||||
)
|
||||
@@ -93,7 +94,8 @@ def test_mtp_override_top_level_precedence_over_nested_text_config(
|
||||
model_type: str, expected_arch: str
|
||||
) -> None:
|
||||
"""Verify that an explicit top-level mtp_num_hidden_layers takes precedence
|
||||
over a nested text_config value."""
|
||||
over a nested text_config value.
|
||||
"""
|
||||
cfg = _multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
|
||||
cfg.mtp_num_hidden_layers = 3
|
||||
overridden = SpeculativeConfig.hf_config_override(cfg)
|
||||
@@ -113,7 +115,8 @@ def test_mtp_override_downloads_real_hf_hub_configs(
|
||||
model_id: str, expected_arch: str
|
||||
) -> None:
|
||||
"""Verify that unmodified real-world checkpoints downloaded via
|
||||
AutoConfig.from_pretrained resolve n_predict=1 from text_config."""
|
||||
AutoConfig.from_pretrained resolve n_predict=1 from text_config.
|
||||
"""
|
||||
hf_config = AutoConfig.from_pretrained(model_id)
|
||||
cfg = SpeculativeConfig.hf_config_override(hf_config)
|
||||
assert cfg.model_type == "qwen3_5_mtp"
|
||||
|
||||
@@ -197,7 +197,8 @@ def test_merge_multimodal_embeddings_no_sync():
|
||||
def test_get_rename_mapper_keeps_only_renames():
|
||||
"""`None` means "do not load", which is meaningless to the consumers of
|
||||
this mapper (LoRA name parsing, quantization config layer lists), and
|
||||
applying it would silently shrink their lists."""
|
||||
applying it would silently shrink their lists.
|
||||
"""
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_regex={re.compile(r"^drop_regex\."): None},
|
||||
orig_to_new_substr={"drop_substr": None, "keep_substr": "kept"},
|
||||
|
||||
@@ -41,9 +41,7 @@ pytestmark = pytest.mark.cpu_test
|
||||
def test_resolve_visual_encoder_outputs(
|
||||
select_layers, num_layers_loaded, max_possible_layers, expected_features
|
||||
):
|
||||
"""
|
||||
Test that offsets are correctly handled for vision feature layers.
|
||||
"""
|
||||
"""Test that offsets are correctly handled for vision feature layers."""
|
||||
encoder_outputs = [torch.tensor([idx]) for idx in range(num_layers_loaded + 1)]
|
||||
output_tensor = resolve_visual_encoder_outputs(
|
||||
encoder_outputs=encoder_outputs,
|
||||
@@ -94,11 +92,9 @@ def test_run_dp_sharded_vision_model(batch_size: int):
|
||||
def run_dp_sharded_vision_model_vs_direct(
|
||||
local_rank: int, world_size: int, batch_size: int, master_port: int
|
||||
):
|
||||
"""
|
||||
Test that run_dp_sharded_vision_model produces the same results as
|
||||
"""Test that run_dp_sharded_vision_model produces the same results as
|
||||
calling the model directly.
|
||||
"""
|
||||
|
||||
# Set random seed for reproducibility
|
||||
set_random_seed(0)
|
||||
|
||||
@@ -282,8 +278,7 @@ def test_run_dp_sharded_mrope_vision_model(batch_size: int):
|
||||
def run_dp_sharded_mrope_vision_model_vs_direct(
|
||||
local_rank: int, world_size: int, batch_size: int, master_port: int
|
||||
):
|
||||
"""
|
||||
Test that run_dp_sharded_mrope_vision_model produces the same results as
|
||||
"""Test that run_dp_sharded_mrope_vision_model produces the same results as
|
||||
calling the model directly.
|
||||
"""
|
||||
# Set random seed for reproducibility
|
||||
@@ -517,7 +512,8 @@ def _reference_input_norm(
|
||||
def test_fused_input_norm_matches_reference(num_patches: int):
|
||||
"""FusedInputNorm must equal the plain affine, including for num_patches
|
||||
above the cuDNN batch-norm grid limit (~65535) that previously raised
|
||||
CUDNN_STATUS_INTERNAL_ERROR (issue #51717)."""
|
||||
CUDNN_STATUS_INTERNAL_ERROR (issue #51717).
|
||||
"""
|
||||
channel = 3
|
||||
patch_size = 14 * 14
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
|
||||
@@ -251,7 +251,8 @@ class UnderscoreInplaceParallelLinears(FourParallelLinears):
|
||||
|
||||
class PlainQKV(nn.Module):
|
||||
"""q/k/v returned as a bare tuple: no reshape stands between the calls and
|
||||
the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it."""
|
||||
the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden: int = 32, heads: int = 4, head_dim: int = 8):
|
||||
super().__init__()
|
||||
@@ -338,7 +339,8 @@ class GuardedVAttention(FakeAttention):
|
||||
"""Gemma 4-style: `v_proj` is called once but also read in a `None` guard.
|
||||
|
||||
The guard is a fusion invariant (the fuser only binds instances where the
|
||||
projection exists), so it folds to `True` and the call still fuses."""
|
||||
projection exists), so it folds to `True` and the call still fuses.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -368,7 +370,8 @@ class TruthyGuardAttention(FakeAttention):
|
||||
|
||||
An `nn.Linear` is always truthy, so the guard folds to `True` and the call
|
||||
still fuses (the projection is a proven invariant, exactly as for a `None`
|
||||
guard)."""
|
||||
guard).
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -399,7 +402,8 @@ class StatementGuardVAttention(FakeAttention):
|
||||
The `if`'s test is not an `ast.Compare`, so the identity-check scan skips it;
|
||||
`_in_boolean_context` folds the test to `True` (an `nn.Linear` is always
|
||||
truthy) and the call inside the now-`if True:` body still fuses, with the
|
||||
live `else` surviving the rewrite."""
|
||||
live `else` surviving the rewrite.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -428,7 +432,8 @@ class BranchedKVAttention(FakeAttention):
|
||||
|
||||
The three calls do not share a block, so the fused GEMM must be hoisted to
|
||||
the innermost block that dominates all of them (the function body). The
|
||||
live `if`/`else` must survive the rewrite."""
|
||||
live `if`/`else` must survive the rewrite.
|
||||
"""
|
||||
|
||||
is_kv_shared_layer = False
|
||||
|
||||
@@ -459,7 +464,8 @@ class RebindArgAttention(FakeAttention):
|
||||
|
||||
The branch is dead at trace time so the match succeeds, but the source
|
||||
rewrite must refuse: hoisting the GEMM above the rebind would feed k/v a
|
||||
different input. Result: no fusion."""
|
||||
different input. Result: no fusion.
|
||||
"""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -490,7 +496,8 @@ class InPlaceMutatedArgAttention(FakeAttention):
|
||||
Like `RebindArgAttention`, but the mutation keeps the name in a `Load`
|
||||
context (`hidden_states.mul_(2)`), so a bare `Name`-store check would miss it.
|
||||
The rewrite must still refuse: hoisting the GEMM above the mutation changes
|
||||
the value k/v see. Result: no fusion."""
|
||||
the value k/v see. Result: no fusion.
|
||||
"""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -520,7 +527,8 @@ class AliasMutatedArgAttention(FakeAttention):
|
||||
|
||||
`hidden_states.view(-1)` shares storage with its base, so `alias.mul_(2)`
|
||||
changes what k/v read without ever naming `hidden_states` as a target. Only
|
||||
tracking names that may alias the input catches it. Result: no fusion."""
|
||||
tracking names that may alias the input catches it. Result: no fusion.
|
||||
"""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -551,7 +559,8 @@ class FunctionalInplaceArgAttention(FakeAttention):
|
||||
|
||||
`F.relu(hidden_states, inplace=True)` writes through an argument, so the
|
||||
name appears only in a `Load` context inside a call that is not a method on
|
||||
it -- invisible to a trailing-underscore method check. Result: no fusion."""
|
||||
it -- invisible to a trailing-underscore method check. Result: no fusion.
|
||||
"""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -581,7 +590,8 @@ class SubscriptMutatedArgAttention(FakeAttention):
|
||||
|
||||
`hidden_states[..., 0] = 0` leaves the name in a `Load` context on the
|
||||
subscript's value, so it too evades a bare `Name`-store check. The rewrite
|
||||
must refuse for the same reason. Result: no fusion."""
|
||||
must refuse for the same reason. Result: no fusion.
|
||||
"""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -611,7 +621,8 @@ class NonFoldableRefAttention(FakeAttention):
|
||||
|
||||
Deleting `v_proj` would break `isinstance(self.v_proj, nn.Linear)`: it reads
|
||||
the projection's type, not just whether it exists, so the fuser cannot fold
|
||||
it and must refuse rather than rewrite it."""
|
||||
it and must refuse rather than rewrite it.
|
||||
"""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -836,7 +847,8 @@ def test_glu_identifies_down_projection():
|
||||
|
||||
It is forced to `RowParallelLinear` in `update_attrs` so its sharded input
|
||||
matches the column-parallel merged gate/up; `None` when there is no such
|
||||
projection to force (fusion of gate/up still applies)."""
|
||||
projection to force (fusion of gate/up still applies).
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
|
||||
assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
|
||||
@@ -997,7 +1009,8 @@ def test_bypass_existence_guard_folds_identity_none_check(expr, expected):
|
||||
The projection is a proven invariant (it exists as an `nn.Linear`), so an
|
||||
`is (not) None` guard has a constant truth value; folding it lets the call be
|
||||
rewritten away without changing the guard's outcome. `==`/`!=` are excluded
|
||||
(see the refusal test) because a subclass may override `__eq__`/`__ne__`."""
|
||||
(see the refusal test) because a subclass may override `__eq__`/`__ne__`.
|
||||
"""
|
||||
funcdef, ref = _guard_funcdef(expr)
|
||||
bypass_existence_guard(funcdef, ref, "v_proj")
|
||||
test = next(node for node in ast.walk(funcdef) if isinstance(node, ast.IfExp)).test
|
||||
@@ -1016,7 +1029,8 @@ def test_bypass_existence_guard_folds_bare_truthiness(expr):
|
||||
|
||||
An `nn.Linear` has no `__bool__`/`__len__`, so it is always truthy; in a pure
|
||||
boolean position (a test, or a `not` operand) the reference can be replaced by
|
||||
`True`, letting the projection be deleted."""
|
||||
`True`, letting the projection be deleted.
|
||||
"""
|
||||
funcdef, ref = _guard_funcdef(expr)
|
||||
bypass_existence_guard(funcdef, ref, "v_proj")
|
||||
# No `self.v_proj` reference survives; a literal `True` took its place.
|
||||
@@ -1031,7 +1045,8 @@ def test_bypass_existence_guard_folds_bare_if_statement():
|
||||
|
||||
The `len(node.ops) == 1` `Compare` scan skips it (it is not a comparison at
|
||||
all); `_in_boolean_context` then matches the `if`'s test and folds it, so the
|
||||
bare-truthiness statement form is handled, not silently dropped."""
|
||||
bare-truthiness statement form is handled, not silently dropped.
|
||||
"""
|
||||
funcdef = ast.parse(
|
||||
"def f(self, a, b):\n if self.v_proj:\n return a\n return b"
|
||||
).body[0]
|
||||
@@ -1063,7 +1078,8 @@ def test_bypass_existence_guard_refuses_non_guard_reference(expr):
|
||||
Rewriting it is unsafe (its value depends on more than the projection's
|
||||
existence), so the fuser refuses rather than change semantics. `==`/`!=` are
|
||||
refused too: `is_linear` accepts `nn.Linear` subclasses that could override
|
||||
`__eq__`/`__ne__`, so equality need not agree with `is (not) None`."""
|
||||
`__eq__`/`__ne__`, so equality need not agree with `is (not) None`.
|
||||
"""
|
||||
funcdef, ref = _guard_funcdef(expr)
|
||||
with pytest.raises(ValueError, match="outside an existence guard"):
|
||||
bypass_existence_guard(funcdef, ref, "v_proj")
|
||||
@@ -1103,7 +1119,8 @@ def test_merged_column_fuser_folds_existence_guard():
|
||||
|
||||
Guard folding lives on `StackedFuser`, not just `QKVFuser`, because every
|
||||
stacked fuser deletes the projections it merges -- a surviving reference to
|
||||
one would fail at runtime."""
|
||||
one would fail at runtime.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
module = GuardedParallelLinears()
|
||||
fuser = MergedColumnParallelFuser.match(trace(module), module)
|
||||
@@ -1129,7 +1146,8 @@ def test_merged_column_fuser_rejects_input_mutation(cls):
|
||||
The mutation may be direct (`x.add_(1)`), through a view that shares the
|
||||
input's storage, or through a call that writes an argument in place
|
||||
(`F.relu(x, inplace=True)`, `torch.relu_(x)`). Each one silently changes the
|
||||
fused result, so the rewrite must refuse rather than fuse."""
|
||||
fused result, so the rewrite must refuse rather than fuse.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
module = cls()
|
||||
fuser = MergedColumnParallelFuser.match(trace(module), module)
|
||||
@@ -1187,7 +1205,8 @@ def test_merged_column_fuser_supports_any_number_of_linears(
|
||||
|
||||
def test_merged_name_is_unique_per_linear_names():
|
||||
"""The computed name must not collide across different projections, so
|
||||
stacking two unrelated fusions can never clobber packed_modules_mapping."""
|
||||
stacking two unrelated fusions can never clobber packed_modules_mapping.
|
||||
"""
|
||||
ab_c = MergedColumnParallelFuser(source_cls="M", linear_names=("a_b", "c"))
|
||||
a_bc = MergedColumnParallelFuser(source_cls="M", linear_names=("a", "b_c"))
|
||||
assert ab_c.merged_name != a_bc.merged_name
|
||||
@@ -1203,7 +1222,8 @@ def test_heterogeneous_instances_fall_back_to_merged_column_fuser(
|
||||
layer whose out_features aren't a multiple of its own head_dim (as a
|
||||
misconfigured or heterogeneous checkpoint might have) fails QKVFuser's
|
||||
validation and must still fuse via the generic fallback, while a
|
||||
well-formed sibling of the same class keeps using QKVFuser."""
|
||||
well-formed sibling of the same class keeps using QKVFuser.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
container = nn.Module()
|
||||
container.compatible = PlainQKV(head_dim=8)
|
||||
@@ -1224,7 +1244,8 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
|
||||
"""A single projection split into q/k/v must be re-sharded, not merged.
|
||||
|
||||
Only the split sizes change: `QKVParallelLinear` loads the packed
|
||||
checkpoint weight as-is, and shards q by heads while replicating k/v."""
|
||||
checkpoint weight as-is, and shards q by heads while replicating k/v.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
meta = PackedQKVAttention(kv_heads=kv_heads)
|
||||
fuser = get_fuser(meta, PackedQKVFuser)
|
||||
@@ -1255,7 +1276,8 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
|
||||
|
||||
def test_per_head_split_is_not_packed_qkv():
|
||||
"""The split must consume the whole projection, else its sizes are head
|
||||
widths and re-sharding by them would be wrong."""
|
||||
widths and re-sharding by them would be wrong.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
|
||||
|
||||
@@ -1375,7 +1397,8 @@ def test_act_and_mul_derived_from_module(default_vllm_config):
|
||||
|
||||
def _wider_model_config(head_dim: int) -> SimpleNamespace:
|
||||
"""A model whose global head size is twice `head_dim`, as a wider layer
|
||||
elsewhere in a heterogeneous checkpoint would make it."""
|
||||
elsewhere in a heterogeneous checkpoint would make it.
|
||||
"""
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_head_size=lambda: 2 * head_dim),
|
||||
quant_config=None,
|
||||
|
||||
@@ -45,7 +45,8 @@ def _attention(q_lora_rank: int | None):
|
||||
|
||||
def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen).
|
||||
"""
|
||||
attn = _attention(q_lora_rank)
|
||||
return MLAFuser.match(trace(attn), attn)
|
||||
|
||||
@@ -81,7 +82,8 @@ def test_discovers_modules_with_q_lora():
|
||||
|
||||
def test_q_lora_stacks_qkv_a_proj():
|
||||
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
|
||||
down-projection, so both checkpoint weights must remap into it."""
|
||||
down-projection, so both checkpoint weights must remap into it.
|
||||
"""
|
||||
fuser = _match(q_lora_rank=64)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
prefix = "model.layers.0.self_attn"
|
||||
@@ -102,7 +104,8 @@ def test_update_forward_rewrites_real_attention(q_lora_rank):
|
||||
The q-LoRA path hoists the merged down-projection to a top-level statement
|
||||
while its calls sit inside the `q_lora_rank` branches, so the region it
|
||||
spans is wide; a check that over-approximates aliasing would silently refuse
|
||||
here and cost the fusion rather than fail loudly."""
|
||||
here and cost the fusion rather than fail loudly.
|
||||
"""
|
||||
fuser = _match(q_lora_rank)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
fuser.update_forward(_attention(q_lora_rank))
|
||||
@@ -126,7 +129,8 @@ class _Norm(nn.Module):
|
||||
|
||||
class RenamedMLA(nn.Module):
|
||||
"""An MLA-shaped attention whose children have non-standard names, proving
|
||||
discovery is by structure and not attribute name."""
|
||||
discovery is by structure and not attribute name.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -153,7 +157,8 @@ class RenamedMLA(nn.Module):
|
||||
|
||||
def test_discovers_modules_under_arbitrary_names():
|
||||
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
|
||||
names, and `match` still locates each projection by dataflow."""
|
||||
names, and `match` still locates each projection by dataflow.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
module = RenamedMLA()
|
||||
fuser = MLAFuser.match(trace(module), module)
|
||||
|
||||
@@ -277,7 +277,8 @@ def test_moe_fuser_matches_scaled_router():
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
scorer is anchored on the routing (last) top-k, not the group one.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(GroupedRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
@@ -287,7 +288,8 @@ def test_moe_fuser_matches_grouped_router():
|
||||
|
||||
def test_moe_fuser_matches_correction_router():
|
||||
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(CorrectionRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
@@ -298,7 +300,8 @@ def test_moe_fuser_matches_correction_router():
|
||||
def test_moe_fuser_reads_router_dtype_from_the_gate():
|
||||
"""A router that computes its logits in fp32 must keep routing in fp32 when
|
||||
rebuilt, even though no config field names the dtype. The cast back to the
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype."""
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
|
||||
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
|
||||
@@ -317,7 +320,8 @@ def test_moe_fuser_detects_shared_experts():
|
||||
|
||||
def test_moe_fuser_skips_shared_detection_without_extra_children():
|
||||
"""With only experts and gate, shared-expert detection (and its block trace)
|
||||
is skipped, so a gate-derived add is not misread as a shared expert."""
|
||||
is skipped, so a gate-derived add is not misread as a shared expert.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNoShared()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
@@ -362,7 +366,8 @@ def test_moe_fuser_declines_unsupported(block_cls):
|
||||
|
||||
def test_moe_fuser_ignores_nested_returns():
|
||||
"""A tuple `return` inside a nested helper must not decline a block whose own
|
||||
forward returns a single tensor."""
|
||||
forward returns a single tensor.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNestedTupleReturn()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
@@ -370,7 +375,8 @@ def test_moe_fuser_ignores_nested_returns():
|
||||
|
||||
def test_moe_fuser_router_requires_connected_dataflow():
|
||||
"""A gate with linear + softmax + top-k present but not wired as a router
|
||||
(top-k selects over the input, not the scored logits) is not detected."""
|
||||
(top-k selects over the input, not the scored logits) is not detected.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(DisconnectedRouter)
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
@@ -83,7 +83,8 @@ class ToleranceRMSNorm(NamedEpsRMSNorm):
|
||||
|
||||
class LiteralEpsRMSNorm(RMSNorm):
|
||||
"""eps is a literal in the source, and an unrelated attribute happens to
|
||||
hold the same value: matching on the value alone would bind to it."""
|
||||
hold the same value: matching on the value alone would bind to it.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__(hidden, eps)
|
||||
@@ -95,7 +96,8 @@ class LiteralEpsRMSNorm(RMSNorm):
|
||||
|
||||
class AmbiguousEpsRMSNorm(nn.Module):
|
||||
"""Two attributes hold the eps value, and the forward reads the second, so
|
||||
their order cannot pick it."""
|
||||
their order cannot pick it.
|
||||
"""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
@@ -231,7 +233,8 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config):
|
||||
|
||||
def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
|
||||
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
|
||||
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`.
|
||||
"""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
|
||||
|
||||
with torch.device("meta"):
|
||||
@@ -247,7 +250,8 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
|
||||
def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
"""Two instances of the same norm class with different eps must fuse to their
|
||||
own eps: the type-cached fuser holds only structure, not this value."""
|
||||
own eps: the type-cached fuser holds only structure, not this value.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = RMSNorm(16, eps=eps)
|
||||
@@ -259,7 +263,8 @@ def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
|
||||
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
"""The eps attribute is identified by holding the traced value, so a norm
|
||||
stays per-instance correct whatever it names it."""
|
||||
stays per-instance correct whatever it names it.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = cls(16, eps=eps)
|
||||
@@ -271,7 +276,8 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
|
||||
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
|
||||
"""A literal eps is recognised as coming from no attribute, even when one
|
||||
holds the same value, and is taken from the traced source instead."""
|
||||
holds the same value, and is taken from the traced source instead.
|
||||
"""
|
||||
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
|
||||
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
|
||||
module = LiteralEpsRMSNorm()
|
||||
@@ -284,7 +290,8 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo
|
||||
|
||||
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
"""When several attributes hold the eps value, the one the forward actually
|
||||
reads is identified, and they are all left as they were found."""
|
||||
reads is identified, and they are all left as they were found.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
|
||||
before = dict(vars(module))
|
||||
@@ -301,7 +308,8 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
|
||||
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
|
||||
plumbing. A full-width input skips the gather and equals a plain norm."""
|
||||
plumbing. A full-width input skips the gather and equals a plain norm.
|
||||
"""
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
@@ -323,7 +331,8 @@ def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
|
||||
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
|
||||
"""A sharded input (narrower than the full-width weight) that does not tile
|
||||
the weight evenly across ranks is rejected before any collective."""
|
||||
the weight evenly across ranks is rejected before any collective.
|
||||
"""
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
|
||||
|
||||
@@ -28,7 +28,8 @@ def fake_hw_layernorm(monkeypatch):
|
||||
A `SimpleNamespace` stands in for the module: `importlib.import_module`
|
||||
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
|
||||
attributes are set at construction (no `ModuleType` attribute-set that mypy
|
||||
rejects, no constant `setattr` that ruff rejects)."""
|
||||
rejects, no constant `setattr` that ruff rejects).
|
||||
"""
|
||||
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, module)
|
||||
return module
|
||||
@@ -81,7 +82,8 @@ def test_act_and_mul_falls_back_for_unknown_activation(
|
||||
@pytest.fixture(scope="module")
|
||||
def tiny_llama_path(tmp_path_factory):
|
||||
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
|
||||
tokenizer) so vLLM can load it like any local checkpoint."""
|
||||
tokenizer) so vLLM can load it like any local checkpoint.
|
||||
"""
|
||||
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
|
||||
|
||||
@@ -29,8 +29,7 @@ def check_outputs_equal(
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
):
|
||||
"""
|
||||
Compare the two sequences generated by different models,
|
||||
"""Compare the two sequences generated by different models,
|
||||
which should be equal.
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
@@ -131,6 +130,7 @@ def check_logprobs_close(
|
||||
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
|
||||
mismatch between the two sequences
|
||||
always_check_logprobs: If true, check logprobs even when tokens match
|
||||
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
@@ -290,6 +290,7 @@ def build_model_context(
|
||||
|
||||
Returns:
|
||||
InputProcessingContext for the model being considered.
|
||||
|
||||
"""
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
@@ -447,8 +448,7 @@ def dummy_hf_overrides(
|
||||
exist_overrides: dict[str, Any] | None = None,
|
||||
use_original_num_layers: bool = False,
|
||||
) -> PretrainedConfig:
|
||||
"""
|
||||
Dummy HF overrides function used to create dummy model
|
||||
"""Dummy HF overrides function used to create dummy model
|
||||
with only minimum nums of layer.
|
||||
"""
|
||||
# Copy because this helper is called more than once
|
||||
|
||||
Reference in New Issue
Block a user