[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -42,8 +42,7 @@ def _assert_close_ulps(
|
||||
compare against the reference's local ulp spacing. ``pre_act`` (the
|
||||
reference pre-activation) adds derivative-propagated slack: near
|
||||
``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by
|
||||
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps.
|
||||
"""
|
||||
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps."""
|
||||
g, r = got.float(), ref.float()
|
||||
tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol)
|
||||
if pre_act is not None:
|
||||
|
||||
@@ -123,8 +123,7 @@ def _check_writes_only_its_own_shard(
|
||||
device: torch.device, tp_size: int, rank: int
|
||||
) -> None:
|
||||
"""Two ranks that swapped both weight rows and write offsets still sum to
|
||||
the right total, so inspect each slice before the final collective.
|
||||
"""
|
||||
the right total, so inspect each slice before the final collective."""
|
||||
transform = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_size)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
@@ -130,8 +130,7 @@ def test_disabled_reproduces_the_plain_residual_sum(
|
||||
|
||||
def test_taps_the_consumer_layer_when_one_follows(recorder, monkeypatch):
|
||||
"""The value the next layer reads is the mixture against *its* weights,
|
||||
so the tap has to reach forward rather than use the current layer's.
|
||||
"""
|
||||
so the tap has to reach forward rather than use the current layer's."""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
|
||||
_call(_stub_model(enabled=True), 1, torch.zeros(2), None, torch.zeros(2))
|
||||
@@ -182,8 +181,7 @@ def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
|
||||
):
|
||||
"""The kernel writes an applied delta back into the prefix in place, which
|
||||
would double-add it into the live residual stream, so the pending output
|
||||
has to arrive already summed into the prefix with `delta` left None.
|
||||
"""
|
||||
has to arrive already summed into the prefix with `delta` left None."""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
prefix_sum = torch.tensor([1.0, 2.0])
|
||||
pending = torch.tensor([0.5, 0.25])
|
||||
|
||||
@@ -727,8 +727,7 @@ def test_cudagraph_capture_metadata_avoids_device_to_host_copy(
|
||||
"""`build_for_cudagraph_capture` is not capture-only: a DP rank with nothing
|
||||
scheduled re-stages its FULL-graph metadata through it on every dummy step,
|
||||
so it must not synchronize the device. The host-side draft counts have to
|
||||
come from `query_start_loc_cpu` and match the device-derived values.
|
||||
"""
|
||||
come from `query_start_loc_cpu` and match the device-derived values."""
|
||||
num_speculative_tokens = 2
|
||||
batch = BatchSpec(seq_lens=[50, 30], query_lens=[3, 3])
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
@@ -60,8 +60,7 @@ def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
|
||||
"""KV-shared layers have q_proj instead of a packed qkv_proj; their
|
||||
redundant K/V tensors in original checkpoints have no parameter and are
|
||||
skipped rather than failing the load.
|
||||
"""
|
||||
skipped rather than failing the load."""
|
||||
model = torch.nn.Module()
|
||||
model.config = SimpleNamespace(num_experts=0)
|
||||
model.start_layer, model.end_layer = 0, 2
|
||||
@@ -93,8 +92,7 @@ def test_gemma4_kv_shared_layer_loads_plain_q_proj() -> None:
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma3n_kv_shared_layer_mapper() -> None:
|
||||
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
|
||||
q_proj and drop the redundant K/V tensors original checkpoints ship.
|
||||
"""
|
||||
q_proj and drop the redundant K/V tensors original checkpoints ship."""
|
||||
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
|
||||
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
|
||||
weights = [
|
||||
|
||||
@@ -274,8 +274,7 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
|
||||
|
||||
def test_mistral_function_call_nested_json():
|
||||
"""Ensure that the function-name regex captures the entire outermost
|
||||
JSON block, including nested braces.
|
||||
"""
|
||||
JSON block, including nested braces."""
|
||||
|
||||
# Create a minimal stub tokenizer that provides the few attributes the
|
||||
# parser accesses (`version` and `get_vocab`).
|
||||
|
||||
@@ -29,8 +29,7 @@ from vllm.v1.worker.gpu.states import RequestState
|
||||
@torch.inference_mode
|
||||
def test_splade_pooler_matches_reference_formula(B, T, H, V):
|
||||
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
|
||||
log1p(relu(logits)) -> max over sequence length (after masking).
|
||||
"""
|
||||
log1p(relu(logits)) -> max over sequence length (after masking)."""
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Prepare [B] sequences of shape [T, H]
|
||||
|
||||
@@ -33,8 +33,7 @@ NUM_TOKENS = 3
|
||||
|
||||
class _StubGate(torch.nn.Module):
|
||||
"""Router stand-in, so the test does not depend on platform-specific
|
||||
``ReplicatedLinear`` weight post-processing.
|
||||
"""
|
||||
``ReplicatedLinear`` weight post-processing."""
|
||||
|
||||
def __init__(self, hidden_size: int, num_experts: int):
|
||||
super().__init__()
|
||||
|
||||
@@ -81,8 +81,7 @@ def test_transformers_audio_generation(
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
|
||||
deadlock and allows worker processes to terminate cleanly, and release
|
||||
GPU memory between test runs until the issue is fixed.
|
||||
"""
|
||||
GPU memory between test runs until the issue is fixed."""
|
||||
# TODO: Remove monkeypatch once
|
||||
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
|
||||
disable_envs_cache()
|
||||
|
||||
@@ -294,8 +294,7 @@ def sample_frames_with_video_metadata(
|
||||
|
||||
def apply_image_size_scaling(image, size: float | tuple[int, int], size_type: SizeType):
|
||||
"""Applies a size scaler to one image; this can be an image size factor,
|
||||
which scales the image while maintaining the aspect ratio
|
||||
"""
|
||||
which scales the image while maintaining the aspect ratio"""
|
||||
# Special case for embeddings; if it's a tensor, it's only valid if we
|
||||
# are considering size factors at constant scale, i.e., we just clone
|
||||
# the tensor
|
||||
|
||||
@@ -108,8 +108,7 @@ class TestDeepseekOCREmptyImagesCrop:
|
||||
|
||||
def test_mismatched_image_size_raises(self, processor):
|
||||
"""Deliberately wrong image_size binding should still be caught
|
||||
by TensorSchema validation.
|
||||
"""
|
||||
by TensorSchema validation."""
|
||||
small_image = Image.new("RGB", (100, 100), color="green")
|
||||
|
||||
result = processor(
|
||||
@@ -143,8 +142,7 @@ class TestDeepseekOCRInputValidation:
|
||||
def test_mismatched_image_count_raises_value_error(self, processor):
|
||||
"""Two ``<image>`` tokens with one image must raise ValueError, not
|
||||
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
|
||||
drop the middle text segment).
|
||||
"""
|
||||
drop the middle text segment)."""
|
||||
image = Image.new("RGB", (100, 100), color="red")
|
||||
with pytest.raises(ValueError, match="does not match number of images"):
|
||||
processor(prompt="<image> and <image>", images=[image])
|
||||
|
||||
@@ -16,8 +16,7 @@ from ...utils import build_model_context
|
||||
class _StubModel:
|
||||
"""Carries only the attributes the token-count method reads from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.).
|
||||
"""
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
|
||||
patch_grid_length: int
|
||||
pooled_grid_length: int
|
||||
@@ -32,8 +31,7 @@ get_mm_lora_token_counts = (
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the LoRA token counts must preserve the
|
||||
frame-level scaling exactly.
|
||||
"""
|
||||
frame-level scaling exactly."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
|
||||
|
||||
@@ -70,8 +70,7 @@ def test_dummy_video_num_frames_override_honors_min_of_two(
|
||||
):
|
||||
"""A ``num_frames`` override below 2 must be ignored (the model needs at
|
||||
least 2 frames), matching the "cannot be less than 2, will be ignored"
|
||||
warning.
|
||||
"""
|
||||
warning."""
|
||||
builder = object.__new__(Molmo2DummyInputsBuilder)
|
||||
builder.info = SimpleNamespace(
|
||||
get_hf_processor=lambda: SimpleNamespace(
|
||||
|
||||
@@ -31,8 +31,7 @@ def _build_video_mm_data(
|
||||
|
||||
``total_num_frames`` is set equal to the ndarray frame count so
|
||||
that HF's ``sample_frames`` indices stay within bounds of the
|
||||
actual tensor that is passed.
|
||||
"""
|
||||
actual tensor that is passed."""
|
||||
video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": original_fps,
|
||||
@@ -238,8 +237,7 @@ def _build_video_embeds_mm_data(
|
||||
grid_thw: tuple[int, int, int] = (2, 4, 4),
|
||||
) -> dict[str, Any]:
|
||||
"""Create an embeds-only video item as an EPD consumer receives it:
|
||||
pre-computed embeddings plus the metadata published by the encoder.
|
||||
"""
|
||||
pre-computed embeddings plus the metadata published by the encoder."""
|
||||
import torch
|
||||
|
||||
t, h, w = grid_thw
|
||||
@@ -258,8 +256,7 @@ def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
"""Embeds-only video input must size the placeholder range from the
|
||||
grid and the real timestamps published by the encoder (EC consumer
|
||||
path); synthesized or missing timestamps would change the token count
|
||||
and break embedding merging downstream.
|
||||
"""
|
||||
and break embedding merging downstream."""
|
||||
# `build_model_context` forces enable_mm_embeds off, so build the
|
||||
# config directly; keep the same online-availability skip behavior.
|
||||
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
|
||||
@@ -304,8 +301,7 @@ def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
|
||||
"""Timestamps are required metadata for video embeds: they size the
|
||||
placeholder range, so omitting them must fail loudly at parse time
|
||||
instead of silently producing a wrong prompt.
|
||||
"""
|
||||
instead of silently producing a wrong prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
|
||||
@@ -116,8 +116,7 @@ def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend.
|
||||
"""
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
|
||||
|
||||
@@ -83,8 +83,7 @@ def _process_two_images(processor_cls, separator: str):
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_multiple_inputs(processor_cls):
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend.
|
||||
"""
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(processor_cls, separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
@@ -103,8 +102,7 @@ def test_image_adjacent_inputs(processor_cls):
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||
item's data dependent on what it was processed with and so uncacheable.
|
||||
"""
|
||||
item's data dependent on what it was processed with and so uncacheable."""
|
||||
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||
image_token = mm_processor.info.get_hf_processor().image_token
|
||||
|
||||
@@ -146,8 +144,7 @@ def _process_one_gemma3_image(processor_cls):
|
||||
@offsets_only
|
||||
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||
inside the placeholder range but masked out of it.
|
||||
"""
|
||||
inside the placeholder range but masked out of it."""
|
||||
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
@@ -159,8 +156,7 @@ def test_legacy_placeholders_hold_only_image_tokens():
|
||||
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||
marker in the unexpanded prompt, not the token the expansion repeats.
|
||||
"""
|
||||
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
@@ -176,8 +172,7 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||
tokenize it before splicing in the expansion.
|
||||
"""
|
||||
tokenize it before splicing in the expansion."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
)
|
||||
@@ -197,8 +192,7 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
@offsets_only
|
||||
def test_missing_replacement_offsets_names_the_processor():
|
||||
"""A processor that reports no replacement offsets cannot be served, which must
|
||||
be said plainly rather than surfacing later as a field config mismatch.
|
||||
"""
|
||||
be said plainly rather than surfacing later as a field config mismatch."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||
@@ -282,8 +276,7 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||
processor produces itself, which is what the legacy path returns.
|
||||
"""
|
||||
processor produces itself, which is what the legacy path returns."""
|
||||
prompt_ids = []
|
||||
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
@@ -305,8 +298,7 @@ def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
def test_nested_image_fields_split_per_image(processor_cls):
|
||||
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||
belonging to each image one dimension further in. Slicing the batch dimension
|
||||
instead handed the first image every row and the second an empty tensor.
|
||||
"""
|
||||
instead handed the first image every row and the second an empty tensor."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
)
|
||||
|
||||
@@ -29,8 +29,7 @@ PROCESSOR_CLASSES = [
|
||||
|
||||
def create_processor(model_id: str, processor_cls):
|
||||
"""Build a processor directly, because the registry only ever builds the one the
|
||||
installed transformers version selects, leaving the other path untested.
|
||||
"""
|
||||
installed transformers version selects, leaving the other path untested."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
@@ -41,8 +40,7 @@ def create_processor(model_id: str, processor_cls):
|
||||
|
||||
def create_cached_processor(model_id: str, processor_cls):
|
||||
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||
cache back so a test can check it was actually used.
|
||||
"""
|
||||
cache back so a test can check it was actually used."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
ctx = InputProcessingContext(
|
||||
|
||||
@@ -18,8 +18,7 @@ from vllm.model_executor.models.cohere_asr import CohereAsrForConditionalGenerat
|
||||
def test_get_num_audio_tokens_streaming_estimate(audio_duration_s, expected_tokens):
|
||||
"""The duration-based estimate must convert ``window_stride`` (seconds) to a
|
||||
sample hop and divide by the encoder subsampling factor. Values are pinned to
|
||||
concrete numbers so a rounding regression is actually caught.
|
||||
"""
|
||||
concrete numbers so a rounding regression is actually caught."""
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
preprocessor={"window_stride": 0.01, "sample_rate": 16000},
|
||||
|
||||
@@ -78,8 +78,7 @@ def test_cosmos3_modelopt_quantizer_weights_mapper():
|
||||
"""ModelOpt/Diffusers FP8 checkpoints ship native fake-quant buffers
|
||||
(``*_quantizer._amax`` / ``._scale``) alongside the vLLM-consumable
|
||||
``weight_scale`` / ``input_scale`` sidecars. vLLM must drop the former
|
||||
(it has no parameter for them) while keeping the latter.
|
||||
"""
|
||||
(it has no parameter for them) while keeping the latter."""
|
||||
from vllm.model_executor.models.cosmos3 import Cosmos3ForConditionalGeneration
|
||||
|
||||
mapper = Cosmos3ForConditionalGeneration.hf_to_vllm_mapper
|
||||
|
||||
@@ -81,8 +81,7 @@ class _FakeTensor:
|
||||
"""Sentinel stand-in for torch.Tensor in load_weights tests. Supports the
|
||||
.detach().clone() chain used by load_weights for buffered mm weights;
|
||||
both methods return self so identity (and the existing equality
|
||||
assertions) are preserved through cloning.
|
||||
"""
|
||||
assertions) are preserved through cloning."""
|
||||
|
||||
def detach(self):
|
||||
return self
|
||||
@@ -166,8 +165,7 @@ def _make_mm_items_with_video_bytes(
|
||||
|
||||
def test_extract_audio_from_videos_passes_max_duration():
|
||||
"""_extract_audio_from_videos must forward VLLM_MAX_AUDIO_DECODE_DURATION_S
|
||||
to load_audio_pyav so decompression-bomb audio is rejected.
|
||||
"""
|
||||
to load_audio_pyav so decompression-bomb audio is rejected."""
|
||||
dummy_audio = (np.zeros(16000, dtype=np.float32), 16000.0)
|
||||
mm_items = _make_mm_items_with_video_bytes(b"\x00" * 64)
|
||||
|
||||
@@ -187,8 +185,7 @@ def test_extract_audio_from_videos_passes_max_duration():
|
||||
|
||||
def test_extract_audio_from_videos_rejects_oversized_audio():
|
||||
"""When load_audio_pyav raises due to duration limit the video is
|
||||
marked as having no audio instead of crashing the server.
|
||||
"""
|
||||
marked as having no audio instead of crashing the server."""
|
||||
mm_items = _make_mm_items_with_video_bytes(b"\x00" * 64)
|
||||
|
||||
processor = object.__new__(NanoNemotronVLMultiModalProcessor)
|
||||
|
||||
@@ -144,8 +144,7 @@ def test_qwen4_exp_mtp_override_sets_draft_config(
|
||||
def test_qwen4_exp_rejects_pipeline_parallel_only_with_ple(ple_layer_ids) -> None:
|
||||
"""PLE needs raw input_ids, which non-first pipeline ranks never see. The
|
||||
rest of the architecture is PP-capable, so the refusal must be conditional
|
||||
-- and must land before the engine spends time loading weights.
|
||||
"""
|
||||
-- and must land before the engine spends time loading weights."""
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_text_config=_text_config(ple_layer_ids=ple_layer_ids),
|
||||
|
||||
@@ -103,8 +103,7 @@ def test_fi_backend_strings_are_registered_mega_moe_backends():
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
|
||||
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
|
||||
instead of silently falling through to the generic FusedMoE path.
|
||||
"""
|
||||
instead of silently falling through to the generic FusedMoE path."""
|
||||
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
|
||||
|
||||
@@ -120,8 +119,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
)
|
||||
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
|
||||
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
|
||||
models validate their own constraints at construction time.
|
||||
"""
|
||||
models validate their own constraints at construction time."""
|
||||
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
|
||||
|
||||
|
||||
@@ -134,8 +132,7 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
"""Every mega backend must qualify for sequence-parallel MoE at
|
||||
TP>1/EP: the predicate once matched only the native backend string,
|
||||
which silently ran the fi backends full-batch with an all-reduce on
|
||||
every rank — 0.42-0.65x native e2e at TP8.
|
||||
"""
|
||||
every rank — 0.42-0.65x native e2e at TP8."""
|
||||
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
@@ -179,8 +176,7 @@ def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
|
||||
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
|
||||
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
|
||||
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
|
||||
device it already bound via BootstrapConfig.device.
|
||||
"""
|
||||
device it already bound via BootstrapConfig.device."""
|
||||
import vllm.utils.flashinfer_moe_ep as mod
|
||||
|
||||
pg = object()
|
||||
@@ -245,8 +241,7 @@ def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
|
||||
|
||||
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
|
||||
"""One 32-element scale group per row: low nibble is the even element,
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group.
|
||||
"""
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group."""
|
||||
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
|
||||
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
|
||||
|
||||
|
||||
@@ -783,8 +783,7 @@ def test_deepseek_v4_mega_moe_stages_shared_scale_tma_layout(shared_block_m):
|
||||
def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
|
||||
"""The loader invokes the model-level PWAL hook for every load format,
|
||||
so it must finalize megamoe + mhc broadcast weights to cover dummy
|
||||
load, which skips load_weights().
|
||||
"""
|
||||
load, which skips load_weights()."""
|
||||
calls = []
|
||||
stub = SimpleNamespace(
|
||||
model=SimpleNamespace(
|
||||
@@ -801,8 +800,7 @@ def test_deepseek_v4_pwal_hook_finalizes_mega_moe_and_mhc_broadcast():
|
||||
def test_deepseek_v4_drafter_pwal_hooks_finalize_mega_moe():
|
||||
"""MTP/DSpark drafters load as their own top-level models, so each needs
|
||||
its own PWAL hook now that the megamoe forward no longer finalizes
|
||||
weights lazily on first use.
|
||||
"""
|
||||
weights lazily on first use."""
|
||||
calls = []
|
||||
mtp = SimpleNamespace(finalize_mega_moe_weights=lambda: calls.append("mtp"))
|
||||
DeepSeekV4MTP.process_weights_after_loading(mtp)
|
||||
|
||||
@@ -215,8 +215,7 @@ def test_v41_dspark_loads_linear_scales(
|
||||
):
|
||||
"""Checkpoint ``.scale`` maps to the quant method's scale parameter and
|
||||
loads untouched. MXFP8 block-scale expansion lives in the KMxfp8Static
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights.
|
||||
"""
|
||||
loader (see tests/quantization/test_modelopt.py), not in load_weights."""
|
||||
from vllm.models.deepseek_v41.nvidia import dspark
|
||||
|
||||
mxfp8 = scale_dtype != torch.float32
|
||||
|
||||
@@ -73,8 +73,7 @@ def test_mtp_override_extracts_n_predict_from_multimodal_wrapper(
|
||||
model_type: str, expected_arch: str
|
||||
) -> None:
|
||||
"""Verify that multimodal wrapper checkpoints with mtp_num_hidden_layers
|
||||
in text_config resolve n_predict and architecture correctly.
|
||||
"""
|
||||
in text_config resolve n_predict and architecture correctly."""
|
||||
cfg = SpeculativeConfig.hf_config_override(
|
||||
_multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
|
||||
)
|
||||
@@ -94,8 +93,7 @@ def test_mtp_override_top_level_precedence_over_nested_text_config(
|
||||
model_type: str, expected_arch: str
|
||||
) -> None:
|
||||
"""Verify that an explicit top-level mtp_num_hidden_layers takes precedence
|
||||
over a nested text_config value.
|
||||
"""
|
||||
over a nested text_config value."""
|
||||
cfg = _multimodal_wrapper_mtp_config(model_type, mtp_layers=2)
|
||||
cfg.mtp_num_hidden_layers = 3
|
||||
overridden = SpeculativeConfig.hf_config_override(cfg)
|
||||
@@ -115,8 +113,7 @@ def test_mtp_override_downloads_real_hf_hub_configs(
|
||||
model_id: str, expected_arch: str
|
||||
) -> None:
|
||||
"""Verify that unmodified real-world checkpoints downloaded via
|
||||
AutoConfig.from_pretrained resolve n_predict=1 from text_config.
|
||||
"""
|
||||
AutoConfig.from_pretrained resolve n_predict=1 from text_config."""
|
||||
hf_config = AutoConfig.from_pretrained(model_id)
|
||||
cfg = SpeculativeConfig.hf_config_override(hf_config)
|
||||
assert cfg.model_type == "qwen3_5_mtp"
|
||||
|
||||
@@ -197,8 +197,7 @@ def test_merge_multimodal_embeddings_no_sync():
|
||||
def test_get_rename_mapper_keeps_only_renames():
|
||||
"""`None` means "do not load", which is meaningless to the consumers of
|
||||
this mapper (LoRA name parsing, quantization config layer lists), and
|
||||
applying it would silently shrink their lists.
|
||||
"""
|
||||
applying it would silently shrink their lists."""
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_regex={re.compile(r"^drop_regex\."): None},
|
||||
orig_to_new_substr={"drop_substr": None, "keep_substr": "kept"},
|
||||
|
||||
@@ -512,8 +512,7 @@ def _reference_input_norm(
|
||||
def test_fused_input_norm_matches_reference(num_patches: int):
|
||||
"""FusedInputNorm must equal the plain affine, including for num_patches
|
||||
above the cuDNN batch-norm grid limit (~65535) that previously raised
|
||||
CUDNN_STATUS_INTERNAL_ERROR (issue #51717).
|
||||
"""
|
||||
CUDNN_STATUS_INTERNAL_ERROR (issue #51717)."""
|
||||
channel = 3
|
||||
patch_size = 14 * 14
|
||||
image_mean = [0.48145466, 0.4578275, 0.40821073]
|
||||
|
||||
@@ -251,8 +251,7 @@ class UnderscoreInplaceParallelLinears(FourParallelLinears):
|
||||
|
||||
class PlainQKV(nn.Module):
|
||||
"""q/k/v returned as a bare tuple: no reshape stands between the calls and
|
||||
the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it.
|
||||
"""
|
||||
the return, so both QKVFuser and MergedColumnParallelFuser can rewrite it."""
|
||||
|
||||
def __init__(self, hidden: int = 32, heads: int = 4, head_dim: int = 8):
|
||||
super().__init__()
|
||||
@@ -339,8 +338,7 @@ class GuardedVAttention(FakeAttention):
|
||||
"""Gemma 4-style: `v_proj` is called once but also read in a `None` guard.
|
||||
|
||||
The guard is a fusion invariant (the fuser only binds instances where the
|
||||
projection exists), so it folds to `True` and the call still fuses.
|
||||
"""
|
||||
projection exists), so it folds to `True` and the call still fuses."""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -370,8 +368,7 @@ class TruthyGuardAttention(FakeAttention):
|
||||
|
||||
An `nn.Linear` is always truthy, so the guard folds to `True` and the call
|
||||
still fuses (the projection is a proven invariant, exactly as for a `None`
|
||||
guard).
|
||||
"""
|
||||
guard)."""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -402,8 +399,7 @@ class StatementGuardVAttention(FakeAttention):
|
||||
The `if`'s test is not an `ast.Compare`, so the identity-check scan skips it;
|
||||
`_in_boolean_context` folds the test to `True` (an `nn.Linear` is always
|
||||
truthy) and the call inside the now-`if True:` body still fuses, with the
|
||||
live `else` surviving the rewrite.
|
||||
"""
|
||||
live `else` surviving the rewrite."""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -432,8 +428,7 @@ class BranchedKVAttention(FakeAttention):
|
||||
|
||||
The three calls do not share a block, so the fused GEMM must be hoisted to
|
||||
the innermost block that dominates all of them (the function body). The
|
||||
live `if`/`else` must survive the rewrite.
|
||||
"""
|
||||
live `if`/`else` must survive the rewrite."""
|
||||
|
||||
is_kv_shared_layer = False
|
||||
|
||||
@@ -464,8 +459,7 @@ class RebindArgAttention(FakeAttention):
|
||||
|
||||
The branch is dead at trace time so the match succeeds, but the source
|
||||
rewrite must refuse: hoisting the GEMM above the rebind would feed k/v a
|
||||
different input. Result: no fusion.
|
||||
"""
|
||||
different input. Result: no fusion."""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -496,8 +490,7 @@ class InPlaceMutatedArgAttention(FakeAttention):
|
||||
Like `RebindArgAttention`, but the mutation keeps the name in a `Load`
|
||||
context (`hidden_states.mul_(2)`), so a bare `Name`-store check would miss it.
|
||||
The rewrite must still refuse: hoisting the GEMM above the mutation changes
|
||||
the value k/v see. Result: no fusion.
|
||||
"""
|
||||
the value k/v see. Result: no fusion."""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -527,8 +520,7 @@ class AliasMutatedArgAttention(FakeAttention):
|
||||
|
||||
`hidden_states.view(-1)` shares storage with its base, so `alias.mul_(2)`
|
||||
changes what k/v read without ever naming `hidden_states` as a target. Only
|
||||
tracking names that may alias the input catches it. Result: no fusion.
|
||||
"""
|
||||
tracking names that may alias the input catches it. Result: no fusion."""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -559,8 +551,7 @@ class FunctionalInplaceArgAttention(FakeAttention):
|
||||
|
||||
`F.relu(hidden_states, inplace=True)` writes through an argument, so the
|
||||
name appears only in a `Load` context inside a call that is not a method on
|
||||
it -- invisible to a trailing-underscore method check. Result: no fusion.
|
||||
"""
|
||||
it -- invisible to a trailing-underscore method check. Result: no fusion."""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -590,8 +581,7 @@ class SubscriptMutatedArgAttention(FakeAttention):
|
||||
|
||||
`hidden_states[..., 0] = 0` leaves the name in a `Load` context on the
|
||||
subscript's value, so it too evades a bare `Name`-store check. The rewrite
|
||||
must refuse for the same reason. Result: no fusion.
|
||||
"""
|
||||
must refuse for the same reason. Result: no fusion."""
|
||||
|
||||
recompute = False
|
||||
|
||||
@@ -621,8 +611,7 @@ class NonFoldableRefAttention(FakeAttention):
|
||||
|
||||
Deleting `v_proj` would break `isinstance(self.v_proj, nn.Linear)`: it reads
|
||||
the projection's type, not just whether it exists, so the fuser cannot fold
|
||||
it and must refuse rather than rewrite it.
|
||||
"""
|
||||
it and must refuse rather than rewrite it."""
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
@@ -847,8 +836,7 @@ def test_glu_identifies_down_projection():
|
||||
|
||||
It is forced to `RowParallelLinear` in `update_attrs` so its sharded input
|
||||
matches the column-parallel merged gate/up; `None` when there is no such
|
||||
projection to force (fusion of gate/up still applies).
|
||||
"""
|
||||
projection to force (fusion of gate/up still applies)."""
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
|
||||
assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
|
||||
@@ -1009,8 +997,7 @@ def test_bypass_existence_guard_folds_identity_none_check(expr, expected):
|
||||
The projection is a proven invariant (it exists as an `nn.Linear`), so an
|
||||
`is (not) None` guard has a constant truth value; folding it lets the call be
|
||||
rewritten away without changing the guard's outcome. `==`/`!=` are excluded
|
||||
(see the refusal test) because a subclass may override `__eq__`/`__ne__`.
|
||||
"""
|
||||
(see the refusal test) because a subclass may override `__eq__`/`__ne__`."""
|
||||
funcdef, ref = _guard_funcdef(expr)
|
||||
bypass_existence_guard(funcdef, ref, "v_proj")
|
||||
test = next(node for node in ast.walk(funcdef) if isinstance(node, ast.IfExp)).test
|
||||
@@ -1029,8 +1016,7 @@ def test_bypass_existence_guard_folds_bare_truthiness(expr):
|
||||
|
||||
An `nn.Linear` has no `__bool__`/`__len__`, so it is always truthy; in a pure
|
||||
boolean position (a test, or a `not` operand) the reference can be replaced by
|
||||
`True`, letting the projection be deleted.
|
||||
"""
|
||||
`True`, letting the projection be deleted."""
|
||||
funcdef, ref = _guard_funcdef(expr)
|
||||
bypass_existence_guard(funcdef, ref, "v_proj")
|
||||
# No `self.v_proj` reference survives; a literal `True` took its place.
|
||||
@@ -1045,8 +1031,7 @@ def test_bypass_existence_guard_folds_bare_if_statement():
|
||||
|
||||
The `len(node.ops) == 1` `Compare` scan skips it (it is not a comparison at
|
||||
all); `_in_boolean_context` then matches the `if`'s test and folds it, so the
|
||||
bare-truthiness statement form is handled, not silently dropped.
|
||||
"""
|
||||
bare-truthiness statement form is handled, not silently dropped."""
|
||||
funcdef = ast.parse(
|
||||
"def f(self, a, b):\n if self.v_proj:\n return a\n return b"
|
||||
).body[0]
|
||||
@@ -1078,8 +1063,7 @@ def test_bypass_existence_guard_refuses_non_guard_reference(expr):
|
||||
Rewriting it is unsafe (its value depends on more than the projection's
|
||||
existence), so the fuser refuses rather than change semantics. `==`/`!=` are
|
||||
refused too: `is_linear` accepts `nn.Linear` subclasses that could override
|
||||
`__eq__`/`__ne__`, so equality need not agree with `is (not) None`.
|
||||
"""
|
||||
`__eq__`/`__ne__`, so equality need not agree with `is (not) None`."""
|
||||
funcdef, ref = _guard_funcdef(expr)
|
||||
with pytest.raises(ValueError, match="outside an existence guard"):
|
||||
bypass_existence_guard(funcdef, ref, "v_proj")
|
||||
@@ -1119,8 +1103,7 @@ def test_merged_column_fuser_folds_existence_guard():
|
||||
|
||||
Guard folding lives on `StackedFuser`, not just `QKVFuser`, because every
|
||||
stacked fuser deletes the projections it merges -- a surviving reference to
|
||||
one would fail at runtime.
|
||||
"""
|
||||
one would fail at runtime."""
|
||||
with torch.device("meta"):
|
||||
module = GuardedParallelLinears()
|
||||
fuser = MergedColumnParallelFuser.match(trace(module), module)
|
||||
@@ -1146,8 +1129,7 @@ def test_merged_column_fuser_rejects_input_mutation(cls):
|
||||
The mutation may be direct (`x.add_(1)`), through a view that shares the
|
||||
input's storage, or through a call that writes an argument in place
|
||||
(`F.relu(x, inplace=True)`, `torch.relu_(x)`). Each one silently changes the
|
||||
fused result, so the rewrite must refuse rather than fuse.
|
||||
"""
|
||||
fused result, so the rewrite must refuse rather than fuse."""
|
||||
with torch.device("meta"):
|
||||
module = cls()
|
||||
fuser = MergedColumnParallelFuser.match(trace(module), module)
|
||||
@@ -1205,8 +1187,7 @@ def test_merged_column_fuser_supports_any_number_of_linears(
|
||||
|
||||
def test_merged_name_is_unique_per_linear_names():
|
||||
"""The computed name must not collide across different projections, so
|
||||
stacking two unrelated fusions can never clobber packed_modules_mapping.
|
||||
"""
|
||||
stacking two unrelated fusions can never clobber packed_modules_mapping."""
|
||||
ab_c = MergedColumnParallelFuser(source_cls="M", linear_names=("a_b", "c"))
|
||||
a_bc = MergedColumnParallelFuser(source_cls="M", linear_names=("a", "b_c"))
|
||||
assert ab_c.merged_name != a_bc.merged_name
|
||||
@@ -1222,8 +1203,7 @@ def test_heterogeneous_instances_fall_back_to_merged_column_fuser(
|
||||
layer whose out_features aren't a multiple of its own head_dim (as a
|
||||
misconfigured or heterogeneous checkpoint might have) fails QKVFuser's
|
||||
validation and must still fuse via the generic fallback, while a
|
||||
well-formed sibling of the same class keeps using QKVFuser.
|
||||
"""
|
||||
well-formed sibling of the same class keeps using QKVFuser."""
|
||||
with torch.device("meta"):
|
||||
container = nn.Module()
|
||||
container.compatible = PlainQKV(head_dim=8)
|
||||
@@ -1244,8 +1224,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
|
||||
"""A single projection split into q/k/v must be re-sharded, not merged.
|
||||
|
||||
Only the split sizes change: `QKVParallelLinear` loads the packed
|
||||
checkpoint weight as-is, and shards q by heads while replicating k/v.
|
||||
"""
|
||||
checkpoint weight as-is, and shards q by heads while replicating k/v."""
|
||||
with torch.device("meta"):
|
||||
meta = PackedQKVAttention(kv_heads=kv_heads)
|
||||
fuser = get_fuser(meta, PackedQKVFuser)
|
||||
@@ -1276,8 +1255,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
|
||||
|
||||
def test_per_head_split_is_not_packed_qkv():
|
||||
"""The split must consume the whole projection, else its sizes are head
|
||||
widths and re-sharding by them would be wrong.
|
||||
"""
|
||||
widths and re-sharding by them would be wrong."""
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
|
||||
|
||||
@@ -1397,8 +1375,7 @@ def test_act_and_mul_derived_from_module(default_vllm_config):
|
||||
|
||||
def _wider_model_config(head_dim: int) -> SimpleNamespace:
|
||||
"""A model whose global head size is twice `head_dim`, as a wider layer
|
||||
elsewhere in a heterogeneous checkpoint would make it.
|
||||
"""
|
||||
elsewhere in a heterogeneous checkpoint would make it."""
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_head_size=lambda: 2 * head_dim),
|
||||
quant_config=None,
|
||||
|
||||
@@ -45,8 +45,7 @@ def _attention(q_lora_rank: int | None):
|
||||
|
||||
def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen).
|
||||
"""
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
|
||||
attn = _attention(q_lora_rank)
|
||||
return MLAFuser.match(trace(attn), attn)
|
||||
|
||||
@@ -82,8 +81,7 @@ def test_discovers_modules_with_q_lora():
|
||||
|
||||
def test_q_lora_stacks_qkv_a_proj():
|
||||
"""The MLA layer reads `q_a_proj` and `kv_a_proj_with_mqa` fused into one
|
||||
down-projection, so both checkpoint weights must remap into it.
|
||||
"""
|
||||
down-projection, so both checkpoint weights must remap into it."""
|
||||
fuser = _match(q_lora_rank=64)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
prefix = "model.layers.0.self_attn"
|
||||
@@ -104,8 +102,7 @@ def test_update_forward_rewrites_real_attention(q_lora_rank):
|
||||
The q-LoRA path hoists the merged down-projection to a top-level statement
|
||||
while its calls sit inside the `q_lora_rank` branches, so the region it
|
||||
spans is wide; a check that over-approximates aliasing would silently refuse
|
||||
here and cost the fusion rather than fail loudly.
|
||||
"""
|
||||
here and cost the fusion rather than fail loudly."""
|
||||
fuser = _match(q_lora_rank)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
fuser.update_forward(_attention(q_lora_rank))
|
||||
@@ -129,8 +126,7 @@ class _Norm(nn.Module):
|
||||
|
||||
class RenamedMLA(nn.Module):
|
||||
"""An MLA-shaped attention whose children have non-standard names, proving
|
||||
discovery is by structure and not attribute name.
|
||||
"""
|
||||
discovery is by structure and not attribute name."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -157,8 +153,7 @@ class RenamedMLA(nn.Module):
|
||||
|
||||
def test_discovers_modules_under_arbitrary_names():
|
||||
"""Discovery is purely structural: `RenamedMLA` gives its children non-standard
|
||||
names, and `match` still locates each projection by dataflow.
|
||||
"""
|
||||
names, and `match` still locates each projection by dataflow."""
|
||||
with torch.device("meta"):
|
||||
module = RenamedMLA()
|
||||
fuser = MLAFuser.match(trace(module), module)
|
||||
|
||||
@@ -277,8 +277,7 @@ def test_moe_fuser_matches_scaled_router():
|
||||
|
||||
def test_moe_fuser_matches_grouped_router():
|
||||
"""Group masking between the score and the routing top-k still matches: the
|
||||
scorer is anchored on the routing (last) top-k, not the group one.
|
||||
"""
|
||||
scorer is anchored on the routing (last) top-k, not the group one."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(GroupedRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
@@ -288,8 +287,7 @@ def test_moe_fuser_matches_grouped_router():
|
||||
|
||||
def test_moe_fuser_matches_correction_router():
|
||||
"""A score-correction bias buffer (DeepSeek-V3 noaux) is allowed; the
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers.
|
||||
"""
|
||||
rebuilt gate carries it in fp32 for FusedMoE's biased routers."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(CorrectionRouter)
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
@@ -300,8 +298,7 @@ def test_moe_fuser_matches_correction_router():
|
||||
def test_moe_fuser_reads_router_dtype_from_the_gate():
|
||||
"""A router that computes its logits in fp32 must keep routing in fp32 when
|
||||
rebuilt, even though no config field names the dtype. The cast back to the
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype.
|
||||
"""
|
||||
activation dtype after the top-k must not be mistaken for the routing dtype."""
|
||||
with torch.device("meta"):
|
||||
fp32 = MoEBlockFuser.match(MoEBlock(Fp32Router), "experts")
|
||||
default = MoEBlockFuser.match(MoEBlock(TopKRouter), "experts")
|
||||
@@ -320,8 +317,7 @@ def test_moe_fuser_detects_shared_experts():
|
||||
|
||||
def test_moe_fuser_skips_shared_detection_without_extra_children():
|
||||
"""With only experts and gate, shared-expert detection (and its block trace)
|
||||
is skipped, so a gate-derived add is not misread as a shared expert.
|
||||
"""
|
||||
is skipped, so a gate-derived add is not misread as a shared expert."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNoShared()
|
||||
fuser = MoEBlockFuser.match(block, "experts")
|
||||
@@ -366,8 +362,7 @@ def test_moe_fuser_declines_unsupported(block_cls):
|
||||
|
||||
def test_moe_fuser_ignores_nested_returns():
|
||||
"""A tuple `return` inside a nested helper must not decline a block whose own
|
||||
forward returns a single tensor.
|
||||
"""
|
||||
forward returns a single tensor."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlockNestedTupleReturn()
|
||||
assert isinstance(MoEBlockFuser.match(block, "experts"), MoEBlockFuser)
|
||||
@@ -375,8 +370,7 @@ def test_moe_fuser_ignores_nested_returns():
|
||||
|
||||
def test_moe_fuser_router_requires_connected_dataflow():
|
||||
"""A gate with linear + softmax + top-k present but not wired as a router
|
||||
(top-k selects over the input, not the scored logits) is not detected.
|
||||
"""
|
||||
(top-k selects over the input, not the scored logits) is not detected."""
|
||||
with torch.device("meta"):
|
||||
block = MoEBlock(DisconnectedRouter)
|
||||
assert MoEBlockFuser.match(block, "experts") is None
|
||||
|
||||
@@ -83,8 +83,7 @@ class ToleranceRMSNorm(NamedEpsRMSNorm):
|
||||
|
||||
class LiteralEpsRMSNorm(RMSNorm):
|
||||
"""eps is a literal in the source, and an unrelated attribute happens to
|
||||
hold the same value: matching on the value alone would bind to it.
|
||||
"""
|
||||
hold the same value: matching on the value alone would bind to it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__(hidden, eps)
|
||||
@@ -96,8 +95,7 @@ class LiteralEpsRMSNorm(RMSNorm):
|
||||
|
||||
class AmbiguousEpsRMSNorm(nn.Module):
|
||||
"""Two attributes hold the eps value, and the forward reads the second, so
|
||||
their order cannot pick it.
|
||||
"""
|
||||
their order cannot pick it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
@@ -233,8 +231,7 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config):
|
||||
|
||||
def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
"""`torch.nn.RMSNorm` (a single `F.rms_norm` call) matches via the fast path;
|
||||
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`.
|
||||
"""
|
||||
its default `eps=None` resolves to `finfo(dtype).eps` in `fuse`."""
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm as VLLMRMSNorm
|
||||
|
||||
with torch.device("meta"):
|
||||
@@ -250,8 +247,7 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
|
||||
def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
"""Two instances of the same norm class with different eps must fuse to their
|
||||
own eps: the type-cached fuser holds only structure, not this value.
|
||||
"""
|
||||
own eps: the type-cached fuser holds only structure, not this value."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = RMSNorm(16, eps=eps)
|
||||
@@ -263,8 +259,7 @@ def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
|
||||
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
"""The eps attribute is identified by holding the traced value, so a norm
|
||||
stays per-instance correct whatever it names it.
|
||||
"""
|
||||
stays per-instance correct whatever it names it."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = cls(16, eps=eps)
|
||||
@@ -276,8 +271,7 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
|
||||
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
|
||||
"""A literal eps is recognised as coming from no attribute, even when one
|
||||
holds the same value, and is taken from the traced source instead.
|
||||
"""
|
||||
holds the same value, and is taken from the traced source instead."""
|
||||
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
|
||||
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
|
||||
module = LiteralEpsRMSNorm()
|
||||
@@ -290,8 +284,7 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo
|
||||
|
||||
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
"""When several attributes hold the eps value, the one the forward actually
|
||||
reads is identified, and they are all left as they were found.
|
||||
"""
|
||||
reads is identified, and they are all left as they were found."""
|
||||
with torch.device("meta"):
|
||||
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
|
||||
before = dict(vars(module))
|
||||
@@ -308,8 +301,7 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
|
||||
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
|
||||
plumbing. A full-width input skips the gather and equals a plain norm.
|
||||
"""
|
||||
plumbing. A full-width input skips the gather and equals a plain norm."""
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
@@ -331,8 +323,7 @@ def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
|
||||
def test_gathered_norm_rejects_uneven_sharding(default_vllm_config):
|
||||
"""A sharded input (narrower than the full-width weight) that does not tile
|
||||
the weight evenly across ranks is rejected before any collective.
|
||||
"""
|
||||
the weight evenly across ranks is rejected before any collective."""
|
||||
from vllm.model_executor.models.transformers.fusers import rms_norm
|
||||
|
||||
norm = rms_norm.TPAwareRMSNorm(hidden_size=8, eps=1e-6)
|
||||
|
||||
@@ -28,8 +28,7 @@ def fake_hw_layernorm(monkeypatch):
|
||||
A `SimpleNamespace` stands in for the module: `importlib.import_module`
|
||||
returns it from `sys.modules` and `getattr` resolves `RMSNorm`, while its
|
||||
attributes are set at construction (no `ModuleType` attribute-set that mypy
|
||||
rejects, no constant `setattr` that ruff rejects).
|
||||
"""
|
||||
rejects, no constant `setattr` that ruff rejects)."""
|
||||
module = types.SimpleNamespace(RMSNorm=type("HwRMSNorm", (), {}))
|
||||
monkeypatch.setitem(sys.modules, HW_MODULE, module)
|
||||
return module
|
||||
@@ -82,8 +81,7 @@ def test_act_and_mul_falls_back_for_unknown_activation(
|
||||
@pytest.fixture(scope="module")
|
||||
def tiny_llama_path(tmp_path_factory):
|
||||
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
|
||||
tokenizer) so vLLM can load it like any local checkpoint.
|
||||
"""
|
||||
tokenizer) so vLLM can load it like any local checkpoint."""
|
||||
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
|
||||
|
||||
Reference in New Issue
Block a user