[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)

Signed-off-by: Taimys <ffy2281693949@gmail.com>
Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com>
Co-authored-by: Taimys <ffy2281693949@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Taimys
2026-09-30 11:06:07 +00:00
committed by GitHub
co-authored by Codex Luyi Xiao Taimys Harry Mellor Claude Opus 5.5
parent c4fa6f36a7
commit 5225f1bacb
118 changed files with 948 additions and 605 deletions
+4 -4
View File
@@ -123,8 +123,8 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
method = object.__new__(AutoGPTQMoEMethod)
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
method.input_dtype = None
method.experts_cls = None
method.moe = SimpleNamespace(w13_num_shards=2)
method.experts_cls = None # type: ignore[assignment] # Weight creation does not select experts.
method.moe = SimpleNamespace(w13_num_shards=2) # type: ignore[assignment]
layer = torch.nn.Module()
method.create_weights(
@@ -170,7 +170,7 @@ def test_routed_experts_loads_per_expert_biases():
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
):
assert RoutedExperts.weight_loader(
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
loader,
w13_bias,
loaded,
@@ -180,7 +180,7 @@ def test_routed_experts_loads_per_expert_biases():
return_success=True,
)
assert RoutedExperts.weight_loader(
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
loader,
w2_bias,
torch.tensor([9.0, 10.0, 11.0, 12.0]),
+9 -9
View File
@@ -193,7 +193,7 @@ class DummyFusedMoE:
def make_config(**overrides) -> INCConfig:
kwargs = {
kwargs: dict[str, Any] = {
"weight_bits": 4,
"group_size": 128,
"sym": True,
@@ -208,7 +208,7 @@ def make_config(**overrides) -> INCConfig:
def make_layer_config(**overrides) -> INCLayerConfig:
kwargs = {
kwargs: dict[str, Any] = {
"bits": 4,
"group_size": 128,
"sym": True,
@@ -423,7 +423,7 @@ def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix(
config = make_config(extra_config={"model.moe_layer": {"bits": 16}})
layer = object.__new__(RoutedExperts)
layer.moe_config = None
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
method = config.get_quant_method(layer, "moe_layer")
@@ -730,7 +730,7 @@ def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
)
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
layer_config = make_layer_config(bits=bits)
method = INCWna16Scheme().get_moe_method(
make_config(), layer, "model.layers.0.mlp", layer_config
@@ -861,7 +861,7 @@ def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe(
config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16")
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
method = config.get_quant_method(layer, "model.layers.0.mlp")
@@ -890,7 +890,7 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4")
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
ignored_method = config.get_quant_method(
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
@@ -1186,7 +1186,7 @@ def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None:
)
layer = object.__new__(RoutedExperts)
layer.moe_config = DummyMoeConfig()
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
method = INCWna16Scheme().get_moe_method(
make_config(),
layer,
@@ -1669,7 +1669,7 @@ def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
when extra_config has bits >= 16."""
config = make_config(extra_config={"layer": {"bits": 16}})
layer = object.__new__(RoutedExperts)
layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
class DummyUnquantizedFusedMoEMethod:
def __init__(self, moe_config) -> None:
@@ -2094,7 +2094,7 @@ def _with_w4a8_kernel(monkeypatch) -> None:
def _dispatch(layer_config=None):
return INCWna16Scheme().get_linear_method(
object(), object(), "layer", layer_config or make_layer_config()
make_config(), object(), "layer", layer_config or make_layer_config()
)
+1
View File
@@ -99,6 +99,7 @@ def test_machete_rejects_invalid_config(
)
can_impl, reason = MacheteLinearKernel.can_implement(config)
assert not can_impl
assert reason is not None
assert expected_reason in reason
+27 -14
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention.attention import (
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import (
Fp8Config,
Fp8LinearMethod,
@@ -79,17 +80,22 @@ def test_deepseek_v41_mxfp8_scale_loading(
for module in (model_module, linear_module):
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: tp_rank)
tp_args = dict(quant_config=quant_config, bias=False)
block = torch.nn.Module()
block.attn = torch.nn.Module()
block.attn.wq_b = ColumnParallelLinear(128, 128, **tp_args)
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(128, [128, 64], **tp_args)
block.attn.wq_b = ColumnParallelLinear(
128, 128, quant_config=quant_config, bias=False
)
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(
128, [128, 64], quant_config=quant_config, bias=False
)
block.ffn = torch.nn.Module()
block.ffn.shared_experts = torch.nn.Module()
block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear(
128, [128, 128], **tp_args
128, [128, 128], quant_config=quant_config, bias=False
)
block.ffn.shared_experts.down_proj = RowParallelLinear(
128, 128, quant_config=quant_config, bias=False
)
block.ffn.shared_experts.down_proj = RowParallelLinear(128, 128, **tp_args)
block.ffn.experts = torch.nn.Module()
block.ffn.experts.register_parameter(
"weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False)
@@ -111,9 +117,11 @@ def test_deepseek_v41_mxfp8_scale_loading(
model.config = SimpleNamespace(num_attention_heads=2)
model.quant_config = quant_config
model.use_sequence_parallel = False
model.get_expert_mapping = lambda: [
("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")
]
monkeypatch.setattr(
model,
"get_expert_mapping",
lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")],
)
checkpoints = [
("attn.wq_b", "attn.wq_b", 128, 128, 0, None),
@@ -211,7 +219,10 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
vllm_config = SimpleNamespace(
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
)
resolved = model_module._linear_scale_param_name(vllm_config, expert_dtype)
resolved = model_module._linear_scale_param_name(
vllm_config, # type: ignore[arg-type]
expert_dtype,
)
assert resolved == scale_name
mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved)
@@ -373,7 +384,7 @@ def test_deepgemm_mxfp8_preserves_weight_and_scale_values(
@pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
def test_mxfp8_bmm_loads_and_projects_grouped_weights(
dist_init, default_vllm_config, prequantized, config_source, num_tokens
dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens
):
"""BMM metadata set after construction selects grouped weight processing."""
from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import (
@@ -398,10 +409,11 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell")
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
quant_config = DeepseekV4FP8Config(
deepseek_config = DeepseekV4FP8Config(
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
)
quant_config._resolved_expert_dtype = "fp4"
deepseek_config._resolved_expert_dtype = "fp4"
quant_config: QuantizationConfig = deepseek_config
if config_source == "mxfp8":
quant_config = get_quantization_config("mxfp8").from_config(
{"quant_method": "mxfp8"}
@@ -432,7 +444,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
model.config = SimpleNamespace(num_attention_heads=2)
model.quant_config = quant_config
model.use_sequence_parallel = False
model.get_expert_mapping = lambda: []
monkeypatch.setattr(model, "get_expert_mapping", lambda: [])
model.load_weights(
[
("layers.0.attn.wo_a.weight", weight),
@@ -443,6 +455,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
weight.float().reshape(8, 32, 16, 32)
* torch.exp2(scales.float() - 127)[:, None, :, None]
).reshape(2, 128, 512)
assert isinstance(linear.quant_method, ModelOptLinearMethod)
linear.quant_method.process_weights_after_loading(linear)
linear.quant_method.process_weights_after_loading(linear)
assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel)
@@ -785,7 +798,7 @@ def test_kv_cache_scale_sync_to_host_copies():
set_default_quant_scales(layer, register_buffer=True)
layer.kv_cache_dtype = "fp8"
method = BaseKVCacheMethod(quant_config=None)
method = BaseKVCacheMethod(quant_config=None) # type: ignore[arg-type] # Scale synchronization does not use quant_config.
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
+2 -1
View File
@@ -8,6 +8,7 @@ These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -53,7 +54,7 @@ def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
set_rocm_aiter(monkeypatch, False)
def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
+2 -1
View File
@@ -10,6 +10,7 @@ CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950.
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -44,7 +45,7 @@ requires_int8_moe = pytest.mark.skipif(
DEVICE = current_platform.device_type
def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
return FusedMoEConfig(
+2
View File
@@ -15,6 +15,7 @@ from tests.quantization.utils import load_model_without_vllm_runner
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
CompressedTensorsConfig,
)
@@ -80,6 +81,7 @@ def test_quantized_lm_head_matches_linear(
dtype=torch.bfloat16, head_dtype=None
)
default_vllm_config.kernel_config.linear_backend = "humming"
quant_config: QuantizationConfig
if quant_format is None:
quant_config = ModelOptNvFp4Config(
quant_method=preset,
+1
View File
@@ -843,6 +843,7 @@ def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
}
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
assert isinstance(config, m.ModelOptMixedPrecisionConfig)
assert config.nvfp4_config.quant_method == "NVFP4"
assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
+7 -5
View File
@@ -218,11 +218,13 @@ def test_gptq_nfirst_weights_pass_through_for_triton():
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
method = object.__new__(MoeWNA16Method)
method.experts_cls = object
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
method.wna16_backend = WNA16MoEBackend.HUMMING
method.moe = object()
method.moe = object() # type: ignore[assignment] # Opaque mocked-config sentinel.
quant_config = object()
method.get_fused_moe_quant_config = lambda layer: quant_config
monkeypatch.setattr(
method, "get_fused_moe_quant_config", lambda layer: quant_config
)
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
captured = {}
kernel = object()
@@ -233,7 +235,7 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
method._setup_kernel(layer)
method._setup_kernel(layer) # type: ignore[arg-type] # Routing-table stub.
assert method.moe_kernel is kernel
assert captured["backend"] == WNA16MoEBackend.HUMMING
@@ -278,7 +280,7 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
),
)
assert method.get_fused_moe_quant_config(layer) is quant_config
assert method.get_fused_moe_quant_config(layer) is quant_config # type: ignore[arg-type] # Identity sentinel.
@pytest.mark.skipif(
+15 -14
View File
@@ -16,6 +16,7 @@ from tests.quantization.utils import (
_test_online_quant_peak_mem_impl,
is_quant_method_supported,
load_model_without_vllm_runner,
quant_config_args,
)
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
@@ -180,7 +181,7 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
config = OnlineQuantizationConfig(args)
other_config = OnlineQuantizationConfig(args)
model = QianfanOCRForConditionalGeneration.__new__(
QianfanOCRForConditionalGeneration
QianfanOCRForConditionalGeneration # type: ignore[type-abstract]
)
model._patch_quant_config(
SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),
@@ -209,9 +210,9 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
monkeypatch,
) -> None:
method = object.__new__(Nvfp4OnlineMoEMethod)
method.moe = SimpleNamespace(is_act_and_mul=True)
method.nvfp4_backend = object()
method.experts_cls = object
method.moe = SimpleNamespace(is_act_and_mul=True) # type: ignore[assignment]
method.nvfp4_backend = object() # type: ignore[assignment] # Opaque mocked-backend sentinel.
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
method.moe_quant_config = None
method.moe_kernel = None
@@ -239,7 +240,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
)
make_kernel = Mock(return_value=kernel)
get_quant_config = Mock(return_value=quant_config)
method.get_fused_moe_quant_config = get_quant_config
monkeypatch.setattr(method, "get_fused_moe_quant_config", get_quant_config)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.online.nvfp4."
@@ -355,7 +356,7 @@ def test_online_prequantized_compatibility(
checkpoint_config = checkpoint_config_factory()
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp8")
quant_config_args(linear="mxfp8")
)
config = checkpoint_config
@@ -408,7 +409,7 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
quant_config = _fully_quantized_quark_config()
prefix = "model.layers.0.self_attn.o_proj"
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
quant_config_args(linear="mxfp8", ignore=[prefix])
)
monkeypatch.setattr(
quant_config.online_quantization_config,
@@ -437,7 +438,7 @@ def test_online_quantization_rejects_prequantized_moe(
prefix = "model.layers.0.mlp.experts"
quant_config = _fully_quantized_quark_config()
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
quant_config_args(linear="mxfp4", moe="mxfp4")
)
with pytest.raises(ValueError, match="pre-quantized layer"):
@@ -460,7 +461,7 @@ def test_activation_only_override_applies_to_checkpoint_method(
model_config = ModelConfig(
model=str(tmp_path),
quantization="compressed-tensors",
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
quantization_config=quant_config_args(moe={"activation": "mxfp8"}),
hf_overrides={
"quantization_config": {
"quant_method": "compressed-tensors",
@@ -514,7 +515,7 @@ def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
model_config = ModelConfig(
model=str(tmp_path),
quantization="awq",
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
quantization_config=quant_config_args(linear="mxfp8"),
)
result = weight_utils.get_quant_config(model_config, LoadConfig())
@@ -574,7 +575,7 @@ def test_online_shorthand_selects_checkpoint_or_online_config(
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
"""Composable configs log their nested online quantization results."""
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
online_config = OnlineQuantizationConfig(quant_config_args(linear="mxfp8"))
online_config.quantized_layers = {
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
@@ -1164,7 +1165,7 @@ def test_online_quantization_records_global_config(
default_vllm_config, dist_init
) -> None:
default_vllm_config.model_config = ModelConfig()
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block"))
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_block"))
prefix = "model.layers.0.self_attn.o_proj"
layer = ColumnParallelLinear(
input_size=1,
@@ -1218,7 +1219,7 @@ def test_online_quantization_targets_reject_unsupported_layer() -> None:
def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor"))
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_tensor"))
config.quantized_layers = {
"model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None),
"model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None),
@@ -1431,7 +1432,7 @@ def test_online_int8_moe_w2_scale_matches_unsharded(monkeypatch) -> None:
layer.w2_weight = torch.nn.Parameter(w2_in, requires_grad=False)
layer.num_experts = layer.local_num_experts = w13.shape[0]
method = SimpleNamespace(moe=SimpleNamespace(tp_size=moe_tp_size))
Int8OnlineMoEMethod._quantize_weights(method, layer)
Int8OnlineMoEMethod._quantize_weights(method, layer) # type: ignore[arg-type]
return layer.w2_weight, layer.w2_scale
full_weight, full_scale = quantize(w2, 1)
+2 -1
View File
@@ -10,6 +10,7 @@ against the pure-torch reference in `reference_mxfp4.py`.
import pytest
import torch
from vllm.config.kernel import MoEBackend
from vllm.config.model import ModelConfig
from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
@@ -282,7 +283,7 @@ def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int):
],
)
def test_online_mxfp4_moe_matches_quark(
moe_backend: str,
moe_backend: MoEBackend,
unpadded_hidden_size: int,
unpadded_intermediate_size: int,
default_vllm_config,
@@ -6,6 +6,7 @@ from unittest.mock import Mock
import pytest
from tests.quantization.utils import quant_config_args, quant_spec
from vllm.config.quantization import (
QUANT_KEY_NAMES,
QuantizationConfigArgs,
@@ -30,7 +31,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
def test_quant_spec_resolves_string_to_quant_key():
spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
spec = quant_spec(weight="mxfp8", activation="fp8_per_token")
assert spec.weight == kMxfp8Dynamic
assert spec.activation == kFp8DynamicTokenSym
@@ -42,12 +43,12 @@ def test_quant_spec_accepts_quant_key_directly():
def test_quant_spec_string_representation_uses_quantization_name():
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
QuantSpec(weight="not_a_real_format")
quant_spec(weight="not_a_real_format")
# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
@@ -55,7 +56,7 @@ def test_quant_spec_rejects_unknown_name():
def test_args_linear_string_resolves_via_quant_key_names():
# A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
args = QuantizationConfigArgs(linear="fp8_per_block_static")
args = quant_config_args(linear="fp8_per_block_static")
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe is None
@@ -65,7 +66,7 @@ def test_args_moe_string_resolves_via_online_shorthand():
# (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
# same per-layer-kind spec the `--quantization fp8_per_block` shorthand
# would).
args = QuantizationConfigArgs(moe="fp8_per_block")
args = quant_config_args(moe="fp8_per_block")
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
@@ -73,11 +74,11 @@ def test_args_string_shorthand_missing_slot_raises():
# int8_per_channel_weight_only sets only `moe`; using it on `linear`
# has no defined spec and should raise rather than silently no-op.
with pytest.raises(ValueError, match="does not define a linear spec"):
QuantizationConfigArgs(linear="int8_per_channel_weight_only")
quant_config_args(linear="int8_per_channel_weight_only")
def test_args_accepts_dict_form():
args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
args = quant_config_args(moe={"activation": "mxfp8"})
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
@@ -91,6 +92,7 @@ def test_targets_reject_non_string_keys():
def test_resolve_shorthand_only_populates_both_slots():
args = resolve_quantization_config("fp8_per_block", None)
assert args is not None
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
@@ -105,6 +107,7 @@ def test_resolve_int8_shorthand_leaves_linear_unset():
# int8_per_channel_weight_only is MoE-only; linear stays None so that
# OnlineQuantizationConfig leaves Linear layers in full precision.
args = resolve_quantization_config("int8_per_channel_weight_only", None)
assert args is not None
assert args.linear is None
assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
@@ -113,6 +116,7 @@ def test_resolve_quantization_config_only():
# When only `quantization_config` is given (e.g. for an already-quantized
# checkpoint that needs an activation override), it's returned as-is.
args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
assert args is not None
assert args.linear is None
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
@@ -124,13 +128,14 @@ def test_resolve_merges_explicit_over_shorthand():
"fp8_per_tensor",
{"linear": "fp8_per_block"},
)
assert args is not None
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
def test_resolve_quantization_config_with_checkpoint_quantization():
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
assert args == QuantizationConfigArgs(linear="fp8_per_block")
assert args == quant_config_args(linear="fp8_per_block")
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
@@ -140,15 +145,15 @@ def test_quant_key_names_round_trip():
# Every advertised name should round-trip through QuantSpec without error
# and produce the same QuantKey it maps to.
for name, expected in QUANT_KEY_NAMES.items():
assert QuantSpec(weight=name).weight == expected, name
assert QuantSpec(activation=name).activation == expected, name
assert quant_spec(weight=name).weight == expected, name
assert quant_spec(activation=name).activation == expected, name
def test_static_block_weight_paired_with_dynamic_block_activation():
# The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
# activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
# rewire the kernel dispatch.
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
spec = quant_spec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
assert spec.weight == kFp8Static128BlockSym
assert spec.activation == kFp8Dynamic128Sym
+8 -5
View File
@@ -22,7 +22,7 @@ from packaging import version
from tests.quantization.utils import load_model_without_vllm_runner
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.cache import CacheConfig
from vllm.config.cache import CacheConfig, CacheDType
from vllm.forward_context import set_forward_context
from vllm.model_executor import parameter
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
@@ -1081,6 +1081,7 @@ def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config):
)
wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts)
assert mcls is QuarkOCP_MX_MoEMethod
assert wk is not None
# The OCP MX MoE constructor should fail loudly for MXFP8.
fake_moe_config = MagicMock()
with pytest.raises(ValueError, match="MXFP8 experts are not supported"):
@@ -1321,7 +1322,6 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
scheme = QuarkW8A8Fp8.__new__(QuarkW8A8Fp8)
scheme.weight_qscheme = "per_channel"
scheme.is_static_input_scheme = False
scheme.input_qscheme = "per_channel"
scheme.activation_quant_key = kFp8DynamicTokenSym
scheme.weight_quant_key = kFp8StaticChannelSym
scheme.out_dtype = dtype
@@ -1343,7 +1343,7 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_quark_fp8_w_per_tensor_a_per_tensor(
kv_cache_dtype: str, monkeypatch, dist_init, workspace_init
kv_cache_dtype: CacheDType, monkeypatch, dist_init, workspace_init
):
model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
checkpoint_scales = {}
@@ -2314,7 +2314,7 @@ class TestQuarkInt4Format:
None,
quant_config.quant_config["global_quant_config"]["weight"],
quant_config.pack_method,
moe_config,
moe_config, # type: ignore[arg-type]
)
layer = _FakeLayer(moe_config)
@@ -2325,7 +2325,10 @@ class TestQuarkInt4Format:
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
requires_grad=False,
)
loader = method.get_weight_loader(layer, weight_loader=None)
loader = method.get_weight_loader(
layer, # type: ignore[arg-type]
weight_loader=None,
)
loader(
param,
raw_zp.clone(),
@@ -75,7 +75,7 @@ class CustomQuantConfig(QuantizationConfig):
def get_name(self) -> QuantizationMethods:
"""Name of the quantization method."""
return "custom_quant"
return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names.
def get_supported_act_dtypes(self) -> list[torch.dtype]:
"""List of supported activation dtypes."""
+2
View File
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
if current_platform.is_rocm():
@@ -126,6 +127,7 @@ def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
def load_weights(model):
model_loader = get_model_loader(load_config)
assert isinstance(model_loader, DefaultModelLoader)
weights_iterator = model_loader.get_all_weights(model_config, model)
model.load_weights(weights_iterator)
@@ -34,7 +34,7 @@ def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch):
outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass(
backend=NvFp4MoeBackend.FLASHINFER_CUTLASS,
layer=layer,
layer=layer, # type: ignore[arg-type]
w13=w13,
w13_scale=w13_scale,
w13_scale_2=weight_scale,
+1 -1
View File
@@ -300,7 +300,7 @@ class TestTurboQuantKVCacheSpec:
# The layer builds an unpacked spec; the worker's spec-collection
# loop applies TQ slot packing via the backend's customize_spec hook.
spec = Attention.get_kv_cache_spec(layer, vllm_config)
spec = Attention.get_kv_cache_spec(layer, vllm_config) # type: ignore[arg-type]
assert isinstance(spec, FullAttentionSpec)
assert spec.kv_quant_mode.is_turboquant
assert spec.state_content_bytes is None
+16 -1
View File
@@ -6,18 +6,31 @@ from typing import Any
import regex as re
import torch
from pydantic import TypeAdapter
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
def quant_spec(**kwargs: Any) -> QuantSpec:
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
return TypeAdapter(QuantSpec).validate_python(kwargs)
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
def _limit_num_hidden_layers(
model: torch.nn.Module, num_hidden_layers: int | None
) -> None:
@@ -59,7 +72,9 @@ def load_model_without_vllm_runner(
**(model_config_kwargs or {}),
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0))
vllm_config_args.setdefault(
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
)
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
num_hidden_layers = hf_overrides.get("num_hidden_layers")