[MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939)
Signed-off-by: Taimys <ffy2281693949@gmail.com> Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Co-authored-by: Taimys <ffy2281693949@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
Luyi Xiao
Taimys
Harry Mellor
Claude Opus 5.5
parent
c4fa6f36a7
commit
5225f1bacb
@@ -123,8 +123,8 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
|
||||
method = object.__new__(AutoGPTQMoEMethod)
|
||||
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
|
||||
method.input_dtype = None
|
||||
method.experts_cls = None
|
||||
method.moe = SimpleNamespace(w13_num_shards=2)
|
||||
method.experts_cls = None # type: ignore[assignment] # Weight creation does not select experts.
|
||||
method.moe = SimpleNamespace(w13_num_shards=2) # type: ignore[assignment]
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
@@ -170,7 +170,7 @@ def test_routed_experts_loads_per_expert_biases():
|
||||
("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])),
|
||||
("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])),
|
||||
):
|
||||
assert RoutedExperts.weight_loader(
|
||||
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
|
||||
loader,
|
||||
w13_bias,
|
||||
loaded,
|
||||
@@ -180,7 +180,7 @@ def test_routed_experts_loads_per_expert_biases():
|
||||
return_success=True,
|
||||
)
|
||||
|
||||
assert RoutedExperts.weight_loader(
|
||||
assert RoutedExperts.weight_loader( # type: ignore[call-overload]
|
||||
loader,
|
||||
w2_bias,
|
||||
torch.tensor([9.0, 10.0, 11.0, 12.0]),
|
||||
|
||||
@@ -193,7 +193,7 @@ class DummyFusedMoE:
|
||||
|
||||
|
||||
def make_config(**overrides) -> INCConfig:
|
||||
kwargs = {
|
||||
kwargs: dict[str, Any] = {
|
||||
"weight_bits": 4,
|
||||
"group_size": 128,
|
||||
"sym": True,
|
||||
@@ -208,7 +208,7 @@ def make_config(**overrides) -> INCConfig:
|
||||
|
||||
|
||||
def make_layer_config(**overrides) -> INCLayerConfig:
|
||||
kwargs = {
|
||||
kwargs: dict[str, Any] = {
|
||||
"bits": 4,
|
||||
"group_size": 128,
|
||||
"sym": True,
|
||||
@@ -423,7 +423,7 @@ def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix(
|
||||
|
||||
config = make_config(extra_config={"model.moe_layer": {"bits": 16}})
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = None
|
||||
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
|
||||
|
||||
method = config.get_quant_method(layer, "moe_layer")
|
||||
|
||||
@@ -730,7 +730,7 @@ def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
|
||||
)
|
||||
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
layer_config = make_layer_config(bits=bits)
|
||||
method = INCWna16Scheme().get_moe_method(
|
||||
make_config(), layer, "model.layers.0.mlp", layer_config
|
||||
@@ -861,7 +861,7 @@ def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe(
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
|
||||
method = config.get_quant_method(layer, "model.layers.0.mlp")
|
||||
|
||||
@@ -890,7 +890,7 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
|
||||
ignored_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
|
||||
@@ -1186,7 +1186,7 @@ def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None:
|
||||
)
|
||||
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
layer.moe_config = DummyMoeConfig() # type: ignore[assignment]
|
||||
method = INCWna16Scheme().get_moe_method(
|
||||
make_config(),
|
||||
layer,
|
||||
@@ -1669,7 +1669,7 @@ def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
|
||||
when extra_config has bits >= 16."""
|
||||
config = make_config(extra_config={"layer": {"bits": 16}})
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config
|
||||
layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub.
|
||||
|
||||
class DummyUnquantizedFusedMoEMethod:
|
||||
def __init__(self, moe_config) -> None:
|
||||
@@ -2094,7 +2094,7 @@ def _with_w4a8_kernel(monkeypatch) -> None:
|
||||
|
||||
def _dispatch(layer_config=None):
|
||||
return INCWna16Scheme().get_linear_method(
|
||||
object(), object(), "layer", layer_config or make_layer_config()
|
||||
make_config(), object(), "layer", layer_config or make_layer_config()
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -99,6 +99,7 @@ def test_machete_rejects_invalid_config(
|
||||
)
|
||||
can_impl, reason = MacheteLinearKernel.can_implement(config)
|
||||
assert not can_impl
|
||||
assert reason is not None
|
||||
assert expected_reason in reason
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention.attention import (
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
Fp8LinearMethod,
|
||||
@@ -79,17 +80,22 @@ def test_deepseek_v41_mxfp8_scale_loading(
|
||||
for module in (model_module, linear_module):
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
tp_args = dict(quant_config=quant_config, bias=False)
|
||||
block = torch.nn.Module()
|
||||
block.attn = torch.nn.Module()
|
||||
block.attn.wq_b = ColumnParallelLinear(128, 128, **tp_args)
|
||||
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(128, [128, 64], **tp_args)
|
||||
block.attn.wq_b = ColumnParallelLinear(
|
||||
128, 128, quant_config=quant_config, bias=False
|
||||
)
|
||||
block.attn.fused_wqa_wkv = MergedColumnParallelLinear(
|
||||
128, [128, 64], quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn = torch.nn.Module()
|
||||
block.ffn.shared_experts = torch.nn.Module()
|
||||
block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear(
|
||||
128, [128, 128], **tp_args
|
||||
128, [128, 128], quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn.shared_experts.down_proj = RowParallelLinear(
|
||||
128, 128, quant_config=quant_config, bias=False
|
||||
)
|
||||
block.ffn.shared_experts.down_proj = RowParallelLinear(128, 128, **tp_args)
|
||||
block.ffn.experts = torch.nn.Module()
|
||||
block.ffn.experts.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False)
|
||||
@@ -111,9 +117,11 @@ def test_deepseek_v41_mxfp8_scale_loading(
|
||||
model.config = SimpleNamespace(num_attention_heads=2)
|
||||
model.quant_config = quant_config
|
||||
model.use_sequence_parallel = False
|
||||
model.get_expert_mapping = lambda: [
|
||||
("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")
|
||||
]
|
||||
monkeypatch.setattr(
|
||||
model,
|
||||
"get_expert_mapping",
|
||||
lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")],
|
||||
)
|
||||
|
||||
checkpoints = [
|
||||
("attn.wq_b", "attn.wq_b", 128, 128, 0, None),
|
||||
@@ -211,7 +219,10 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
|
||||
vllm_config = SimpleNamespace(
|
||||
quant_config=SimpleNamespace(weight_block_size=weight_block_size)
|
||||
)
|
||||
resolved = model_module._linear_scale_param_name(vllm_config, expert_dtype)
|
||||
resolved = model_module._linear_scale_param_name(
|
||||
vllm_config, # type: ignore[arg-type]
|
||||
expert_dtype,
|
||||
)
|
||||
assert resolved == scale_name
|
||||
|
||||
mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved)
|
||||
@@ -373,7 +384,7 @@ def test_deepgemm_mxfp8_preserves_weight_and_scale_values(
|
||||
@pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
dist_init, default_vllm_config, prequantized, config_source, num_tokens
|
||||
dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens
|
||||
):
|
||||
"""BMM metadata set after construction selects grouped weight processing."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import (
|
||||
@@ -398,10 +409,11 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell")
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
quant_config = DeepseekV4FP8Config(
|
||||
deepseek_config = DeepseekV4FP8Config(
|
||||
is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32]
|
||||
)
|
||||
quant_config._resolved_expert_dtype = "fp4"
|
||||
deepseek_config._resolved_expert_dtype = "fp4"
|
||||
quant_config: QuantizationConfig = deepseek_config
|
||||
if config_source == "mxfp8":
|
||||
quant_config = get_quantization_config("mxfp8").from_config(
|
||||
{"quant_method": "mxfp8"}
|
||||
@@ -432,7 +444,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
model.config = SimpleNamespace(num_attention_heads=2)
|
||||
model.quant_config = quant_config
|
||||
model.use_sequence_parallel = False
|
||||
model.get_expert_mapping = lambda: []
|
||||
monkeypatch.setattr(model, "get_expert_mapping", lambda: [])
|
||||
model.load_weights(
|
||||
[
|
||||
("layers.0.attn.wo_a.weight", weight),
|
||||
@@ -443,6 +455,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights(
|
||||
weight.float().reshape(8, 32, 16, 32)
|
||||
* torch.exp2(scales.float() - 127)[:, None, :, None]
|
||||
).reshape(2, 128, 512)
|
||||
assert isinstance(linear.quant_method, ModelOptLinearMethod)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
linear.quant_method.process_weights_after_loading(linear)
|
||||
assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel)
|
||||
@@ -785,7 +798,7 @@ def test_kv_cache_scale_sync_to_host_copies():
|
||||
set_default_quant_scales(layer, register_buffer=True)
|
||||
layer.kv_cache_dtype = "fp8"
|
||||
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method = BaseKVCacheMethod(quant_config=None) # type: ignore[arg-type] # Scale synchronization does not use quant_config.
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
|
||||
@@ -8,6 +8,7 @@ These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -53,7 +54,7 @@ def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch):
|
||||
set_rocm_aiter(monkeypatch, False)
|
||||
|
||||
|
||||
def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
|
||||
@@ -10,6 +10,7 @@ CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -44,7 +45,7 @@ requires_int8_moe = pytest.mark.skipif(
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig:
|
||||
def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return FusedMoEConfig(
|
||||
|
||||
@@ -15,6 +15,7 @@ from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
@@ -80,6 +81,7 @@ def test_quantized_lm_head_matches_linear(
|
||||
dtype=torch.bfloat16, head_dtype=None
|
||||
)
|
||||
default_vllm_config.kernel_config.linear_backend = "humming"
|
||||
quant_config: QuantizationConfig
|
||||
if quant_format is None:
|
||||
quant_config = ModelOptNvFp4Config(
|
||||
quant_method=preset,
|
||||
|
||||
@@ -843,6 +843,7 @@ def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
|
||||
}
|
||||
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
|
||||
|
||||
assert isinstance(config, m.ModelOptMixedPrecisionConfig)
|
||||
assert config.nvfp4_config.quant_method == "NVFP4"
|
||||
assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
|
||||
|
||||
|
||||
@@ -218,11 +218,13 @@ def test_gptq_nfirst_weights_pass_through_for_triton():
|
||||
|
||||
def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
method = object.__new__(MoeWNA16Method)
|
||||
method.experts_cls = object
|
||||
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
|
||||
method.wna16_backend = WNA16MoEBackend.HUMMING
|
||||
method.moe = object()
|
||||
method.moe = object() # type: ignore[assignment] # Opaque mocked-config sentinel.
|
||||
quant_config = object()
|
||||
method.get_fused_moe_quant_config = lambda layer: quant_config
|
||||
monkeypatch.setattr(
|
||||
method, "get_fused_moe_quant_config", lambda layer: quant_config
|
||||
)
|
||||
layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None))
|
||||
captured = {}
|
||||
kernel = object()
|
||||
@@ -233,7 +235,7 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel)
|
||||
|
||||
method._setup_kernel(layer)
|
||||
method._setup_kernel(layer) # type: ignore[arg-type] # Routing-table stub.
|
||||
|
||||
assert method.moe_kernel is kernel
|
||||
assert captured["backend"] == WNA16MoEBackend.HUMMING
|
||||
@@ -278,7 +280,7 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
|
||||
),
|
||||
)
|
||||
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config # type: ignore[arg-type] # Identity sentinel.
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -16,6 +16,7 @@ from tests.quantization.utils import (
|
||||
_test_online_quant_peak_mem_impl,
|
||||
is_quant_method_supported,
|
||||
load_model_without_vllm_runner,
|
||||
quant_config_args,
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
@@ -180,7 +181,7 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized(
|
||||
config = OnlineQuantizationConfig(args)
|
||||
other_config = OnlineQuantizationConfig(args)
|
||||
model = QianfanOCRForConditionalGeneration.__new__(
|
||||
QianfanOCRForConditionalGeneration
|
||||
QianfanOCRForConditionalGeneration # type: ignore[type-abstract]
|
||||
)
|
||||
model._patch_quant_config(
|
||||
SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),
|
||||
@@ -209,9 +210,9 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
method = object.__new__(Nvfp4OnlineMoEMethod)
|
||||
method.moe = SimpleNamespace(is_act_and_mul=True)
|
||||
method.nvfp4_backend = object()
|
||||
method.experts_cls = object
|
||||
method.moe = SimpleNamespace(is_act_and_mul=True) # type: ignore[assignment]
|
||||
method.nvfp4_backend = object() # type: ignore[assignment] # Opaque mocked-backend sentinel.
|
||||
method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel.
|
||||
method.moe_quant_config = None
|
||||
method.moe_kernel = None
|
||||
|
||||
@@ -239,7 +240,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
|
||||
)
|
||||
make_kernel = Mock(return_value=kernel)
|
||||
get_quant_config = Mock(return_value=quant_config)
|
||||
method.get_fused_moe_quant_config = get_quant_config
|
||||
monkeypatch.setattr(method, "get_fused_moe_quant_config", get_quant_config)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.online.nvfp4."
|
||||
@@ -355,7 +356,7 @@ def test_online_prequantized_compatibility(
|
||||
checkpoint_config = checkpoint_config_factory()
|
||||
|
||||
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8")
|
||||
quant_config_args(linear="mxfp8")
|
||||
)
|
||||
config = checkpoint_config
|
||||
|
||||
@@ -408,7 +409,7 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp8", ignore=[prefix])
|
||||
quant_config_args(linear="mxfp8", ignore=[prefix])
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
quant_config.online_quantization_config,
|
||||
@@ -437,7 +438,7 @@ def test_online_quantization_rejects_prequantized_moe(
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(linear="mxfp4", moe="mxfp4")
|
||||
quant_config_args(linear="mxfp4", moe="mxfp4")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
@@ -460,7 +461,7 @@ def test_activation_only_override_applies_to_checkpoint_method(
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="compressed-tensors",
|
||||
quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}),
|
||||
quantization_config=quant_config_args(moe={"activation": "mxfp8"}),
|
||||
hf_overrides={
|
||||
"quantization_config": {
|
||||
"quant_method": "compressed-tensors",
|
||||
@@ -514,7 +515,7 @@ def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None:
|
||||
model_config = ModelConfig(
|
||||
model=str(tmp_path),
|
||||
quantization="awq",
|
||||
quantization_config=QuantizationConfigArgs(linear="mxfp8"),
|
||||
quantization_config=quant_config_args(linear="mxfp8"),
|
||||
)
|
||||
|
||||
result = weight_utils.get_quant_config(model_config, LoadConfig())
|
||||
@@ -574,7 +575,7 @@ def test_online_shorthand_selects_checkpoint_or_online_config(
|
||||
|
||||
def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
|
||||
"""Composable configs log their nested online quantization results."""
|
||||
online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
|
||||
online_config = OnlineQuantizationConfig(quant_config_args(linear="mxfp8"))
|
||||
online_config.quantized_layers = {
|
||||
"model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
"model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None),
|
||||
@@ -1164,7 +1165,7 @@ def test_online_quantization_records_global_config(
|
||||
default_vllm_config, dist_init
|
||||
) -> None:
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block"))
|
||||
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_block"))
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=1,
|
||||
@@ -1218,7 +1219,7 @@ def test_online_quantization_targets_reject_unsupported_layer() -> None:
|
||||
|
||||
|
||||
def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
|
||||
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor"))
|
||||
config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_tensor"))
|
||||
config.quantized_layers = {
|
||||
"model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None),
|
||||
"model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None),
|
||||
@@ -1431,7 +1432,7 @@ def test_online_int8_moe_w2_scale_matches_unsharded(monkeypatch) -> None:
|
||||
layer.w2_weight = torch.nn.Parameter(w2_in, requires_grad=False)
|
||||
layer.num_experts = layer.local_num_experts = w13.shape[0]
|
||||
method = SimpleNamespace(moe=SimpleNamespace(tp_size=moe_tp_size))
|
||||
Int8OnlineMoEMethod._quantize_weights(method, layer)
|
||||
Int8OnlineMoEMethod._quantize_weights(method, layer) # type: ignore[arg-type]
|
||||
return layer.w2_weight, layer.w2_scale
|
||||
|
||||
full_weight, full_scale = quantize(w2, 1)
|
||||
|
||||
@@ -10,6 +10,7 @@ against the pure-torch reference in `reference_mxfp4.py`.
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
@@ -282,7 +283,7 @@ def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int):
|
||||
],
|
||||
)
|
||||
def test_online_mxfp4_moe_matches_quark(
|
||||
moe_backend: str,
|
||||
moe_backend: MoEBackend,
|
||||
unpadded_hidden_size: int,
|
||||
unpadded_intermediate_size: int,
|
||||
default_vllm_config,
|
||||
|
||||
@@ -6,6 +6,7 @@ from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import quant_config_args, quant_spec
|
||||
from vllm.config.quantization import (
|
||||
QUANT_KEY_NAMES,
|
||||
QuantizationConfigArgs,
|
||||
@@ -30,7 +31,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
|
||||
|
||||
def test_quant_spec_resolves_string_to_quant_key():
|
||||
spec = QuantSpec(weight="mxfp8", activation="fp8_per_token")
|
||||
spec = quant_spec(weight="mxfp8", activation="fp8_per_token")
|
||||
assert spec.weight == kMxfp8Dynamic
|
||||
assert spec.activation == kFp8DynamicTokenSym
|
||||
|
||||
@@ -42,12 +43,12 @@ def test_quant_spec_accepts_quant_key_directly():
|
||||
|
||||
|
||||
def test_quant_spec_string_representation_uses_quantization_name():
|
||||
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
|
||||
assert str(quant_spec(weight="mxfp4")) == "mxfp4"
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
QuantSpec(weight="not_a_real_format")
|
||||
quant_spec(weight="not_a_real_format")
|
||||
|
||||
|
||||
# ---- QuantizationConfigArgs string shorthand on linear/moe ----------------
|
||||
@@ -55,7 +56,7 @@ def test_quant_spec_rejects_unknown_name():
|
||||
|
||||
def test_args_linear_string_resolves_via_quant_key_names():
|
||||
# A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=<key>).
|
||||
args = QuantizationConfigArgs(linear="fp8_per_block_static")
|
||||
args = quant_config_args(linear="fp8_per_block_static")
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe is None
|
||||
|
||||
@@ -65,7 +66,7 @@ def test_args_moe_string_resolves_via_online_shorthand():
|
||||
# (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the
|
||||
# same per-layer-kind spec the `--quantization fp8_per_block` shorthand
|
||||
# would).
|
||||
args = QuantizationConfigArgs(moe="fp8_per_block")
|
||||
args = quant_config_args(moe="fp8_per_block")
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
|
||||
@@ -73,11 +74,11 @@ def test_args_string_shorthand_missing_slot_raises():
|
||||
# int8_per_channel_weight_only sets only `moe`; using it on `linear`
|
||||
# has no defined spec and should raise rather than silently no-op.
|
||||
with pytest.raises(ValueError, match="does not define a linear spec"):
|
||||
QuantizationConfigArgs(linear="int8_per_channel_weight_only")
|
||||
quant_config_args(linear="int8_per_channel_weight_only")
|
||||
|
||||
|
||||
def test_args_accepts_dict_form():
|
||||
args = QuantizationConfigArgs(moe={"activation": "mxfp8"})
|
||||
args = quant_config_args(moe={"activation": "mxfp8"})
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
|
||||
@@ -91,6 +92,7 @@ def test_targets_reject_non_string_keys():
|
||||
|
||||
def test_resolve_shorthand_only_populates_both_slots():
|
||||
args = resolve_quantization_config("fp8_per_block", None)
|
||||
assert args is not None
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
|
||||
@@ -105,6 +107,7 @@ def test_resolve_int8_shorthand_leaves_linear_unset():
|
||||
# int8_per_channel_weight_only is MoE-only; linear stays None so that
|
||||
# OnlineQuantizationConfig leaves Linear layers in full precision.
|
||||
args = resolve_quantization_config("int8_per_channel_weight_only", None)
|
||||
assert args is not None
|
||||
assert args.linear is None
|
||||
assert args.moe == QuantSpec(weight=kInt8StaticChannelSym)
|
||||
|
||||
@@ -113,6 +116,7 @@ def test_resolve_quantization_config_only():
|
||||
# When only `quantization_config` is given (e.g. for an already-quantized
|
||||
# checkpoint that needs an activation override), it's returned as-is.
|
||||
args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}})
|
||||
assert args is not None
|
||||
assert args.linear is None
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
@@ -124,13 +128,14 @@ def test_resolve_merges_explicit_over_shorthand():
|
||||
"fp8_per_tensor",
|
||||
{"linear": "fp8_per_block"},
|
||||
)
|
||||
assert args is not None
|
||||
assert args.linear == QuantSpec(weight=kFp8Static128BlockSym)
|
||||
assert args.moe == QuantSpec(weight=kFp8StaticTensorSym)
|
||||
|
||||
|
||||
def test_resolve_quantization_config_with_checkpoint_quantization():
|
||||
args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"})
|
||||
assert args == QuantizationConfigArgs(linear="fp8_per_block")
|
||||
assert args == quant_config_args(linear="fp8_per_block")
|
||||
|
||||
|
||||
# ---- QUANT_KEY_NAMES coverage --------------------------------------------
|
||||
@@ -140,15 +145,15 @@ def test_quant_key_names_round_trip():
|
||||
# Every advertised name should round-trip through QuantSpec without error
|
||||
# and produce the same QuantKey it maps to.
|
||||
for name, expected in QUANT_KEY_NAMES.items():
|
||||
assert QuantSpec(weight=name).weight == expected, name
|
||||
assert QuantSpec(activation=name).activation == expected, name
|
||||
assert quant_spec(weight=name).weight == expected, name
|
||||
assert quant_spec(activation=name).activation == expected, name
|
||||
|
||||
|
||||
def test_static_block_weight_paired_with_dynamic_block_activation():
|
||||
# The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic
|
||||
# activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly
|
||||
# rewire the kernel dispatch.
|
||||
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
spec = quant_spec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
assert spec.weight == kFp8Static128BlockSym
|
||||
assert spec.activation == kFp8Dynamic128Sym
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ from packaging import version
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.cache import CacheConfig, CacheDType
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
|
||||
@@ -1081,6 +1081,7 @@ def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config):
|
||||
)
|
||||
wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts)
|
||||
assert mcls is QuarkOCP_MX_MoEMethod
|
||||
assert wk is not None
|
||||
# The OCP MX MoE constructor should fail loudly for MXFP8.
|
||||
fake_moe_config = MagicMock()
|
||||
with pytest.raises(ValueError, match="MXFP8 experts are not supported"):
|
||||
@@ -1321,7 +1322,6 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
|
||||
scheme = QuarkW8A8Fp8.__new__(QuarkW8A8Fp8)
|
||||
scheme.weight_qscheme = "per_channel"
|
||||
scheme.is_static_input_scheme = False
|
||||
scheme.input_qscheme = "per_channel"
|
||||
scheme.activation_quant_key = kFp8DynamicTokenSym
|
||||
scheme.weight_quant_key = kFp8StaticChannelSym
|
||||
scheme.out_dtype = dtype
|
||||
@@ -1343,7 +1343,7 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_quark_fp8_w_per_tensor_a_per_tensor(
|
||||
kv_cache_dtype: str, monkeypatch, dist_init, workspace_init
|
||||
kv_cache_dtype: CacheDType, monkeypatch, dist_init, workspace_init
|
||||
):
|
||||
model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
|
||||
checkpoint_scales = {}
|
||||
@@ -2314,7 +2314,7 @@ class TestQuarkInt4Format:
|
||||
None,
|
||||
quant_config.quant_config["global_quant_config"]["weight"],
|
||||
quant_config.pack_method,
|
||||
moe_config,
|
||||
moe_config, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
layer = _FakeLayer(moe_config)
|
||||
@@ -2325,7 +2325,10 @@ class TestQuarkInt4Format:
|
||||
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
loader = method.get_weight_loader(layer, weight_loader=None)
|
||||
loader = method.get_weight_loader(
|
||||
layer, # type: ignore[arg-type]
|
||||
weight_loader=None,
|
||||
)
|
||||
loader(
|
||||
param,
|
||||
raw_zp.clone(),
|
||||
|
||||
@@ -75,7 +75,7 @@ class CustomQuantConfig(QuantizationConfig):
|
||||
|
||||
def get_name(self) -> QuantizationMethods:
|
||||
"""Name of the quantization method."""
|
||||
return "custom_quant"
|
||||
return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names.
|
||||
|
||||
def get_supported_act_dtypes(self) -> list[torch.dtype]:
|
||||
"""List of supported activation dtypes."""
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
@@ -126,6 +127,7 @@ def test_online_quant_config_dict_json(vllm_runner, enable_pickle):
|
||||
|
||||
def load_weights(model):
|
||||
model_loader = get_model_loader(load_config)
|
||||
assert isinstance(model_loader, DefaultModelLoader)
|
||||
weights_iterator = model_loader.get_all_weights(model_config, model)
|
||||
model.load_weights(weights_iterator)
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch):
|
||||
|
||||
outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
backend=NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
layer=layer,
|
||||
layer=layer, # type: ignore[arg-type]
|
||||
w13=w13,
|
||||
w13_scale=w13_scale,
|
||||
w13_scale_2=weight_scale,
|
||||
|
||||
@@ -300,7 +300,7 @@ class TestTurboQuantKVCacheSpec:
|
||||
|
||||
# The layer builds an unpacked spec; the worker's spec-collection
|
||||
# loop applies TQ slot packing via the backend's customize_spec hook.
|
||||
spec = Attention.get_kv_cache_spec(layer, vllm_config)
|
||||
spec = Attention.get_kv_cache_spec(layer, vllm_config) # type: ignore[arg-type]
|
||||
assert isinstance(spec, FullAttentionSpec)
|
||||
assert spec.kv_quant_mode.is_turboquant
|
||||
assert spec.state_content_bytes is None
|
||||
|
||||
@@ -6,18 +6,31 @@ from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.model_executor.layers.quantization import get_quantization_config
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def quant_spec(**kwargs: Any) -> QuantSpec:
|
||||
"""Build a `QuantSpec` from user-facing names such as `"mxfp8"`."""
|
||||
return TypeAdapter(QuantSpec).validate_python(kwargs)
|
||||
|
||||
|
||||
def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs:
|
||||
"""Build `QuantizationConfigArgs` from user-facing shorthand values."""
|
||||
return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs)
|
||||
|
||||
|
||||
def _limit_num_hidden_layers(
|
||||
model: torch.nn.Module, num_hidden_layers: int | None
|
||||
) -> None:
|
||||
@@ -59,7 +72,9 @@ def load_model_without_vllm_runner(
|
||||
**(model_config_kwargs or {}),
|
||||
)
|
||||
vllm_config_args = dict(vllm_config_kwargs or {})
|
||||
vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0))
|
||||
vllm_config_args.setdefault(
|
||||
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
|
||||
)
|
||||
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
|
||||
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
|
||||
num_hidden_layers = hf_overrides.get("num_hidden_layers")
|
||||
|
||||
Reference in New Issue
Block a user