diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 78f342a1d7..e391d516da 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -228,6 +228,85 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales( ] +def test_deepseek_v41_vl_exposes_quant_mappings_on_class(): + """``configure_quant_config`` reads both mappings off the class, before + ``__init__`` builds the instance mapper. Without them a Quark config's + per-layer keys never match this wrapper's ``language_model.``-rooted + prefixes and every attention shard falls back to the global spec.""" + from vllm.models.deepseek_v41.amd.vl_model import DeepseekV41ForCausalLM + + assert "fused_wqa_wkv" in DeepseekV41ForCausalLM.packed_modules_mapping + mapper = DeepseekV41ForCausalLM.hf_to_vllm_mapper.get_rename_mapper() + assert mapper.apply_list(["layers.0.attn.wq_a"]) == [ + "language_model.model.layers.0.attn.wq_a" + ] + + +def test_deepseek_v41_engram_scale_accepts_quark_name(): + """Quark exports name the engram scale ``embed.weight_scale``; the + ``\\.scale$`` rules only match a literal ``.scale`` suffix, so without an + explicit rule the tensor is never routed and loading fails.""" + from vllm.models.deepseek_v41.amd import model as model_module + + mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale") + weight = torch.empty(0) + mapped = [ + name + for name, _ in mapper.apply( + [ + ("layers.1.engram.embed.weight_scale", weight), + ("layers.1.engram.embed.scale", weight), + ] + ) + ] + assert mapped == [ + "model.layers.1.engram.embed_tokens.weight_scale_inv", + "model.layers.1.engram.embed_tokens.weight_scale_inv", + ] + + +def test_deepseek_v41_declines_quark_configs(): + """``from_config`` rewrites a Quark config into a single global FP8 scheme, + which would discard the per-layer specs of a mixed-precision export (MXFP4 + experts + 2-D block MXFP8 attention in DeepSeek-V4.1-Flash). QuarkConfig + knows how to dispatch each scheme, so it must handle every Quark export.""" + from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config + + hf_config = SimpleNamespace(model_type="deepseek_v41") + mxfp4_global = { + "global_quant_config": { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32} + }, + "quant_method": "quark", + } + + assert ( + DeepseekV4FP8Config.override_quantization_method( + mxfp4_global, None, hf_config=hf_config + ) + is None + ) + + mixed = { + **mxfp4_global, + "layer_quant_config": { + "layers.0.attn.wkv": { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [32, 32], + } + } + }, + } + assert ( + DeepseekV4FP8Config.override_quantization_method( + mixed, None, hf_config=hf_config + ) + is None + ) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA") @pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu]) @pytest.mark.parametrize( diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index f2f8ed6bd6..54d3127af0 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -909,6 +909,184 @@ def test_quant_method_dispatch_target(case): assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls) +def test_quant_method_dispatch_mxfp8_2d_block(default_vllm_config): + """Shape B (DeepSeek-V4.1): 32x32 per-block e8m0 routes to QuarkOCP_MX. + + Verifies that get_quant_method and get_quant_method_target agree (the + short-circuit that previously caused them to diverge has been removed). + """ + from vllm.model_executor.layers.quantization.quark.schemes import QuarkOCP_MX + from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Static + + default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) + mxfp8_spec = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [32, 32], + "symmetric": True, + "is_dynamic": False, + "scale_type": "float8_e8m0fnu", + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "symmetric": True, + "is_dynamic": True, + }, + } + config = QuarkConfig( + { + "global_quant_config": { + "weight": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + }, + "layer_type_quant_config": {}, + "layer_quant_config": {"layers.0.attn.wkv": mxfp8_spec}, + "exclude": [], + } + ) + + class TestLinear(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + # get_quant_method_target now routes MXFP8 through the matcher chain. + wk, ak, mcls = config.get_quant_method_target("layers.0.attn.wkv", LinearBase) + assert wk == kMxfp8Static + assert mcls is QuarkLinearMethod + + linear = TestLinear() + method = config.get_quant_method(linear, "layers.0.attn.wkv") + assert isinstance(method, QuarkLinearMethod) + assert isinstance(linear.scheme, QuarkOCP_MX) + assert linear.scheme.weight_quant_key == kMxfp8Static + assert linear.scheme.scale_block_rows == 32 + + # Experts still fall through to the global MXFP4 spec. + assert ( + config.get_quant_method_target("layers.0.ffn.experts", RoutedExperts)[2] + is not QuarkLinearMethod + ) + + # 128x128 per-block FP8 (DeepSeek V4) keeps its existing Quark scheme. + v4_spec = { + "weight": {**mxfp8_spec["weight"], "block_size": [128, 128]}, + "input_tensors": {**mxfp8_spec["input_tensors"], "group_size": 128}, + } + v4_config = _make_qtensor_config(v4_spec["weight"], v4_spec["input_tensors"]) + assert v4_config.get_quant_method_target("linear", LinearBase)[2] is ( + QuarkLinearMethod + ) + linear = TestLinear() + assert isinstance(v4_config.get_quant_method(linear, "linear"), QuarkLinearMethod) + assert isinstance(linear.scheme, QuarkW8A8Fp8PerBlock) + + +def test_quant_method_dispatch_mxfp8_canonical(default_vllm_config): + """Shape A (canonical MX): per_group/group_size=32/e8m0 FP8 routes + to QuarkOCP_MX with scale_block_rows == 1. + + This covers the canonical MXFP8 spelling (no 2-D block). Validated by + synthetic dispatch test only — no canonical-MXFP8 checkpoint on hand. + """ + from vllm.model_executor.layers.quantization.quark.schemes import QuarkOCP_MX + from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Static + + default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) + mxfp8_canonical_spec = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + } + config = QuarkConfig( + { + "global_quant_config": mxfp8_canonical_spec, + "layer_type_quant_config": {}, + "layer_quant_config": {}, + "exclude": [], + } + ) + + wk, ak, mcls = config.get_quant_method_target("linear", LinearBase) + assert wk == kMxfp8Static + assert mcls is QuarkLinearMethod + + class TestLinear(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + linear = TestLinear() + method = config.get_quant_method(linear, "linear") + assert isinstance(method, QuarkLinearMethod) + assert isinstance(linear.scheme, QuarkOCP_MX) + assert linear.scheme.weight_quant_key == kMxfp8Static + assert linear.scheme.scale_block_rows == 1 + + +def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config): + """MXFP8 in a MoE config raises ValueError — experts are unsupported.""" + from vllm.model_executor.layers.quantization.quark.quark_moe import ( + QuarkOCP_MX_MoEMethod, + ) + + default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) + mxfp8_spec = { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + "input_tensors": { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + } + config = QuarkConfig( + { + "global_quant_config": mxfp8_spec, + "layer_type_quant_config": {}, + "layer_quant_config": {}, + "exclude": [], + } + ) + wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts) + assert mcls is QuarkOCP_MX_MoEMethod + # The OCP MX MoE constructor should fail loudly for MXFP8. + fake_moe_config = MagicMock() + with pytest.raises(ValueError, match="MXFP8 experts are not supported"): + QuarkOCP_MX_MoEMethod(fake_moe_config, wk, ak) + + @pytest.mark.parametrize( ("weight", "input_tensors"), [ @@ -2313,3 +2491,107 @@ def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric) ) assert torch.equal(actual, expected) + + +# --------------------------------------------------------------------------- +# override_quantization_method: Quark always delegates to QuarkConfig +# --------------------------------------------------------------------------- + +_QUARK_MXFP4_CFG = { + "quant_method": "quark", + "global_quant_config": { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}, + }, +} + +_QUARK_MXFP8_CFG = { + "quant_method": "quark", + "global_quant_config": { + "weight": { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "block_size": [32, 32], + "scale_type": "float8_e8m0fnu", + "symmetric": True, + }, + }, +} + +_QUARK_MIXED_CFG = { + "quant_method": "quark", + "global_quant_config": { + "weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}, + }, + "layer_quant_config": {"layers.0.attn": {"weight": {"dtype": "fp8_e4m3"}}}, +} + +_FP8_CFG = {"quant_method": "fp8"} + +_DSV4_FP8_CFG = {"quant_method": "deepseek_v4_fp8"} + + +@pytest.mark.parametrize( + "quant_config_cls", + [ + "deepseek_v41", + "deepseek_v4", + ], +) +@pytest.mark.parametrize( + "hf_quant_cfg, expect_none", + [ + (_QUARK_MXFP4_CFG, True), + (_QUARK_MXFP8_CFG, True), + (_QUARK_MIXED_CFG, True), + (_FP8_CFG, False), + (_DSV4_FP8_CFG, False), + ], + ids=[ + "quark-mxfp4", + "quark-mxfp8", + "quark-mixed", + "native-fp8", + "deepseek-v4-fp8", + ], +) +def test_quark_override_delegates_to_quark_config( + quant_config_cls, hf_quant_cfg, expect_none +): + """Any quant_method=='quark' must return None (delegate to QuarkConfig). + + Native FP8 configs should still be claimed by the model-specific config. + """ + import importlib + + if quant_config_cls == "deepseek_v41": + # Module was renamed from deepseek_v4_1 to deepseek_v41 upstream. + for mod_name in ( + "vllm.models.deepseek_v41.quant_config", + "vllm.models.deepseek_v4_1.quant_config", + ): + try: + mod = importlib.import_module(mod_name) + break + except ModuleNotFoundError: + continue + else: + pytest.skip("deepseek_v41 quant_config not found") + model_type = "deepseek_v41" + else: + mod = importlib.import_module("vllm.models.deepseek_v4.quant_config") + model_type = "deepseek_v4" + + config_cls = mod.DeepseekV4FP8Config + hf_config = SimpleNamespace(model_type=model_type) + result = config_cls.override_quantization_method( + hf_quant_cfg, None, hf_config=hf_config + ) + + if expect_none: + assert result is None, ( + f"Quark config should delegate to QuarkConfig (return None), got {result!r}" + ) + else: + assert result == "deepseek_v4_fp8", ( + f"Native FP8 should be claimed by DeepseekV4FP8Config, got {result!r}" + ) diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 2be85d8176..2951f578b0 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -45,6 +45,8 @@ from vllm.model_executor.layers.quantization.quark.utils import ( from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( _ACTIVATION_QUANT_KEY_MAP, _WEIGHT_QUANT_KEY_MAP, + OCP_MX_BLOCK_SIZE, + ocp_mx_weight_dtype_and_rows, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -158,6 +160,20 @@ class QuarkConfig(QuantizationConfig): self.quant_config = quant_config_with_hf_to_vllm_mapper + def _scale_block_rows(self, prefix: str, layer_type: type[torch.nn.Module]) -> int: + """How many weight rows one checkpoint scale row covers. + + 1 for the canonical 1-D per-group MX spelling; ``block_size[0]`` for + MXFP8 exports that spell the scale 2-D (DeepSeek-V4.1 uses 32). The + scheme cannot work this out itself: it has no handle on the layer name, + and tensor-parallel sharding narrows the loaded scale so the ratio is + not recoverable from shapes either. + """ + config = self._find_matched_config(prefix, layer_type) + weight = self._unwrap_single_quant_config(config.get("weight")) + normalized = ocp_mx_weight_dtype_and_rows(weight) + return normalized[1] if normalized is not None else 1 + def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> "QuantizeMethodBase | None": @@ -175,6 +191,11 @@ class QuarkConfig(QuantizationConfig): scheme_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, + scale_block_rows=( + self._scale_block_rows(prefix, type(layer)) + if scheme_cls is QuarkOCP_MX + else 1 + ), weight_config=self._find_matched_config(prefix, type(layer)).get( "weight" ), @@ -640,52 +661,18 @@ class QuarkConfig(QuantizationConfig): The rationale for checking only the weight type is that the model loading concept and process primarily concerns the weights themselves. """ - # Confirm weights quantized. - if not isinstance(weight_quant, dict): + # Confirm weights quantized, and normalize the two OCP MX config + # dialects (1-D per_group, and MXFP8's 2-D per_block) to a dtype. + normalized = ocp_mx_weight_dtype_and_rows(weight_quant) + if normalized is None: logger.debug( - "Quark model's weight quantization is incompatible with OCP_MX format: " - "weight_quant is not a dictionary." + "Quark model's weight quantization is incompatible with OCP MX " + "format: %s", + weight_quant, ) return QuantKeyMatch(False, None, None) - # Input and weight qscheme needs to be per group. - if weight_quant.get("qscheme") != "per_group": - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "weight is not per_group." - ) - return QuantKeyMatch(False, None, None) - - # Input and weight group size needs to be 32. - if weight_quant.get("group_size") != 32: - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "group_size of weight is not 32." - ) - return QuantKeyMatch(False, None, None) - - # Activations and weight scales need to be in e8m0 format. - if weight_quant.get("scale_format") != "e8m0": - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "scale_format of weight is not e8m0." - ) - return QuantKeyMatch(False, None, None) - - # Input and weight dtypes need to be any of fp4, - # fp6_e3m2 or fp6_e3m2, possibly mixed. - if weight_quant.get("dtype") not in { - "fp4", - "fp6_e3m2", - "fp6_e2m3", - }: - logger.debug( - "Quark model's weight quantization is incompatible with OCP MX format: " - "dtype is not in {fp4, fp6_e3m2, fp6_e2m3}." - ) - return QuantKeyMatch(False, None, None) - - weight_dtype = weight_quant["dtype"].replace("fp", "mxfp") + weight_dtype, _ = normalized weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype] if input_quant is None: activation_quant_key = None @@ -703,6 +690,14 @@ class QuarkConfig(QuantizationConfig): "input scales." ) return QuantKeyMatch(False, None, None) + elif ( + input_quant["dtype"] == "fp8_e4m3" + and input_quant.get("qscheme") == "per_group" + and input_quant.get("group_size") == OCP_MX_BLOCK_SIZE + ): + # MXFP8 activations. Must be checked before the per-tensor FP8 + # branch below, which would otherwise claim them. + activation_quant_key = _ACTIVATION_QUANT_KEY_MAP["mxfp8_e4m3"] elif input_quant["dtype"] == "fp8_e4m3": activation_quant_key = kFp8DynamicTensorSym else: @@ -903,6 +898,7 @@ class QuarkConfig(QuantizationConfig): scheme_cls: type["QuarkScheme"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, + scale_block_rows: int = 1, weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" @@ -942,6 +938,9 @@ class QuarkConfig(QuantizationConfig): ): kwargs["weight_quant_key"] = weight_quant_key + if scheme_cls is QuarkOCP_MX: + kwargs["scale_block_rows"] = scale_block_rows + scheme = scheme_cls(**kwargs) # Raise error if device does not support the scheme diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 2e7fc12484..8da0aba97c 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1587,7 +1587,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): raise ValueError( f"Unsupported OCP MX dtype combination for MoE: " f"input_dtype={self.input_dtype}, weight_dtype={self.weight_dtype}. " - f"Please check that the combination is supported in OCP_MX_Scheme." + f"MXFP8 experts are not supported through the MoE path; only " + f"the linear path handles MXFP8. Supported MoE weight dtypes " + f"are mxfp4, mxfp6_e3m2, and mxfp6_e2m3." ) # TODO(bowenbao): refactor and introduce backends for other OCP MX schemes, @@ -1676,10 +1678,16 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): if quant_dtype == "mxfp4": assert dim % 2 == 0 return dim // 2 - else: + elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}: # FP6 packs 4 * 6 = 24 bits on 3 bytes. assert (dim * 3) % 4 == 0 return (dim * 3) // 4 + else: + raise NotImplementedError( + f"Unsupported quant_dtype in QuarkOCP_MX_MoEMethod." + f"get_packed_dim: {quant_dtype}. MXFP8 is not supported for " + f"MoE experts." + ) def create_weights( self, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index 50263863d8..d12dd62e01 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -10,22 +10,31 @@ from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( MxFp4LinearKernel, MxFp6LinearKernel, + Mxfp8LinearKernel, init_mxfp4_linear_kernel, init_mxfp6_linear_kernel, + init_mxfp8_linear_kernel, +) +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_SCALE_DTYPE, + MXFP8_VALUE_DTYPE, ) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( _ACTIVATION_QUANT_KEY_MAP, _WEIGHT_QUANT_KEY_MAP, OCP_MX_BLOCK_SIZE, + OCP_MX_UNPACKED_DTYPES, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kMxfp4Static, kMxfp6E2M3Static, kMxfp6E3M2Static, + kMxfp8Static, ) from vllm.model_executor.parameter import ( GroupQuantScaleParameter, + ModelWeightParameter, PackedvLLMParameter, ) from vllm.platforms import current_platform @@ -36,7 +45,7 @@ logger = init_logger(__name__) class QuarkOCP_MX(QuarkScheme): - ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel + ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None] supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()] @@ -44,9 +53,10 @@ class QuarkOCP_MX(QuarkScheme): self, weight_quant_key: QuantKey, activation_quant_key: QuantKey | None, + scale_block_rows: int = 1, ): super().__init__(weight_quant_key, activation_quant_key) - + self.scale_block_rows = scale_block_rows self.weight_dtype = next( dtype for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items() @@ -62,11 +72,21 @@ class QuarkOCP_MX(QuarkScheme): else None ) - if self.weight_dtype == "mxfp4": - self.packed_factor: int | Fraction = 2 + self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES + + if self.is_unpacked: + self.packed_factor: int | Fraction = 1 + elif self.weight_dtype == "mxfp4": + self.packed_factor = 2 else: self.packed_factor = Fraction(numerator=8, denominator=6) + if self.is_unpacked: + # MXFP8 runs on the shared kernel abstraction, which picks a native + # backend where one exists and emulates otherwise. The MXFP4/MXFP6 + # warnings below do not apply. + return + if not current_platform.supports_mx(): logger.warning_once( "The current platform does not support native MXFP4/MXFP6 " @@ -88,7 +108,9 @@ class QuarkOCP_MX(QuarkScheme): ) def get_packed_dim(self, dim: int, quant_dtype: str): - if quant_dtype == "mxfp4": + if quant_dtype in OCP_MX_UNPACKED_DTYPES: + return dim + elif quant_dtype == "mxfp4": assert dim % 2 == 0 return dim // 2 elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}: @@ -102,6 +124,26 @@ class QuarkOCP_MX(QuarkScheme): "open an issue." ) + def _expanding_scale_loader(self, weight_loader: Callable) -> Callable: + """Expand a 2-D block scale to one row per weight row. + + A ``block_size=[R, 32]`` export carries one e8m0 value per R weight + rows; the kernels want one per row. At ``R == 1`` the layouts already + coincide, so the loader is passed through untouched. + """ + if self.scale_block_rows == 1: + return weight_loader + + rows = self.scale_block_rows + + def expanding_loader(param, loaded_weight, *args, **kwargs): + loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave( + rows, dim=0 + ) + return weight_loader(param, loaded_weight, *args, **kwargs) + + return expanding_loader + @classmethod def get_min_capability(cls) -> int: return 70 @@ -130,36 +172,68 @@ class QuarkOCP_MX(QuarkScheme): "tensor-parallel sharding for this layer." ) - output_size_per_partition = sum(output_partition_sizes) - layer.logical_widths = output_partition_sizes + if self.is_unpacked: + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.params_dtype = params_dtype - # WEIGHT - weight = PackedvLLMParameter( - data=torch.empty( - output_size_per_partition, - self.get_packed_dim(input_size_per_partition, self.weight_dtype), - dtype=torch.uint8, - ), - input_dim=1, - output_dim=0, - packed_dim=1, - packed_factor=self.packed_factor, - weight_loader=weight_loader, - ) - layer.register_parameter("weight", weight) + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=MXFP8_VALUE_DTYPE, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) - # WEIGHT SCALE - weight_scale = GroupQuantScaleParameter( - data=torch.empty( - output_size_per_partition, - input_size_per_partition // OCP_MX_BLOCK_SIZE, - dtype=torch.uint8, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) - layer.register_parameter("weight_scale", weight_scale) + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=MXFP8_SCALE_DTYPE, + ), + input_dim=1, + output_dim=0, + weight_loader=self._expanding_scale_loader(weight_loader), + ) + layer.register_parameter("weight_scale", weight_scale) + layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE] + else: + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + + # WEIGHT + weight = PackedvLLMParameter( + data=torch.empty( + output_size_per_partition, + self.get_packed_dim(input_size_per_partition, self.weight_dtype), + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + packed_dim=1, + packed_factor=self.packed_factor, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) if self.weight_quant_key == kMxfp4Static: self.ocp_mx_linear = init_mxfp4_linear_kernel( @@ -170,6 +244,16 @@ class QuarkOCP_MX(QuarkScheme): weight_quant_key=self.weight_quant_key, activation_quant_key=self.activation_quant_key, ) + elif self.weight_quant_key == kMxfp8Static: + self.ocp_mx_linear = init_mxfp8_linear_kernel( + weight_shape=layer.weight.shape + ) + else: + raise NotImplementedError( + "No OCP MX linear kernel for weight_quant_key=" + f"{self.weight_quant_key}. Something is wrong, please open an " + "issue." + ) def apply_weights( self, diff --git a/vllm/model_executor/layers/quantization/utils/config_utils.py b/vllm/model_executor/layers/quantization/utils/config_utils.py index 1beb988ac2..eb50ed1a23 100644 --- a/vllm/model_executor/layers/quantization/utils/config_utils.py +++ b/vllm/model_executor/layers/quantization/utils/config_utils.py @@ -44,7 +44,12 @@ def get_quark_ocp_mx_group_size( return None assert weight_quant is not None - return int(weight_quant["group_size"]) + group_size = weight_quant.get("group_size") + if group_size is None: + block_size = weight_quant.get("block_size") + if isinstance(block_size, list | tuple) and len(block_size) >= 2: + group_size = block_size[1] + return int(group_size) if group_size is not None else None def is_shared_expert_quant_fse_compatible( diff --git a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py index af47ea2213..6fa93c0a2b 100644 --- a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py +++ b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import Enum +from typing import Any from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( @@ -11,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kMxfp6E2M3Static, kMxfp6E3M2Dynamic, kMxfp6E3M2Static, + kMxfp8Dynamic, + kMxfp8Static, ) logger = init_logger(__name__) @@ -21,14 +24,19 @@ _WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = { "mxfp4": kMxfp4Static, "mxfp6_e3m2": kMxfp6E3M2Static, "mxfp6_e2m3": kMxfp6E2M3Static, + "mxfp8_e4m3": kMxfp8Static, } _ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = { "mxfp4": kMxfp4Dynamic, "mxfp6_e3m2": kMxfp6E3M2Dynamic, "mxfp6_e2m3": kMxfp6E2M3Dynamic, + "mxfp8_e4m3": kMxfp8Dynamic, } +# Unpacked OCP MX dtypes: one byte per element, no `packed_factor`. +OCP_MX_UNPACKED_DTYPES = {"mxfp8_e4m3"} + OCP_MX_DTYPES = { "mxfp4", "mxfp6_e3m2", @@ -39,6 +47,56 @@ OCP_MX_DTYPES = { } +def ocp_mx_weight_dtype_and_rows( + weight_quant: dict[str, Any] | None, +) -> tuple[str, int] | None: + """Normalize a Quark weight quant config to ``(mx_dtype, scale_block_rows)``. + + Quark spells OCP MX weights two ways. The canonical one is 1-D per-group: + ``qscheme="per_group"``, ``group_size=32``, ``scale_format="e8m0"``. MXFP8 + checkpoints may instead use a 2-D per-block spelling + (``qscheme="per_block"``, ``block_size=[R, 32]``, + ``scale_type="float8_e8m0fnu"``), which carries one scale per ``R`` weight + rows rather than one per row. ``scale_block_rows`` is that ``R``; it is 1 + for the canonical spelling, where the two layouts coincide. + + Returns ``None`` when the config is not an OCP MX weight quantization. + """ + if not isinstance(weight_quant, dict): + return None + + dtype = weight_quant.get("dtype") + if not isinstance(dtype, str): + return None + mx_dtype = dtype.replace("fp", "mxfp") + if mx_dtype not in _WEIGHT_QUANT_KEY_MAP: + return None + + qscheme = weight_quant.get("qscheme") + if qscheme == "per_group": + if weight_quant.get("group_size") != OCP_MX_BLOCK_SIZE: + return None + if weight_quant.get("scale_format") != "e8m0": + return None + return mx_dtype, 1 + + if qscheme == "per_block": + # Only the unpacked dtypes are accepted here: expanding a 2-D scale + # over a sub-byte packed weight has no checkpoint to validate against. + if mx_dtype not in OCP_MX_UNPACKED_DTYPES: + return None + block_size = list(weight_quant.get("block_size") or []) + if len(block_size) != 2 or block_size[1] != OCP_MX_BLOCK_SIZE: + return None + if weight_quant.get("scale_type") != "float8_e8m0fnu": + return None + if weight_quant.get("symmetric") is not True or weight_quant.get("is_dynamic"): + return None + return mx_dtype, block_size[0] + + return None + + class OCP_MX_Scheme(str, Enum): w_mxfp4 = "w_mxfp4" w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4" diff --git a/vllm/models/deepseek_v4/quant_config.py b/vllm/models/deepseek_v4/quant_config.py index 2ffedc97d3..be8846fc79 100644 --- a/vllm/models/deepseek_v4/quant_config.py +++ b/vllm/models/deepseek_v4/quant_config.py @@ -135,15 +135,15 @@ class DeepseekV4FP8Config(Fp8Config): def override_quantization_method( cls, hf_quant_cfg, user_quant, hf_config=None ) -> QuantizationMethods | None: + # Quark checkpoints are always handled by QuarkConfig. + if ( + isinstance(hf_quant_cfg, dict) + and hf_quant_cfg.get("quant_method") == "quark" + ): + return None if not ( isinstance(hf_quant_cfg, dict) - and ( - hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") - or ( - hf_quant_cfg.get("quant_method") == "quark" - and cls._is_quark_mxfp4_ocp(hf_quant_cfg) - ) - ) + and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") ): return None model_type = getattr(hf_config, "model_type", None) diff --git a/vllm/models/deepseek_v41/amd/model.py b/vllm/models/deepseek_v41/amd/model.py index 1383bac820..0cad301cb6 100644 --- a/vllm/models/deepseek_v41/amd/model.py +++ b/vllm/models/deepseek_v41/amd/model.py @@ -895,6 +895,11 @@ def _make_deepseek_v4_weights_mapper( # renames the engram fp8 table but not its scale; route the # scale explicitly to the same module. re.compile(r"(engram\.embed)\.scale$"): r"\1_tokens.weight_scale_inv", + # Quark exports spell the same tensor ``embed.weight_scale``, + # which the ``\.scale$`` rules never match. + re.compile( + r"(engram\.embed)\.weight_scale$" + ): r"\1_tokens.weight_scale_inv", re.compile(r"\.scale$"): f".{linear_scale_name}", } else: @@ -908,6 +913,11 @@ def _make_deepseek_v4_weights_mapper( ): r"\1.weight_scale_inv", # Same engram reroute as the fp4 branch above. re.compile(r"(engram\.embed)\.scale$"): r"\1_tokens.weight_scale_inv", + # Quark exports spell the same tensor ``embed.weight_scale``, + # which the ``\.scale$`` rules never match. + re.compile( + r"(engram\.embed)\.weight_scale$" + ): r"\1_tokens.weight_scale_inv", re.compile(r"\.scale$"): f".{linear_scale_name}", } return WeightsMapper( diff --git a/vllm/models/deepseek_v41/amd/vl_model.py b/vllm/models/deepseek_v41/amd/vl_model.py index 3c57df2ad2..5ed045029b 100644 --- a/vllm/models/deepseek_v41/amd/vl_model.py +++ b/vllm/models/deepseek_v41/amd/vl_model.py @@ -79,20 +79,22 @@ class DeepseekV4VLImagePixelInputs(TensorSchema): types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})] +_VL_PREFIX_MAPPING: dict[str, str | None] = { + "layers.": "language_model.model.layers.", + "embed.": "language_model.model.embed.", + "norm.": "language_model.model.norm.", + "hc_head": "language_model.model.hc_head", + "mtp.": "language_model.model.mtp.", +} + + def _make_deepseek_v4_vl_weights_mapper( expert_dtype: str, linear_scale_name: str ) -> WeightsMapper: """Text-checkpoint mapping rules re-rooted under ``language_model.``.""" base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name) - orig_to_new_prefix: dict[str, str | None] = { - "layers.": "language_model.model.layers.", - "embed.": "language_model.model.embed.", - "norm.": "language_model.model.norm.", - "hc_head": "language_model.model.hc_head", - "mtp.": "language_model.model.mtp.", - } return WeightsMapper( - orig_to_new_prefix=orig_to_new_prefix, + orig_to_new_prefix=dict(_VL_PREFIX_MAPPING), orig_to_new_regex=base.orig_to_new_regex, orig_to_new_suffix={ "head.weight": "language_model.lm_head.weight", @@ -131,6 +133,22 @@ class DeepseekV41ForCausalLM( supports_encoder_tp_data = True + # Both of these are read off the *class* by + # ``configure_quant_config``/``SupportsQuant``, before ``__init__`` runs, + # to translate a quantization config's checkpoint-style module names into + # runtime prefixes. ``__init__`` later overwrites the instance + # ``hf_to_vllm_mapper`` with the full weight-loading mapper; only the + # prefix rules matter here. Without the class attributes, per-layer specs + # naming ``layers.N.attn.wq_a`` never match this wrapper's + # ``language_model.model.layers.N.attn.wq_a`` and silently fall back to + # the global spec. + packed_modules_mapping = DeepseekV41LLMForCausalLM.packed_modules_mapping + + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix=dict(_VL_PREFIX_MAPPING), + orig_to_new_substr={".shared_experts.w2": ".shared_experts.down_proj"}, + ) + # The MoE router needs raw token ids to detect image-span tokens # (all carrying image_token_id, see common/mm_preprocess.py) and apply # bias_vl. diff --git a/vllm/models/deepseek_v41/quant_config.py b/vllm/models/deepseek_v41/quant_config.py index c8361b2da5..bed60ae758 100644 --- a/vllm/models/deepseek_v41/quant_config.py +++ b/vllm/models/deepseek_v41/quant_config.py @@ -139,15 +139,17 @@ class DeepseekV4FP8Config(Fp8Config): def override_quantization_method( cls, hf_quant_cfg, user_quant, hf_config=None ) -> QuantizationMethods | None: + # Quark checkpoints are always handled by QuarkConfig, which knows + # how to dispatch each per-layer scheme (MXFP4, MXFP8, etc.). + # ``from_config`` below rewrites the config into a single global FP8 + # scheme, which would discard per-layer specs — so never claim Quark. + if isinstance(hf_quant_cfg, dict) and ( + hf_quant_cfg.get("quant_method") == "quark" + ): + return None if not ( isinstance(hf_quant_cfg, dict) - and ( - hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") - or ( - hf_quant_cfg.get("quant_method") == "quark" - and cls._is_quark_mxfp4_ocp(hf_quant_cfg) - ) - ) + and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8") ): return None model_type = getattr(hf_config, "model_type", None)