From da788334bc0683cc44a58b4624e3f5a3c09a09e0 Mon Sep 17 00:00:00 2001 From: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:19:31 +0800 Subject: [PATCH] Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972) Signed-off-by: Jimmy Adams Signed-off-by: Andreas Karatzas Signed-off-by: huji Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: Andreas Karatzas Co-authored-by: huji Co-authored-by: Cursor --- .buildkite/test-amd.yaml | 23 ++++ .../configs/DeepSeek-V4-Flash-NVFP4.yaml | 15 +++ .../gsm8k/configs/DeepSeek-V4-Pro-NVFP4.yaml | 15 +++ .../gsm8k/configs/models-gfx950-large.txt | 8 ++ tests/quantization/test_quark.py | 97 ++++++++++++++ .../layers/fused_moe/oracle/nvfp4.py | 1 + .../layers/quantization/quark/quark.py | 34 ++++- .../layers/quantization/quark/quark_moe.py | 13 +- .../quantization/quark/schemes/__init__.py | 3 +- .../quark/schemes/quark_w8a8_fp8.py | 119 +++++++++++++++++- vllm/models/deepseek_v4/amd/model.py | 21 +++- 11 files changed, 332 insertions(+), 17 deletions(-) create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V4-Flash-NVFP4.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V4-Pro-NVFP4.yaml create mode 100644 tests/evals/gsm8k/configs/models-gfx950-large.txt diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 7b62fda8bb..c6baa13971 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3341,6 +3341,29 @@ steps: - export VLLM_USE_DEEP_GEMM=0 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4 +- label: LM Eval Large Models (8xB200-8xMI355) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_8 + optional: true + num_gpus: 8 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/evals/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt + #--------------------------------------------------------- mi355 ยท examples ----------------------------------------------------------# - label: Examples # TBD diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-NVFP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-NVFP4.yaml new file mode 100644 index 0000000000..c784cf594a --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-NVFP4.yaml @@ -0,0 +1,15 @@ +model_name: "amd/DeepSeek-V4-Flash-NVFP4" +accuracy_threshold: 0.92 +num_questions: 1319 +num_fewshot: 8 +startup_max_wait_seconds: 1800 +server_args: >- + --tensor-parallel-size 8 + --max-model-len 4096 + --trust-remote-code + --kv-cache-dtype fp8 + --tokenizer-mode deepseek_v4 + --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MOE: "1" diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Pro-NVFP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Pro-NVFP4.yaml new file mode 100644 index 0000000000..aac11559a5 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Pro-NVFP4.yaml @@ -0,0 +1,15 @@ +model_name: "amd/DeepSeek-V4-Pro-NVFP4" +accuracy_threshold: 0.92 +num_questions: 1319 +num_fewshot: 8 +startup_max_wait_seconds: 3600 +server_args: >- + --tensor-parallel-size 8 + --max-model-len 4096 + --trust-remote-code + --kv-cache-dtype fp8 + --tokenizer-mode deepseek_v4 + --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MOE: "1" diff --git a/tests/evals/gsm8k/configs/models-gfx950-large.txt b/tests/evals/gsm8k/configs/models-gfx950-large.txt new file mode 100644 index 0000000000..aa7d00bebc --- /dev/null +++ b/tests/evals/gsm8k/configs/models-gfx950-large.txt @@ -0,0 +1,8 @@ +# GFX950 model configurations for GSM8K evaluation +# TP8 models only +# DeepSeek-V4-Pro-NVFP4.yaml is temporarily disabled: repeated CI failures due to +# cold NFS reads of the 850GiB checkpoint exceeding startup_max_wait_seconds +# (see https://github.com/vllm-project/vllm/pull/47972). Re-enable once the +# startup timeout fix has landed and is verified green. +# DeepSeek-V4-Pro-NVFP4.yaml +DeepSeek-V4-Flash-NVFP4.yaml diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 8440db1071..c59ef11f65 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -19,8 +19,10 @@ from packaging import version from vllm._aiter_ops import is_aiter_found_and_supported from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501 + QuarkConfig, QuarkLinearMethod, QuarkW8A8Fp8, + QuarkW8A8Fp8PerBlock, QuarkW8A8Int8, ) from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501 @@ -80,6 +82,101 @@ def enable_pickle(monkeypatch): monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") +def test_quark_config_has_no_model_specific_fused_mappings(): + config = QuarkConfig({}) + + assert "gate_up_proj" not in config.packed_modules_mapping + assert "fused_wqa_wkv" not in config.packed_modules_mapping + + +def test_quark_config_preserves_existing_packed_modules_mapping(): + class CustomQuarkConfig(QuarkConfig): + packed_modules_mapping = {"custom_proj": ["a", "b"]} + + config = CustomQuarkConfig({}) + + assert config.packed_modules_mapping["custom_proj"] == ["a", "b"] + + +def test_quark_fp8_w8a8_detects_per_block_config(): + config = QuarkConfig({}) + weight_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + } + input_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 128, + "symmetric": True, + } + + assert config._is_fp8_w8a8(weight_config, input_config) + + +def test_quark_fp8_w8a8_rejects_per_block_static_input(): + config = QuarkConfig({}) + weight_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + } + input_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": False, + "group_size": 128, + "symmetric": True, + } + + assert not config._is_fp8_w8a8(weight_config, input_config) + + +def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch(): + config = QuarkConfig({}) + weight_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + } + input_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 64, + "symmetric": True, + } + + assert not config._is_fp8_w8a8(weight_config, input_config) + + +def test_quark_w8a8_fp8_per_block_requires_block_size(): + weight_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "symmetric": True, + } + input_config = { + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 128, + "symmetric": True, + } + + with pytest.raises(ValueError, match="requires `block_size`"): + QuarkW8A8Fp8PerBlock(weight_config, input_config) + + @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) @pytest.mark.parametrize("tp", [1]) def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp): diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 30c8458cbd..d540e3ae0c 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -193,6 +193,7 @@ def select_nvfp4_moe_backend( NvFp4MoeBackend.FLASHINFER_CUTEDSL, NvFp4MoeBackend.VLLM_CUTLASS, NvFp4MoeBackend.MARLIN, + NvFp4MoeBackend.EMULATION, NvFp4MoeBackend.HUMMING, } diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 694116827c..7d3b0c4f93 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -33,6 +33,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( QuarkScheme, QuarkW4A8_MXFP4_FP8, QuarkW8A8Fp8, + QuarkW8A8Fp8PerBlock, QuarkW8A8Int8, ) from vllm.model_executor.layers.quantization.quark.utils import ( @@ -66,6 +67,14 @@ class QuarkConfig(QuantizationConfig): if kv_cache_group is None: kv_cache_group = [] self.quant_config = quant_config + # Copy the class-level default (which a subclass may override, e.g. the + # DeepSeek-V4 Quark config) so per-instance edits don't mutate the class. + # Read from the class rather than ``self`` because the base + # ``QuantizationConfig.__init__`` sets an empty instance-level + # ``packed_modules_mapping`` that would otherwise shadow the override. + self.packed_modules_mapping = dict( + getattr(type(self), "packed_modules_mapping", {}) + ) self.kv_cache_group = kv_cache_group self.kv_cache_config = kv_cache_config self.pack_method = pack_method @@ -341,11 +350,30 @@ class QuarkConfig(QuantizationConfig): "per_tensor", "per_channel", ] + block_size = list(weight_quant.get("block_size") or []) + is_per_block_weight = ( + weight_quant.get("qscheme") == "per_block" + and block_size == [128, 128] + and weight_quant.get("symmetric") is True + ) - if not (is_fp8_dtype and is_static_weight and is_per_tensor_or_channel_weight): + if not is_fp8_dtype or not is_static_weight: return False - # Dynamic quantization is always supported if weights supported. + if is_per_block_weight: + if not input_quant.get("is_dynamic"): + return False + return ( + input_quant.get("qscheme") == "per_group" + and input_quant.get("group_size") == block_size[1] + and input_quant.get("symmetric") is True + ) + + if not is_per_tensor_or_channel_weight: + return False + + # Dynamic quantization is always supported if tensor/channel weights + # are supported. if input_quant.get("is_dynamic"): return True @@ -640,6 +668,8 @@ class QuarkConfig(QuantizationConfig): QuarkW8A8Fp8.get_min_capability(), error=False ) if is_fp8_w8a8_supported: + if weight_config.get("qscheme") == "per_block": + return QuarkW8A8Fp8PerBlock(weight_config, input_config) return QuarkW8A8Fp8(weight_config, input_config) elif self._is_static_tensor_w8a8(weight_config, input_config): weight_qscheme = cast(str, weight_config.get("qscheme")) diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 508c715aec..54e10fa76e 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1590,15 +1590,16 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): Convert NVFP4 MoE weights into kernel format and setup the kernel. """ - if not torch.allclose( + # Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale. + if self.moe.is_act_and_mul and not torch.allclose( layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1] ): - raise ValueError("Different global scales for w1 and w3 is not supported.") + logger.warning_once( + "w1_weight_scale_2 must match w3_weight_scale_2. " + "Accuracy may be affected." + ) - # Use a single gscale for w13 - w13_weight_scale_2 = torch.maximum( - layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1] - ).contiguous() + w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous() w2_weight_scale_2 = layer.w2_weight_scale_2 diff --git a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py index 1ef5824fec..4db1ba3845 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py @@ -5,12 +5,13 @@ from .quark_nvfp4 import QuarkNVFP4 from .quark_ocp_mx import QuarkOCP_MX from .quark_scheme import QuarkScheme from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8 -from .quark_w8a8_fp8 import QuarkW8A8Fp8 +from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock from .quark_w8a8_int8 import QuarkW8A8Int8 __all__ = [ "QuarkScheme", "QuarkW8A8Fp8", + "QuarkW8A8Fp8PerBlock", "QuarkW8A8Int8", "QuarkOCP_MX", "QuarkW4A8_MXFP4_FP8", diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py index 280159700e..f7a9861e1d 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -9,13 +9,18 @@ from torch.nn import Parameter from vllm.config import get_current_vllm_config from vllm.logger import init_logger -from vllm.model_executor.kernels.linear import ( - init_fp8_linear_kernel, -) +from vllm.model_executor.kernels.linear import init_fp8_linear_kernel from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + create_fp8_scale_parameter, + create_fp8_weight_parameter, + validate_fp8_block_shape, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, + kFp8Dynamic128Sym, kFp8DynamicTokenSym, + kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, ) @@ -24,13 +29,14 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( requantize_with_max_scale, ) from vllm.model_executor.parameter import ( + BlockQuantScaleParameter, ChannelQuantScaleParameter, ModelWeightParameter, PerTensorScaleParameter, ) from vllm.platforms import current_platform -__all__ = ["QuarkW8A8Fp8"] +__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"] logger = init_logger(__name__) @@ -46,6 +52,11 @@ class QuarkW8A8Fp8(QuarkScheme): self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic")) self.input_qscheme = cast(str, input_config.get("qscheme")) + if self.weight_qscheme == "per_block": + raise ValueError( + "Quark W8A8 FP8 per-block weight quantization should use " + "QuarkW8A8Fp8PerBlock." + ) per_token_activation = ( not self.is_static_input_scheme and self.input_qscheme == "per_channel" ) @@ -196,3 +207,103 @@ class QuarkW8A8Fp8(QuarkScheme): bias: torch.Tensor | None = None, ) -> torch.Tensor: return self.fp8_linear.apply_weights(layer, x, bias) + + +class QuarkW8A8Fp8PerBlock(QuarkScheme): + def __init__( + self, weight_config: dict[str, Any], input_config: dict[str, Any] | None + ): + self.weight_qscheme = cast(str, weight_config.get("qscheme")) + if self.weight_qscheme != "per_block": + raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.") + block_size = weight_config.get("block_size") + if not block_size: + raise ValueError( + "Quark W8A8 FP8 per-block weight quantization requires " + "`block_size` in the weight quantization config." + ) + self.weight_config = weight_config + self.weight_block_size = list(block_size) + self.activation_quant_key = kFp8Dynamic128Sym + self.weight_quant_key = kFp8Static128BlockSym + self.out_dtype = torch.get_default_dtype() + self.input_dtype = get_current_vllm_config().model_config.dtype + + @classmethod + def get_min_capability(cls) -> int: + return QuarkW8A8Fp8.get_min_capability() + + def process_weights_after_loading(self, layer) -> None: + self.fp8_linear.process_weights_after_loading(layer) + + def create_weights( + self, + layer: torch.nn.Module, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + input_size = kwargs.get("input_size", input_size_per_partition) + output_size = kwargs.get("output_size", sum(output_partition_sizes)) + output_size_per_partition = sum(output_partition_sizes) + + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + layer.weight_block_size = self.weight_block_size + + validate_fp8_block_shape( + layer, + input_size, + output_size, + input_size_per_partition, + output_partition_sizes, + self.weight_block_size, + ) + + weight = create_fp8_weight_parameter( + output_size_per_partition, + input_size_per_partition, + weight_loader, + ) + layer.register_parameter("weight", weight) + + scale_dtype = ( + torch.float8_e8m0fnu + if self.weight_config.get("scale_type") == "float8_e8m0fnu" + else None + ) + weight_scale = create_fp8_scale_parameter( + BlockQuantScaleParameter, + output_partition_sizes, + input_size_per_partition, + self.weight_block_size, + weight_loader, + scale_dtype=scale_dtype, + ) + # DeepSeek V4 weight mappers route checkpoint ".scale" tensors here. + layer.register_parameter("weight_scale_inv", weight_scale) + + self.fp8_linear = init_fp8_linear_kernel( + activation_quant_key=self.activation_quant_key, + weight_quant_key=self.weight_quant_key, + weight_shape=layer.weight.shape, + input_dtype=self.input_dtype, + out_dtype=self.out_dtype, + module_name=self.__class__.__name__, + ) + logger.info_once( + "Selected %s for QuarkW8A8Fp8PerBlock", + type(self.fp8_linear).__name__, + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.fp8_linear.apply_weights(layer, x, bias) diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index becfd1d4f7..d626febdc0 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -750,6 +750,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() + def _resolve_param_name(name: str) -> str: + inv_name = f"{name}_inv" + if name not in params_dict and inv_name in params_dict: + return inv_name + return name + # TP for attention tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() @@ -803,10 +809,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if is_pp_missing_parameter(name, self): break - param = params_dict[name] + param_name = _resolve_param_name(name) + param = params_dict[param_name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) - loaded_params.add(name) + loaded_params.add(param_name) break else: if ".experts." in name: @@ -857,12 +864,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): else: if is_pp_missing_parameter(name, self): continue - param = params_dict[name] + param_name = _resolve_param_name(name) + param = params_dict[param_name] weight_loader = getattr( param, "weight_loader", default_weight_loader ) weight_loader(param, loaded_weight) - loaded_params.add(name) + loaded_params.add(param_name) continue return loaded_params @@ -934,6 +942,7 @@ def _make_deepseek_v4_weights_mapper( "head.weight": "lm_head.weight", "embed.weight": "embed_tokens.weight", ".ffn.gate.bias": ".ffn.gate.e_score_correction_bias", + ".input_scale": ".input_scale_2", }, orig_to_new_substr=substr_map, ) @@ -945,6 +954,10 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3): # Default mapper assumes the original FP4-expert checkpoint layout. # Overridden per-instance in __init__ when expert_dtype != "fp4". hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4") + packed_modules_mapping = { + "gate_up_proj": ["w1", "w3"], + "fused_wqa_wkv": ["wq_a", "wkv"], + } def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__()