Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)

Signed-off-by: Jimmy Adams <Adam_ji@outlook.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: huji <huidong.ji@amd.com>
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: huji <huidong.ji@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jimmy-adams
2026-08-07 01:19:31 -07:00
committed by GitHub
co-authored by Andreas Karatzas huji Cursor
parent 0de0362ea1
commit da788334bc
11 changed files with 332 additions and 17 deletions
+23
View File
@@ -3341,6 +3341,29 @@ steps:
- export VLLM_USE_DEEP_GEMM=0
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
- label: LM Eval Large Models (8xB200-8xMI355) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_8
optional: true
num_gpus: 8
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/quantization/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/model_executor/layers/layernorm.py
- csrc/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- tests/evals/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt
#--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
- label: Examples # TBD
@@ -0,0 +1,15 @@
model_name: "amd/DeepSeek-V4-Flash-NVFP4"
accuracy_threshold: 0.92
num_questions: 1319
num_fewshot: 8
startup_max_wait_seconds: 1800
server_args: >-
--tensor-parallel-size 8
--max-model-len 4096
--trust-remote-code
--kv-cache-dtype fp8
--tokenizer-mode deepseek_v4
--compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
@@ -0,0 +1,15 @@
model_name: "amd/DeepSeek-V4-Pro-NVFP4"
accuracy_threshold: 0.92
num_questions: 1319
num_fewshot: 8
startup_max_wait_seconds: 3600
server_args: >-
--tensor-parallel-size 8
--max-model-len 4096
--trust-remote-code
--kv-cache-dtype fp8
--tokenizer-mode deepseek_v4
--compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
env:
VLLM_ROCM_USE_AITER: "1"
VLLM_ROCM_USE_AITER_MOE: "1"
@@ -0,0 +1,8 @@
# GFX950 model configurations for GSM8K evaluation
# TP8 models only
# DeepSeek-V4-Pro-NVFP4.yaml is temporarily disabled: repeated CI failures due to
# cold NFS reads of the 850GiB checkpoint exceeding startup_max_wait_seconds
# (see https://github.com/vllm-project/vllm/pull/47972). Re-enable once the
# startup timeout fix has landed and is verified green.
# DeepSeek-V4-Pro-NVFP4.yaml
DeepSeek-V4-Flash-NVFP4.yaml
+97
View File
@@ -19,8 +19,10 @@ from packaging import version
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
QuarkConfig,
QuarkLinearMethod,
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
)
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
@@ -80,6 +82,101 @@ def enable_pickle(monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def test_quark_config_has_no_model_specific_fused_mappings():
config = QuarkConfig({})
assert "gate_up_proj" not in config.packed_modules_mapping
assert "fused_wqa_wkv" not in config.packed_modules_mapping
def test_quark_config_preserves_existing_packed_modules_mapping():
class CustomQuarkConfig(QuarkConfig):
packed_modules_mapping = {"custom_proj": ["a", "b"]}
config = CustomQuarkConfig({})
assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
def test_quark_fp8_w8a8_detects_per_block_config():
config = QuarkConfig({})
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
}
assert config._is_fp8_w8a8(weight_config, input_config)
def test_quark_fp8_w8a8_rejects_per_block_static_input():
config = QuarkConfig({})
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": False,
"group_size": 128,
"symmetric": True,
}
assert not config._is_fp8_w8a8(weight_config, input_config)
def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
config = QuarkConfig({})
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 64,
"symmetric": True,
}
assert not config._is_fp8_w8a8(weight_config, input_config)
def test_quark_w8a8_fp8_per_block_requires_block_size():
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
}
with pytest.raises(ValueError, match="requires `block_size`"):
QuarkW8A8Fp8PerBlock(weight_config, input_config)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("tp", [1])
def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):
@@ -193,6 +193,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.VLLM_CUTLASS,
NvFp4MoeBackend.MARLIN,
NvFp4MoeBackend.EMULATION,
NvFp4MoeBackend.HUMMING,
}
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
QuarkScheme,
QuarkW4A8_MXFP4_FP8,
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
)
from vllm.model_executor.layers.quantization.quark.utils import (
@@ -66,6 +67,14 @@ class QuarkConfig(QuantizationConfig):
if kv_cache_group is None:
kv_cache_group = []
self.quant_config = quant_config
# Copy the class-level default (which a subclass may override, e.g. the
# DeepSeek-V4 Quark config) so per-instance edits don't mutate the class.
# Read from the class rather than ``self`` because the base
# ``QuantizationConfig.__init__`` sets an empty instance-level
# ``packed_modules_mapping`` that would otherwise shadow the override.
self.packed_modules_mapping = dict(
getattr(type(self), "packed_modules_mapping", {})
)
self.kv_cache_group = kv_cache_group
self.kv_cache_config = kv_cache_config
self.pack_method = pack_method
@@ -341,11 +350,30 @@ class QuarkConfig(QuantizationConfig):
"per_tensor",
"per_channel",
]
block_size = list(weight_quant.get("block_size") or [])
is_per_block_weight = (
weight_quant.get("qscheme") == "per_block"
and block_size == [128, 128]
and weight_quant.get("symmetric") is True
)
if not (is_fp8_dtype and is_static_weight and is_per_tensor_or_channel_weight):
if not is_fp8_dtype or not is_static_weight:
return False
# Dynamic quantization is always supported if weights supported.
if is_per_block_weight:
if not input_quant.get("is_dynamic"):
return False
return (
input_quant.get("qscheme") == "per_group"
and input_quant.get("group_size") == block_size[1]
and input_quant.get("symmetric") is True
)
if not is_per_tensor_or_channel_weight:
return False
# Dynamic quantization is always supported if tensor/channel weights
# are supported.
if input_quant.get("is_dynamic"):
return True
@@ -640,6 +668,8 @@ class QuarkConfig(QuantizationConfig):
QuarkW8A8Fp8.get_min_capability(), error=False
)
if is_fp8_w8a8_supported:
if weight_config.get("qscheme") == "per_block":
return QuarkW8A8Fp8PerBlock(weight_config, input_config)
return QuarkW8A8Fp8(weight_config, input_config)
elif self._is_static_tensor_w8a8(weight_config, input_config):
weight_qscheme = cast(str, weight_config.get("qscheme"))
@@ -1590,15 +1590,16 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
Convert NVFP4 MoE weights into kernel format and setup the kernel.
"""
if not torch.allclose(
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
if self.moe.is_act_and_mul and not torch.allclose(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
):
raise ValueError("Different global scales for w1 and w3 is not supported.")
logger.warning_once(
"w1_weight_scale_2 must match w3_weight_scale_2. "
"Accuracy may be affected."
)
# Use a single gscale for w13
w13_weight_scale_2 = torch.maximum(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
).contiguous()
w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()
w2_weight_scale_2 = layer.w2_weight_scale_2
@@ -5,12 +5,13 @@ from .quark_nvfp4 import QuarkNVFP4
from .quark_ocp_mx import QuarkOCP_MX
from .quark_scheme import QuarkScheme
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
from .quark_w8a8_fp8 import QuarkW8A8Fp8
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
from .quark_w8a8_int8 import QuarkW8A8Int8
__all__ = [
"QuarkScheme",
"QuarkW8A8Fp8",
"QuarkW8A8Fp8PerBlock",
"QuarkW8A8Int8",
"QuarkOCP_MX",
"QuarkW4A8_MXFP4_FP8",
@@ -9,13 +9,18 @@ from torch.nn import Parameter
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
create_fp8_weight_parameter,
validate_fp8_block_shape,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
@@ -24,13 +29,14 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
requantize_with_max_scale,
)
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
from vllm.platforms import current_platform
__all__ = ["QuarkW8A8Fp8"]
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
logger = init_logger(__name__)
@@ -46,6 +52,11 @@ class QuarkW8A8Fp8(QuarkScheme):
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
self.input_qscheme = cast(str, input_config.get("qscheme"))
if self.weight_qscheme == "per_block":
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization should use "
"QuarkW8A8Fp8PerBlock."
)
per_token_activation = (
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
)
@@ -196,3 +207,103 @@ class QuarkW8A8Fp8(QuarkScheme):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
class QuarkW8A8Fp8PerBlock(QuarkScheme):
def __init__(
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
):
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
if self.weight_qscheme != "per_block":
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
block_size = weight_config.get("block_size")
if not block_size:
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization requires "
"`block_size` in the weight quantization config."
)
self.weight_config = weight_config
self.weight_block_size = list(block_size)
self.activation_quant_key = kFp8Dynamic128Sym
self.weight_quant_key = kFp8Static128BlockSym
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@classmethod
def get_min_capability(cls) -> int:
return QuarkW8A8Fp8.get_min_capability()
def process_weights_after_loading(self, layer) -> None:
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs.get("input_size", input_size_per_partition)
output_size = kwargs.get("output_size", sum(output_partition_sizes))
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
layer.weight_block_size = self.weight_block_size
validate_fp8_block_shape(
layer,
input_size,
output_size,
input_size_per_partition,
output_partition_sizes,
self.weight_block_size,
)
weight = create_fp8_weight_parameter(
output_size_per_partition,
input_size_per_partition,
weight_loader,
)
layer.register_parameter("weight", weight)
scale_dtype = (
torch.float8_e8m0fnu
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
else None
)
weight_scale = create_fp8_scale_parameter(
BlockQuantScaleParameter,
output_partition_sizes,
input_size_per_partition,
self.weight_block_size,
weight_loader,
scale_dtype=scale_dtype,
)
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
layer.register_parameter("weight_scale_inv", weight_scale)
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
logger.info_once(
"Selected %s for QuarkW8A8Fp8PerBlock",
type(self.fp8_linear).__name__,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
+17 -4
View File
@@ -750,6 +750,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _resolve_param_name(name: str) -> str:
inv_name = f"{name}_inv"
if name not in params_dict and inv_name in params_dict:
return inv_name
return name
# TP for attention
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
@@ -803,10 +809,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if is_pp_missing_parameter(name, self):
break
param = params_dict[name]
param_name = _resolve_param_name(name)
param = params_dict[param_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name)
loaded_params.add(param_name)
break
else:
if ".experts." in name:
@@ -857,12 +864,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
else:
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
param_name = _resolve_param_name(name)
param = params_dict[param_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
loaded_params.add(param_name)
continue
return loaded_params
@@ -934,6 +942,7 @@ def _make_deepseek_v4_weights_mapper(
"head.weight": "lm_head.weight",
"embed.weight": "embed_tokens.weight",
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
".input_scale": ".input_scale_2",
},
orig_to_new_substr=substr_map,
)
@@ -945,6 +954,10 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
# Default mapper assumes the original FP4-expert checkpoint layout.
# Overridden per-instance in __init__ when expert_dtype != "fp4".
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
packed_modules_mapping = {
"gate_up_proj": ["w1", "w3"],
"fused_wqa_wkv": ["wq_a", "wkv"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()