Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)

Signed-off-by: Jimmy Adams <Adam_ji@outlook.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: huji <huidong.ji@amd.com>
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: huji <huidong.ji@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
jimmy-adams
2026-08-07 01:19:31 -07:00
committed by GitHub
co-authored by Andreas Karatzas huji Cursor
parent 0de0362ea1
commit da788334bc
11 changed files with 332 additions and 17 deletions
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
QuarkScheme,
QuarkW4A8_MXFP4_FP8,
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
)
from vllm.model_executor.layers.quantization.quark.utils import (
@@ -66,6 +67,14 @@ class QuarkConfig(QuantizationConfig):
if kv_cache_group is None:
kv_cache_group = []
self.quant_config = quant_config
# Copy the class-level default (which a subclass may override, e.g. the
# DeepSeek-V4 Quark config) so per-instance edits don't mutate the class.
# Read from the class rather than ``self`` because the base
# ``QuantizationConfig.__init__`` sets an empty instance-level
# ``packed_modules_mapping`` that would otherwise shadow the override.
self.packed_modules_mapping = dict(
getattr(type(self), "packed_modules_mapping", {})
)
self.kv_cache_group = kv_cache_group
self.kv_cache_config = kv_cache_config
self.pack_method = pack_method
@@ -341,11 +350,30 @@ class QuarkConfig(QuantizationConfig):
"per_tensor",
"per_channel",
]
block_size = list(weight_quant.get("block_size") or [])
is_per_block_weight = (
weight_quant.get("qscheme") == "per_block"
and block_size == [128, 128]
and weight_quant.get("symmetric") is True
)
if not (is_fp8_dtype and is_static_weight and is_per_tensor_or_channel_weight):
if not is_fp8_dtype or not is_static_weight:
return False
# Dynamic quantization is always supported if weights supported.
if is_per_block_weight:
if not input_quant.get("is_dynamic"):
return False
return (
input_quant.get("qscheme") == "per_group"
and input_quant.get("group_size") == block_size[1]
and input_quant.get("symmetric") is True
)
if not is_per_tensor_or_channel_weight:
return False
# Dynamic quantization is always supported if tensor/channel weights
# are supported.
if input_quant.get("is_dynamic"):
return True
@@ -640,6 +668,8 @@ class QuarkConfig(QuantizationConfig):
QuarkW8A8Fp8.get_min_capability(), error=False
)
if is_fp8_w8a8_supported:
if weight_config.get("qscheme") == "per_block":
return QuarkW8A8Fp8PerBlock(weight_config, input_config)
return QuarkW8A8Fp8(weight_config, input_config)
elif self._is_static_tensor_w8a8(weight_config, input_config):
weight_qscheme = cast(str, weight_config.get("qscheme"))
@@ -1590,15 +1590,16 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
Convert NVFP4 MoE weights into kernel format and setup the kernel.
"""
if not torch.allclose(
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
if self.moe.is_act_and_mul and not torch.allclose(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
):
raise ValueError("Different global scales for w1 and w3 is not supported.")
logger.warning_once(
"w1_weight_scale_2 must match w3_weight_scale_2. "
"Accuracy may be affected."
)
# Use a single gscale for w13
w13_weight_scale_2 = torch.maximum(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
).contiguous()
w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()
w2_weight_scale_2 = layer.w2_weight_scale_2
@@ -5,12 +5,13 @@ from .quark_nvfp4 import QuarkNVFP4
from .quark_ocp_mx import QuarkOCP_MX
from .quark_scheme import QuarkScheme
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
from .quark_w8a8_fp8 import QuarkW8A8Fp8
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
from .quark_w8a8_int8 import QuarkW8A8Int8
__all__ = [
"QuarkScheme",
"QuarkW8A8Fp8",
"QuarkW8A8Fp8PerBlock",
"QuarkW8A8Int8",
"QuarkOCP_MX",
"QuarkW4A8_MXFP4_FP8",
@@ -9,13 +9,18 @@ from torch.nn import Parameter
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
)
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
create_fp8_weight_parameter,
validate_fp8_block_shape,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
@@ -24,13 +29,14 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
requantize_with_max_scale,
)
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
from vllm.platforms import current_platform
__all__ = ["QuarkW8A8Fp8"]
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
logger = init_logger(__name__)
@@ -46,6 +52,11 @@ class QuarkW8A8Fp8(QuarkScheme):
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
self.input_qscheme = cast(str, input_config.get("qscheme"))
if self.weight_qscheme == "per_block":
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization should use "
"QuarkW8A8Fp8PerBlock."
)
per_token_activation = (
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
)
@@ -196,3 +207,103 @@ class QuarkW8A8Fp8(QuarkScheme):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
class QuarkW8A8Fp8PerBlock(QuarkScheme):
def __init__(
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
):
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
if self.weight_qscheme != "per_block":
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
block_size = weight_config.get("block_size")
if not block_size:
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization requires "
"`block_size` in the weight quantization config."
)
self.weight_config = weight_config
self.weight_block_size = list(block_size)
self.activation_quant_key = kFp8Dynamic128Sym
self.weight_quant_key = kFp8Static128BlockSym
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@classmethod
def get_min_capability(cls) -> int:
return QuarkW8A8Fp8.get_min_capability()
def process_weights_after_loading(self, layer) -> None:
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs.get("input_size", input_size_per_partition)
output_size = kwargs.get("output_size", sum(output_partition_sizes))
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
layer.weight_block_size = self.weight_block_size
validate_fp8_block_shape(
layer,
input_size,
output_size,
input_size_per_partition,
output_partition_sizes,
self.weight_block_size,
)
weight = create_fp8_weight_parameter(
output_size_per_partition,
input_size_per_partition,
weight_loader,
)
layer.register_parameter("weight", weight)
scale_dtype = (
torch.float8_e8m0fnu
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
else None
)
weight_scale = create_fp8_scale_parameter(
BlockQuantScaleParameter,
output_partition_sizes,
input_size_per_partition,
self.weight_block_size,
weight_loader,
scale_dtype=scale_dtype,
)
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
layer.register_parameter("weight_scale_inv", weight_scale)
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
logger.info_once(
"Selected %s for QuarkW8A8Fp8PerBlock",
type(self.fp8_linear).__name__,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)