Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)
Signed-off-by: Jimmy Adams <Adam_ji@outlook.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: huji <huidong.ji@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Andreas Karatzas
huji
Cursor
parent
0de0362ea1
commit
da788334bc
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
QuarkScheme,
|
||||
QuarkW4A8_MXFP4_FP8,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
@@ -66,6 +67,14 @@ class QuarkConfig(QuantizationConfig):
|
||||
if kv_cache_group is None:
|
||||
kv_cache_group = []
|
||||
self.quant_config = quant_config
|
||||
# Copy the class-level default (which a subclass may override, e.g. the
|
||||
# DeepSeek-V4 Quark config) so per-instance edits don't mutate the class.
|
||||
# Read from the class rather than ``self`` because the base
|
||||
# ``QuantizationConfig.__init__`` sets an empty instance-level
|
||||
# ``packed_modules_mapping`` that would otherwise shadow the override.
|
||||
self.packed_modules_mapping = dict(
|
||||
getattr(type(self), "packed_modules_mapping", {})
|
||||
)
|
||||
self.kv_cache_group = kv_cache_group
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.pack_method = pack_method
|
||||
@@ -341,11 +350,30 @@ class QuarkConfig(QuantizationConfig):
|
||||
"per_tensor",
|
||||
"per_channel",
|
||||
]
|
||||
block_size = list(weight_quant.get("block_size") or [])
|
||||
is_per_block_weight = (
|
||||
weight_quant.get("qscheme") == "per_block"
|
||||
and block_size == [128, 128]
|
||||
and weight_quant.get("symmetric") is True
|
||||
)
|
||||
|
||||
if not (is_fp8_dtype and is_static_weight and is_per_tensor_or_channel_weight):
|
||||
if not is_fp8_dtype or not is_static_weight:
|
||||
return False
|
||||
|
||||
# Dynamic quantization is always supported if weights supported.
|
||||
if is_per_block_weight:
|
||||
if not input_quant.get("is_dynamic"):
|
||||
return False
|
||||
return (
|
||||
input_quant.get("qscheme") == "per_group"
|
||||
and input_quant.get("group_size") == block_size[1]
|
||||
and input_quant.get("symmetric") is True
|
||||
)
|
||||
|
||||
if not is_per_tensor_or_channel_weight:
|
||||
return False
|
||||
|
||||
# Dynamic quantization is always supported if tensor/channel weights
|
||||
# are supported.
|
||||
if input_quant.get("is_dynamic"):
|
||||
return True
|
||||
|
||||
@@ -640,6 +668,8 @@ class QuarkConfig(QuantizationConfig):
|
||||
QuarkW8A8Fp8.get_min_capability(), error=False
|
||||
)
|
||||
if is_fp8_w8a8_supported:
|
||||
if weight_config.get("qscheme") == "per_block":
|
||||
return QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
return QuarkW8A8Fp8(weight_config, input_config)
|
||||
elif self._is_static_tensor_w8a8(weight_config, input_config):
|
||||
weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
|
||||
@@ -1590,15 +1590,16 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
|
||||
if not torch.allclose(
|
||||
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
|
||||
if self.moe.is_act_and_mul and not torch.allclose(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
):
|
||||
raise ValueError("Different global scales for w1 and w3 is not supported.")
|
||||
logger.warning_once(
|
||||
"w1_weight_scale_2 must match w3_weight_scale_2. "
|
||||
"Accuracy may be affected."
|
||||
)
|
||||
|
||||
# Use a single gscale for w13
|
||||
w13_weight_scale_2 = torch.maximum(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
).contiguous()
|
||||
w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()
|
||||
|
||||
w2_weight_scale_2 = layer.w2_weight_scale_2
|
||||
|
||||
|
||||
@@ -5,12 +5,13 @@ from .quark_nvfp4 import QuarkNVFP4
|
||||
from .quark_ocp_mx import QuarkOCP_MX
|
||||
from .quark_scheme import QuarkScheme
|
||||
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
|
||||
from .quark_w8a8_fp8 import QuarkW8A8Fp8
|
||||
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
|
||||
from .quark_w8a8_int8 import QuarkW8A8Int8
|
||||
|
||||
__all__ = [
|
||||
"QuarkScheme",
|
||||
"QuarkW8A8Fp8",
|
||||
"QuarkW8A8Fp8PerBlock",
|
||||
"QuarkW8A8Int8",
|
||||
"QuarkOCP_MX",
|
||||
"QuarkW4A8_MXFP4_FP8",
|
||||
|
||||
@@ -9,13 +9,18 @@ from torch.nn import Parameter
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
create_fp8_weight_parameter,
|
||||
validate_fp8_block_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
@@ -24,13 +29,14 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
requantize_with_max_scale,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
__all__ = ["QuarkW8A8Fp8"]
|
||||
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -46,6 +52,11 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
|
||||
self.input_qscheme = cast(str, input_config.get("qscheme"))
|
||||
|
||||
if self.weight_qscheme == "per_block":
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization should use "
|
||||
"QuarkW8A8Fp8PerBlock."
|
||||
)
|
||||
per_token_activation = (
|
||||
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
|
||||
)
|
||||
@@ -196,3 +207,103 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
def __init__(
|
||||
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
|
||||
):
|
||||
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
if self.weight_qscheme != "per_block":
|
||||
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
|
||||
block_size = weight_config.get("block_size")
|
||||
if not block_size:
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization requires "
|
||||
"`block_size` in the weight quantization config."
|
||||
)
|
||||
self.weight_config = weight_config
|
||||
self.weight_block_size = list(block_size)
|
||||
self.activation_quant_key = kFp8Dynamic128Sym
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return QuarkW8A8Fp8.get_min_capability()
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
input_size = kwargs.get("input_size", input_size_per_partition)
|
||||
output_size = kwargs.get("output_size", sum(output_partition_sizes))
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
validate_fp8_block_shape(
|
||||
layer,
|
||||
input_size,
|
||||
output_size,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
self.weight_block_size,
|
||||
)
|
||||
|
||||
weight = create_fp8_weight_parameter(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
scale_dtype = (
|
||||
torch.float8_e8m0fnu
|
||||
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
|
||||
else None
|
||||
)
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
BlockQuantScaleParameter,
|
||||
output_partition_sizes,
|
||||
input_size_per_partition,
|
||||
self.weight_block_size,
|
||||
weight_loader,
|
||||
scale_dtype=scale_dtype,
|
||||
)
|
||||
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
|
||||
layer.register_parameter("weight_scale_inv", weight_scale)
|
||||
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
logger.info_once(
|
||||
"Selected %s for QuarkW8A8Fp8PerBlock",
|
||||
type(self.fp8_linear).__name__,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
Reference in New Issue
Block a user