[ROCm][Quantization] Support AMD Quark per-block FP8 for fused MoE layers (#52263)

Signed-off-by: huji <huidong.ji@amd.com>
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
This commit is contained in:
jimmy-adams
2026-09-08 10:30:26 +00:00
committed by GitHub
parent 5db652225f
commit e41a17e606
5 changed files with 283 additions and 56 deletions
@@ -8,7 +8,6 @@ from compressed_tensors.quantization import (
QuantizationStrategy,
)
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
@@ -31,6 +30,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_input_tensor_strategy_moe,
process_fp8_weight_tensor_strategy_moe,
validate_fp8_block_shape_moe,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
@@ -129,28 +129,14 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
if self.block_quant:
assert self.weight_block_size is not None
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
self.weight_block_size[0],
self.weight_block_size[1],
)
# NOTE: To ensure proper alignment of the block-wise quantization
# scales, the output_size of the weights for both the gate and up
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
validate_fp8_block_shape_moe(
intermediate_size_per_partition,
self.weight_block_size,
)
# WEIGHTS
w13_weight = torch.nn.Parameter(
+12 -22
View File
@@ -53,6 +53,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_weight_tensor_strategy,
process_fp8_weight_tensor_strategy_moe,
validate_fp8_block_shape,
validate_fp8_block_shape_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
@@ -575,34 +576,23 @@ class Fp8MoEMethod(FusedMoEMethodBase):
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
self.weight_block_size[0],
self.weight_block_size[1],
)
# NOTE: To ensure proper alignment of the block-wise quantization
# scales, the output_size of the weights for both the gate and up
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
validate_fp8_block_shape_moe(
intermediate_size_per_partition,
self.weight_block_size,
)
else:
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
tp_size = get_tensor_model_parallel_world_size()
if intermediate_size_per_partition % block_n != 0:
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(
@@ -57,6 +57,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
select_nvfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
validate_fp8_block_shape_moe,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
@@ -66,8 +69,10 @@ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4W4A8StaticChannelSym,
@@ -230,10 +235,12 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
kFp8StaticTensorSym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Dynamic128Sym,
]
supported_weight_quant_keys = [
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8Static128BlockSym,
]
def __init__(
@@ -244,14 +251,18 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
):
super().__init__(moe, weight_quant_key, activation_quant_key)
self.weight_dtype = "fp8"
self.weight_qscheme = (
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
)
self.input_qscheme = (
"per_channel"
if activation_quant_key == kFp8DynamicTokenSym
else "per_tensor"
)
if weight_quant_key == kFp8StaticChannelSym:
self.weight_qscheme = "per_channel"
elif weight_quant_key == kFp8Static128BlockSym:
self.weight_qscheme = "per_block"
else:
self.weight_qscheme = "per_tensor"
if activation_quant_key == kFp8DynamicTokenSym:
self.input_qscheme = "per_channel"
elif activation_quant_key == kFp8Dynamic128Sym:
self.input_qscheme = "per_group"
else:
self.input_qscheme = "per_tensor"
self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
per_tensor = (
self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor"
@@ -259,16 +270,31 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
per_channel = (
self.weight_qscheme == "per_channel" and self.input_qscheme == "per_channel"
)
self.act_quant_group_shape = (
GroupShape.PER_TOKEN if per_channel else GroupShape.PER_TENSOR
per_block = (
self.weight_qscheme == "per_block" and self.input_qscheme == "per_group"
)
if not (per_tensor or per_channel):
if not (per_tensor or per_channel or per_block):
raise ValueError(
"For FP8 Fused MoE layers, only per-tensor and per-channel "
"scales for weights and activations are supported. Found "
f"{self.weight_qscheme}, {self.input_qscheme}"
"For FP8 Fused MoE layers, only per-tensor, per-channel and "
"per-block scales for weights and activations are supported. "
f"Found {self.weight_qscheme}, {self.input_qscheme}"
) # noqa E501
# One scale per (block_n, block_k) tile of the weight, so the block
# shape is carried by the weight key's group shape.
self.weight_block_size: list[int] | None = (
list(weight_quant_key.scale.group_shape) if per_block else None
)
self.block_quant = self.weight_block_size is not None
if per_channel:
self.act_quant_group_shape = GroupShape.PER_TOKEN
elif per_block:
assert self.weight_block_size is not None
self.act_quant_group_shape = GroupShape(1, self.weight_block_size[1])
else:
self.act_quant_group_shape = GroupShape.PER_TENSOR
self.fp8_backend, self.experts_cls = select_fp8_moe_backend(
config=moe,
weight_key=self.weight_quant_key,
@@ -293,6 +319,15 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
layer.weight_block_size = None
params_dtype = torch.float8_e4m3fn
if self.block_quant:
assert self.weight_block_size is not None
layer.weight_block_size = self.weight_block_size
block_n, block_k = self.weight_block_size[0], self.weight_block_size[1]
validate_fp8_block_shape_moe(
intermediate_size_per_partition,
self.weight_block_size,
)
# WEIGHTS
w13_weight = torch.nn.Parameter(
torch.zeros(
@@ -368,6 +403,35 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
elif self.weight_qscheme == "per_block":
# One scale per (block_n, block_k) tile of each expert's weight.
w13_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
self.moe.w13_num_shards
* ((intermediate_size_per_partition + block_n - 1) // block_n),
(hidden_size + block_k - 1) // block_k,
dtype=torch.float32,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
w2_weight_scale = torch.nn.Parameter(
torch.ones(
num_experts,
(hidden_size + block_n - 1) // block_n,
(intermediate_size_per_partition + block_k - 1) // block_k,
dtype=torch.float32,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
# Add PER-BLOCK quantization for RoutedExperts.weight_loader.
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
# INPUT_SCALES
if self.static_input_scales:
@@ -548,6 +612,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
a2_scale=layer.w2_input_scale,
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
block_shape=self.weight_block_size,
per_act_token_quant=self.input_qscheme == "per_channel",
per_out_ch_quant=self.weight_qscheme == "per_channel",
swiglu_limit=getattr(layer, "swiglu_limit", None),
@@ -1265,6 +1265,35 @@ def validate_fp8_block_shape(
)
def validate_fp8_block_shape_moe(
intermediate_size_per_partition: int,
block_size: list[int],
) -> None:
"""Validate fused MoE block quantization shapes for tensor parallelism."""
from vllm.distributed import get_tensor_model_parallel_world_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = block_size[0], block_size[1]
# NOTE: To ensure proper alignment of the block-wise quantization
# scales, the output_size of the weights for both the gate and up
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
def create_fp8_weight_parameter(
output_size_per_partition: int,
input_size_per_partition: int,