[ROCm][Quantization] Support AMD Quark per-block FP8 for fused MoE layers (#52263)

Signed-off-by: huji <huidong.ji@amd.com>
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
This commit is contained in:
jimmy-adams
2026-09-08 10:30:26 +00:00
committed by GitHub
parent 5db652225f
commit e41a17e606
5 changed files with 283 additions and 56 deletions
+12 -22
View File
@@ -53,6 +53,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
process_fp8_weight_tensor_strategy,
process_fp8_weight_tensor_strategy_moe,
validate_fp8_block_shape,
validate_fp8_block_shape_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
@@ -575,34 +576,23 @@ class Fp8MoEMethod(FusedMoEMethodBase):
assert self.moe_block_shape is not None
moe_block_shape = self.moe_block_shape
layer.weight_block_size = self.weight_block_size
tp_size = get_tensor_model_parallel_world_size()
block_n, block_k = (
self.weight_block_size[0],
self.weight_block_size[1],
)
# NOTE: To ensure proper alignment of the block-wise quantization
# scales, the output_size of the weights for both the gate and up
# layers must be divisible by block_n.
# Required by column parallel or enabling merged weights
if intermediate_size_per_partition % block_n != 0:
if self.weight_scale_refine is None:
raise ValueError(
f"The output_size of gate's and up's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_n = {block_n}."
)
if self.weight_scale_refine is None:
validate_fp8_block_shape_moe(
intermediate_size_per_partition,
self.weight_block_size,
)
else:
# Use the refined block grid for the scale parameters; the
# loader upsamples the checkpoint scales accordingly.
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
# Required by row parallel
if self.weight_scale_refine is None:
raise ValueError(
f"The input_size of down's weight = "
f"{intermediate_size_per_partition} is not divisible by "
f"weight quantization block_k = {block_k}."
)
block_n, block_k = moe_block_shape
tp_size = get_tensor_model_parallel_world_size()
if intermediate_size_per_partition % block_n != 0:
block_n, block_k = moe_block_shape
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
block_n, block_k = moe_block_shape
# WEIGHTS
w13_weight = torch.nn.Parameter(