[ROCm][Quantization] Support AMD Quark per-block FP8 for fused MoE layers (#52263)
Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
This commit is contained in:
@@ -53,6 +53,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_weight_tensor_strategy,
|
||||
process_fp8_weight_tensor_strategy_moe,
|
||||
validate_fp8_block_shape,
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
@@ -575,34 +576,23 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_block_shape is not None
|
||||
moe_block_shape = self.moe_block_shape
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
block_n, block_k = (
|
||||
self.weight_block_size[0],
|
||||
self.weight_block_size[1],
|
||||
)
|
||||
# NOTE: To ensure proper alignment of the block-wise quantization
|
||||
# scales, the output_size of the weights for both the gate and up
|
||||
# layers must be divisible by block_n.
|
||||
# Required by column parallel or enabling merged weights
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
if self.weight_scale_refine is None:
|
||||
validate_fp8_block_shape_moe(
|
||||
intermediate_size_per_partition,
|
||||
self.weight_block_size,
|
||||
)
|
||||
else:
|
||||
# Use the refined block grid for the scale parameters; the
|
||||
# loader upsamples the checkpoint scales accordingly.
|
||||
block_n, block_k = moe_block_shape
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
# Required by row parallel
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
block_n, block_k = moe_block_shape
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
block_n, block_k = moe_block_shape
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
block_n, block_k = moe_block_shape
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
|
||||
Reference in New Issue
Block a user