[ROCm][Quantization] Support AMD Quark per-block FP8 for fused MoE layers (#52263)
Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
This commit is contained in:
@@ -11,7 +11,7 @@ import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from importlib.util import find_spec
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
@@ -26,6 +26,7 @@ from vllm.config.cache import CacheConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
@@ -48,6 +49,7 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
QuarkMoEMethod,
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
QuarkW8A8Fp8MoEMethod,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
@@ -247,6 +249,26 @@ QTENSOR_CONFIGS = [
|
||||
act_quant_key=kFp8Dynamic128Sym,
|
||||
dispatch_cls=QuarkW8A8Fp8PerBlock,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_dynamic_block_fp32_moe",
|
||||
weight={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
},
|
||||
input_tensors={
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
},
|
||||
weight_quant_key=kFp8Static128BlockSym,
|
||||
act_quant_key=kFp8Dynamic128Sym,
|
||||
dispatch_cls=QuarkW8A8Fp8MoEMethod,
|
||||
),
|
||||
QTensorConfig(
|
||||
name="fp8_w8a8_block_static_input",
|
||||
weight={
|
||||
@@ -968,12 +990,111 @@ def test_quant_method_dispatch_instantiation(case, monkeypatch, default_vllm_con
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
# default_vllm_config carries no model, but the FP8 and OCP MX methods
|
||||
# read the model type off the HF config.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe."
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace())
|
||||
),
|
||||
)
|
||||
|
||||
layer = TestRoutedExperts()
|
||||
method = config.get_quant_method(layer, "experts")
|
||||
|
||||
assert isinstance(method, case.dispatch_cls)
|
||||
|
||||
|
||||
QUARK_MOE_MODULE = "vllm.model_executor.layers.quantization.quark.quark_moe"
|
||||
# validate_fp8_block_shape_moe imports this from vllm.distributed when called,
|
||||
# so the patch has to target the source module rather than a local binding.
|
||||
TP_WORLD_SIZE = "vllm.distributed.get_tensor_model_parallel_world_size"
|
||||
|
||||
|
||||
def _make_per_block_fp8_moe_method(
|
||||
activation_quant_key: QuantKey = kFp8Dynamic128Sym,
|
||||
) -> QuarkW8A8Fp8MoEMethod:
|
||||
return QuarkW8A8Fp8MoEMethod(
|
||||
_make_test_moe_config(),
|
||||
kFp8Static128BlockSym,
|
||||
activation_quant_key,
|
||||
)
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_moe_per_block_requires_dynamic_group_input():
|
||||
with (
|
||||
patch(
|
||||
f"{QUARK_MOE_MODULE}.select_fp8_moe_backend",
|
||||
return_value=(Mock(), Mock()),
|
||||
),
|
||||
patch(f"{QUARK_MOE_MODULE}.get_current_vllm_config"),
|
||||
pytest.raises(ValueError, match="per-block scales"),
|
||||
):
|
||||
_make_per_block_fp8_moe_method(kFp8StaticTensorSym)
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_moe_per_block_weight_shapes():
|
||||
with (
|
||||
patch(
|
||||
f"{QUARK_MOE_MODULE}.select_fp8_moe_backend",
|
||||
return_value=(Mock(), Mock()),
|
||||
),
|
||||
patch(f"{QUARK_MOE_MODULE}.get_current_vllm_config"),
|
||||
patch(TP_WORLD_SIZE, return_value=1),
|
||||
):
|
||||
method = _make_per_block_fp8_moe_method()
|
||||
assert method.block_quant
|
||||
assert method.weight_block_size == [128, 128]
|
||||
|
||||
layer = torch.nn.Module()
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=4,
|
||||
hidden_size=512,
|
||||
intermediate_size_per_partition=256,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
w13_num_shards = method.moe.w13_num_shards
|
||||
assert layer.weight_block_size == [128, 128]
|
||||
assert layer.w13_weight.shape == (4, w13_num_shards * 256, 512)
|
||||
assert layer.w2_weight.shape == (4, 512, 256)
|
||||
# Quark exports block scales as `weight_scale`, like the per-tensor and
|
||||
# per-channel schemes, so all schemes register the same parameter name.
|
||||
assert not hasattr(layer, "w13_weight_scale_inv")
|
||||
assert not hasattr(layer, "w2_weight_scale_inv")
|
||||
# One scale per 128x128 tile of each expert's weight.
|
||||
assert layer.w13_weight_scale.shape == (
|
||||
4,
|
||||
w13_num_shards * (256 // 128),
|
||||
512 // 128,
|
||||
)
|
||||
assert layer.w2_weight_scale.shape == (4, 512 // 128, 256 // 128)
|
||||
# The loader shards block scales on the block grid, not per row.
|
||||
for scale in (layer.w13_weight_scale, layer.w2_weight_scale):
|
||||
assert scale.quant_method == FusedMoeWeightScaleSupported.BLOCK.value
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_moe_per_block_rejects_misaligned_partition():
|
||||
with (
|
||||
patch(
|
||||
f"{QUARK_MOE_MODULE}.select_fp8_moe_backend",
|
||||
return_value=(Mock(), Mock()),
|
||||
),
|
||||
patch(f"{QUARK_MOE_MODULE}.get_current_vllm_config"),
|
||||
patch(TP_WORLD_SIZE, return_value=1),
|
||||
pytest.raises(ValueError, match="not divisible by"),
|
||||
):
|
||||
_make_per_block_fp8_moe_method().create_weights(
|
||||
torch.nn.Module(),
|
||||
num_experts=4,
|
||||
hidden_size=512,
|
||||
intermediate_size_per_partition=192,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_quark_fp8_w_per_tensor_a_per_tensor(
|
||||
kv_cache_dtype: str, monkeypatch, dist_init, workspace_init
|
||||
@@ -1111,6 +1232,42 @@ def test_quark_int8_w8a8_moe(vllm_runner, tp):
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp", [1])
|
||||
def test_quark_fp8_w8a8_per_block_moe(vllm_runner, tp):
|
||||
"""Test per-block (128x128) FP8 MoE quantization with a tiny Qwen3 MoE model."""
|
||||
model_path = "Adamji/tiny-qwen3-moe-fp8-per-block"
|
||||
with vllm_runner(
|
||||
model_path,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=tp,
|
||||
gpu_memory_utilization=0.1,
|
||||
) as llm:
|
||||
|
||||
def check_model(model):
|
||||
experts = model.model.layers[0].mlp.experts
|
||||
method = experts._quant_method
|
||||
assert isinstance(method, QuarkW8A8Fp8MoEMethod), (
|
||||
f"Expected QuarkW8A8Fp8MoEMethod, got {type(method)}"
|
||||
)
|
||||
assert method.weight_qscheme == "per_block"
|
||||
assert method.weight_block_size == [128, 128]
|
||||
|
||||
# hidden_size=128, moe_intermediate_size=256 and 4 experts, so one
|
||||
# scale per 128x128 tile, with w13 stacking gate on top of up.
|
||||
routed_experts = experts.routed_experts
|
||||
assert routed_experts.w13_weight_scale.shape == (4, 4, 1)
|
||||
assert routed_experts.w2_weight_scale.shape == (4, 1, 2)
|
||||
# Quark exports the block scales under the same name as the
|
||||
# per-tensor and per-channel schemes.
|
||||
assert not hasattr(routed_experts, "w13_weight_scale_inv")
|
||||
assert not hasattr(routed_experts, "w2_weight_scale_inv")
|
||||
|
||||
llm.apply_model(check_model)
|
||||
|
||||
output = llm.generate_greedy("Hello", max_tokens=4)
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (on_gfx950() or on_gfx942()),
|
||||
reason="Quark W4A8 (INT4-FP8) MoE requires the AITER kernel on gfx942/gfx950",
|
||||
|
||||
+5
-19
@@ -8,7 +8,6 @@ from compressed_tensors.quantization import (
|
||||
QuantizationStrategy,
|
||||
)
|
||||
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -31,6 +30,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
process_fp8_weight_tensor_strategy_moe,
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
@@ -129,28 +129,14 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
block_n, block_k = (
|
||||
self.weight_block_size[0],
|
||||
self.weight_block_size[1],
|
||||
)
|
||||
# NOTE: To ensure proper alignment of the block-wise quantization
|
||||
# scales, the output_size of the weights for both the gate and up
|
||||
# layers must be divisible by block_n.
|
||||
# Required by column parallel or enabling merged weights
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
# Required by row parallel
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
validate_fp8_block_shape_moe(
|
||||
intermediate_size_per_partition,
|
||||
self.weight_block_size,
|
||||
)
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
process_fp8_weight_tensor_strategy,
|
||||
process_fp8_weight_tensor_strategy_moe,
|
||||
validate_fp8_block_shape,
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
@@ -575,34 +576,23 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_block_shape is not None
|
||||
moe_block_shape = self.moe_block_shape
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
block_n, block_k = (
|
||||
self.weight_block_size[0],
|
||||
self.weight_block_size[1],
|
||||
)
|
||||
# NOTE: To ensure proper alignment of the block-wise quantization
|
||||
# scales, the output_size of the weights for both the gate and up
|
||||
# layers must be divisible by block_n.
|
||||
# Required by column parallel or enabling merged weights
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
if self.weight_scale_refine is None:
|
||||
validate_fp8_block_shape_moe(
|
||||
intermediate_size_per_partition,
|
||||
self.weight_block_size,
|
||||
)
|
||||
else:
|
||||
# Use the refined block grid for the scale parameters; the
|
||||
# loader upsamples the checkpoint scales accordingly.
|
||||
block_n, block_k = moe_block_shape
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
# Required by row parallel
|
||||
if self.weight_scale_refine is None:
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
block_n, block_k = moe_block_shape
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
block_n, block_k = moe_block_shape
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
block_n, block_k = moe_block_shape
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
|
||||
@@ -57,6 +57,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
select_nvfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
@@ -66,8 +69,10 @@ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
@@ -230,10 +235,12 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
kFp8StaticTensorSym,
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Dynamic128Sym,
|
||||
]
|
||||
supported_weight_quant_keys = [
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8Static128BlockSym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
@@ -244,14 +251,18 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
):
|
||||
super().__init__(moe, weight_quant_key, activation_quant_key)
|
||||
self.weight_dtype = "fp8"
|
||||
self.weight_qscheme = (
|
||||
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
|
||||
)
|
||||
self.input_qscheme = (
|
||||
"per_channel"
|
||||
if activation_quant_key == kFp8DynamicTokenSym
|
||||
else "per_tensor"
|
||||
)
|
||||
if weight_quant_key == kFp8StaticChannelSym:
|
||||
self.weight_qscheme = "per_channel"
|
||||
elif weight_quant_key == kFp8Static128BlockSym:
|
||||
self.weight_qscheme = "per_block"
|
||||
else:
|
||||
self.weight_qscheme = "per_tensor"
|
||||
if activation_quant_key == kFp8DynamicTokenSym:
|
||||
self.input_qscheme = "per_channel"
|
||||
elif activation_quant_key == kFp8Dynamic128Sym:
|
||||
self.input_qscheme = "per_group"
|
||||
else:
|
||||
self.input_qscheme = "per_tensor"
|
||||
self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
|
||||
per_tensor = (
|
||||
self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor"
|
||||
@@ -259,16 +270,31 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
per_channel = (
|
||||
self.weight_qscheme == "per_channel" and self.input_qscheme == "per_channel"
|
||||
)
|
||||
self.act_quant_group_shape = (
|
||||
GroupShape.PER_TOKEN if per_channel else GroupShape.PER_TENSOR
|
||||
per_block = (
|
||||
self.weight_qscheme == "per_block" and self.input_qscheme == "per_group"
|
||||
)
|
||||
if not (per_tensor or per_channel):
|
||||
if not (per_tensor or per_channel or per_block):
|
||||
raise ValueError(
|
||||
"For FP8 Fused MoE layers, only per-tensor and per-channel "
|
||||
"scales for weights and activations are supported. Found "
|
||||
f"{self.weight_qscheme}, {self.input_qscheme}"
|
||||
"For FP8 Fused MoE layers, only per-tensor, per-channel and "
|
||||
"per-block scales for weights and activations are supported. "
|
||||
f"Found {self.weight_qscheme}, {self.input_qscheme}"
|
||||
) # noqa E501
|
||||
|
||||
# One scale per (block_n, block_k) tile of the weight, so the block
|
||||
# shape is carried by the weight key's group shape.
|
||||
self.weight_block_size: list[int] | None = (
|
||||
list(weight_quant_key.scale.group_shape) if per_block else None
|
||||
)
|
||||
self.block_quant = self.weight_block_size is not None
|
||||
|
||||
if per_channel:
|
||||
self.act_quant_group_shape = GroupShape.PER_TOKEN
|
||||
elif per_block:
|
||||
assert self.weight_block_size is not None
|
||||
self.act_quant_group_shape = GroupShape(1, self.weight_block_size[1])
|
||||
else:
|
||||
self.act_quant_group_shape = GroupShape.PER_TENSOR
|
||||
|
||||
self.fp8_backend, self.experts_cls = select_fp8_moe_backend(
|
||||
config=moe,
|
||||
weight_key=self.weight_quant_key,
|
||||
@@ -293,6 +319,15 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
layer.weight_block_size = None
|
||||
params_dtype = torch.float8_e4m3fn
|
||||
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
block_n, block_k = self.weight_block_size[0], self.weight_block_size[1]
|
||||
validate_fp8_block_shape_moe(
|
||||
intermediate_size_per_partition,
|
||||
self.weight_block_size,
|
||||
)
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -368,6 +403,35 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
elif self.weight_qscheme == "per_block":
|
||||
# One scale per (block_n, block_k) tile of each expert's weight.
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards
|
||||
* ((intermediate_size_per_partition + block_n - 1) // block_n),
|
||||
(hidden_size + block_k - 1) // block_k,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
(hidden_size + block_n - 1) // block_n,
|
||||
(intermediate_size_per_partition + block_k - 1) // block_k,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
# Add PER-BLOCK quantization for RoutedExperts.weight_loader.
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
# INPUT_SCALES
|
||||
if self.static_input_scales:
|
||||
@@ -548,6 +612,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
a2_scale=layer.w2_input_scale,
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
block_shape=self.weight_block_size,
|
||||
per_act_token_quant=self.input_qscheme == "per_channel",
|
||||
per_out_ch_quant=self.weight_qscheme == "per_channel",
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
|
||||
@@ -1265,6 +1265,35 @@ def validate_fp8_block_shape(
|
||||
)
|
||||
|
||||
|
||||
def validate_fp8_block_shape_moe(
|
||||
intermediate_size_per_partition: int,
|
||||
block_size: list[int],
|
||||
) -> None:
|
||||
"""Validate fused MoE block quantization shapes for tensor parallelism."""
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
|
||||
# NOTE: To ensure proper alignment of the block-wise quantization
|
||||
# scales, the output_size of the weights for both the gate and up
|
||||
# layers must be divisible by block_n.
|
||||
# Required by column parallel or enabling merged weights
|
||||
if intermediate_size_per_partition % block_n != 0:
|
||||
raise ValueError(
|
||||
f"The output_size of gate's and up's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
if tp_size > 1 and intermediate_size_per_partition % block_k != 0:
|
||||
# Required by row parallel
|
||||
raise ValueError(
|
||||
f"The input_size of down's weight = "
|
||||
f"{intermediate_size_per_partition} is not divisible by "
|
||||
f"weight quantization block_k = {block_k}."
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_weight_parameter(
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
|
||||
Reference in New Issue
Block a user