Refactor AWQ Marlin MoE onto modular WNA16 oracle (#42483)
Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com> Signed-off-by: Siddharth Bedekar <104613085+bedeks@users.noreply.github.com> Co-authored-by: Robert Shaw <robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
OpenAI Codex
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f5d3dc7115
commit
78e7a7b9b0
@@ -957,9 +957,14 @@ def awq_marlin_moe_quant_config(
|
||||
group_size: int,
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
a1_gscale: torch.Tensor | None = None,
|
||||
a2_gscale: torch.Tensor | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for awq marlin quantization.
|
||||
|
||||
a1_gscale / a2_gscale are optional global scales applied to activation
|
||||
quantization scales when Marlin runs with 8-bit activations.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
@@ -977,8 +982,8 @@ def awq_marlin_moe_quant_config(
|
||||
raise ValueError(f"Unsupported weight_bits: {weight_bits}")
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(dtype=None, shape=a_shape),
|
||||
_a2=FusedMoEQuantDesc(dtype=None, shape=a_shape),
|
||||
_a1=FusedMoEQuantDesc(dtype=None, shape=a_shape, alpha_or_gscale=a1_gscale),
|
||||
_a2=FusedMoEQuantDesc(dtype=None, shape=a_shape, alpha_or_gscale=a2_gscale),
|
||||
_w1=FusedMoEQuantDesc(weight_dtype, w_shape, w1_scale, None, w1_zp, w1_bias),
|
||||
_w2=FusedMoEQuantDesc(weight_dtype, w_shape, w2_scale, None, w2_zp, w2_bias),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user