iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix][Quantization] Fix dynamic INT8 W8A8 MoE config being built as W8A16 (#50833)

Browse Source 
Signed-off-by: Hank <hcc.mayday@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Hank_
2026-08-07 01:21:13 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent c84789c40b
commit b8db7f4abd
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/model_executor/layers/fused_moe/oracle/int8.py
+5 -4
View File
@@ -183,9 +183,10 @@ def make_int8_moe_quant_config(
per_act_token_quant: bool = False,
layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
assert (a1_scale is None and a2_scale is None) or (
a1_scale is not None and a2_scale is not None
), "a1_scale and a2_scale must both be provided or both be None"
if (a1_scale is None) != (a2_scale is None):
raise ValueError("a1_scale and a2_scale must both be provided or both be None")
scales_absent = a1_scale is None and a2_scale is None
if int8_backend == Int8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -196,7 +197,7 @@ def make_int8_moe_quant_config(
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
if a1_scale is None or a2_scale is None:
if scales_absent and not per_act_token_quant:
return int8_w8a16_moe_quant_config(
w1_scale=w1_scale,
w2_scale=w2_scale,