refactor: Change scaling factors calculation for flashinfer FusedMoE (#22812)
Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -24,8 +24,8 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig, QuantizeMethodBase)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
apply_flashinfer_per_tensor_scale_fp8, rotate_flashinfer_fp8_moe_weights,
|
||||
swap_w13_to_w31)
|
||||
apply_flashinfer_per_tensor_scale_fp8, register_moe_scaling_factors,
|
||||
rotate_flashinfer_fp8_moe_weights, swap_w13_to_w31)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_col_major_tma_aligned_tensor, requant_weight_ue8m0_inplace)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
@@ -694,6 +694,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w2_weight = layer.w2_weight.data
|
||||
w2_weight_scale_inv = layer.w2_weight_scale_inv.data
|
||||
if not self.block_quant:
|
||||
register_moe_scaling_factors(layer)
|
||||
rotate_flashinfer_fp8_moe_weights(w13_weight, w2_weight)
|
||||
else:
|
||||
w13_weight = layer.w13_weight.data
|
||||
|
||||
Reference in New Issue
Block a user