From 5b2a9422f0f4cbdd69a9fed1dc1605838314ff81 Mon Sep 17 00:00:00 2001 From: danisereb Date: Thu, 5 Feb 2026 19:25:55 +0200 Subject: [PATCH] [BugFix] Fix LoRA Fp8 (#33879) Signed-off-by: Daniel Serebrenik --- vllm/lora/layers/fused_moe.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index c2b35fbb10..4d4e053cff 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -130,14 +130,20 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): self.base_layer.ensure_moe_quant_config_init() quant_config = self.base_layer.quant_method.moe_quant_config - prepare_finalize = MoEPrepareAndFinalizeNoEP() - m_fused_moe_fn = FusedMoEModularKernel( - prepare_finalize, - self.base_layer.quant_method.select_gemm_impl( - prepare_finalize, self.base_layer - ), - self.base_layer.shared_experts, - ) + if getattr(self.base_layer.quant_method, "supports_internal_mk", False): + # Use the existing modular kernel from the quant method + m_fused_moe_fn = self.base_layer.quant_method.moe_mk + else: + # Create a new modular kernel via select_gemm_impl + prepare_finalize = MoEPrepareAndFinalizeNoEP() + m_fused_moe_fn = FusedMoEModularKernel( + prepare_finalize, + self.base_layer.quant_method.select_gemm_impl( + prepare_finalize, self.base_layer + ), + self.base_layer.shared_experts, + ) + if quant_config.use_mxfp4_w4a16: assert isinstance( m_fused_moe_fn.fused_experts, (MarlinExperts, UnfusedOAITritonExperts)