[Bugfix] Disable shared expert overlap if Marlin MoE is used (#28410)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -424,6 +424,7 @@ class AWQMoEMethod(FusedMoEMethodBase):
|
||||
if self.quant_config.weight_bits != 4:
|
||||
raise ValueError("AWQMoEMethod only supports 4bit now.")
|
||||
self.quant_type = scalar_types.uint4
|
||||
self.use_marlin = True
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
|
||||
@@ -1342,6 +1342,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
f"{WNA16_SUPPORTED_BITS}",
|
||||
)
|
||||
self.quant_type = WNA16_SUPPORTED_TYPES_MAP[self.num_bits]
|
||||
self.use_marlin = True
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
|
||||
@@ -482,6 +482,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
|
||||
self.quant_type = scalar_types.uint8b128
|
||||
else:
|
||||
raise ValueError("GPTQMarlinMoEMethod only supports int4 and int8 now.")
|
||||
self.use_marlin = True
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
|
||||
@@ -216,6 +216,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.mxfp4_backend = get_mxfp4_backend(moe.is_lora_enabled)
|
||||
self.use_marlin = self.mxfp4_backend == Mxfp4Backend.MARLIN
|
||||
self.max_capture_size = (
|
||||
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user