diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 1ca28fbf01..78c64bac61 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -467,10 +467,17 @@ class QuarkConfig(QuantizationConfig): layer_name.replace(proj_name, shard_proj_name) for shard_proj_name in shard_proj_names ] - shard_configs = [ - self._find_matched_config(shard_name, module) - for shard_name in shard_names - ] + + shard_configs = [] + for shard_name in shard_names: + if shard_name == layer_name: + config = cast( + dict[str, Any], self.quant_config.get("global_quant_config") + ) + else: + config = self._find_matched_config(shard_name, module) + shard_configs.append(config) + if not all( deep_compare(q_config, shard_configs[0]) for q_config in shard_configs ):