[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-08 12:04:46 -04:00
committed by GitHub
co-authored by OpenAI Codex Misha Goin
parent e2414ec1db
commit 9f3d70e081
24 changed files with 984 additions and 73 deletions
+23 -6
View File
@@ -15,7 +15,11 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.config.quantization import (
QuantizationConfigArgs,
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization import get_quantization_config
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.platforms import current_platform
@@ -40,10 +44,17 @@ def _limit_num_hidden_layers(
original_load_weights = model.load_weights
def should_load_weight(name: str) -> bool:
for prefix in ("model.layers.", "layers."):
if name.startswith(prefix):
layer_idx = int(name.removeprefix(prefix).split(".", 1)[0])
return layer_idx < num_hidden_layers
layers_prefix = "layers."
if name.startswith(layers_prefix):
layer_name = name.removeprefix(layers_prefix)
elif ".layers." in name:
layer_name = name.split(".layers.", maxsplit=1)[1]
else:
return True
layer_idx_str = layer_name.split(".", maxsplit=1)[0]
if layer_idx_str.isdigit():
return int(layer_idx_str) < num_hidden_layers
return True
def load_weights(weights):
@@ -71,13 +82,19 @@ def load_model_without_vllm_runner(
quantization=quantization,
**(model_config_kwargs or {}),
)
model_config.quantization_config = resolve_quantization_config(
model_config.quantization, model_config.quantization_config
)
vllm_config_args = dict(vllm_config_kwargs or {})
vllm_config_args.setdefault(
"compilation_config", CompilationConfig(mode=CompilationMode.NONE)
)
vllm_config = VllmConfig(model_config=model_config, **vllm_config_args)
hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {}
num_hidden_layers = hf_overrides.get("num_hidden_layers")
text_config_overrides = hf_overrides.get("text_config") or {}
num_hidden_layers = text_config_overrides.get(
"num_hidden_layers", hf_overrides.get("num_hidden_layers")
)
with set_current_vllm_config(vllm_config):
model_loader = model_loader_cls(vllm_config.load_config)