[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
@@ -28,6 +28,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
import vllm.version
from vllm.config import ModelConfig
from vllm.config.utils import normalize_value
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_cache.utils import (
format_socket_role_suffix,
@@ -227,12 +228,17 @@ def verify_peer_is_owner(conn: socket.socket) -> None:
raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}")
def _hash_quant_config(quant_config: Any) -> str:
if quant_config is None:
return ""
def _hash_quant_config(quant_config: Any, runtime_quant_config: Any = None) -> str:
if hasattr(quant_config, "to_dict"):
quant_config = quant_config.to_dict()
payload = json.dumps(quant_config, sort_keys=True, default=str)
payload = json.dumps(
{
"checkpoint": quant_config,
"runtime": normalize_value(runtime_quant_config),
},
sort_keys=True,
default=str,
)
return safe_hash(payload.encode(), usedforsecurity=False).hexdigest()
@@ -333,7 +339,10 @@ class WeightCacheKey:
pp_rank=pp_rank,
dtype=str(model_config.dtype),
quantization=model_config.quantization,
quant_config_hash=_hash_quant_config(quant_config),
quant_config_hash=_hash_quant_config(
quant_config=quant_config,
runtime_quant_config=model_config.quantization_config,
),
revision=model_config.revision,
vllm_version=vllm.version.__version__,
is_draft=is_draft,
@@ -200,6 +200,9 @@ def get_quant_config(
QuantizationConfigArgs,
resolve_quantization_config,
)
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptQuantConfigBase,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
@@ -220,6 +223,14 @@ def get_quant_config(
)
return checkpoint_config
def make_checkpoint_config(config: dict[str, Any]) -> QuantizationConfig:
if issubclass(quant_cls, ModelOptQuantConfigBase):
assert online_args is None or isinstance(
online_args, QuantizationConfigArgs
)
config = {**config, "_online_quantization_args": online_args}
return maybe_compose_online_quantization(quant_cls.from_config(config))
# Read the quantization config from the HF model config, if available.
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
# some vision model may keep quantization_config in their text_config
@@ -260,9 +271,7 @@ def get_quant_config(
):
pass # fall through to file-based loading below
else:
return maybe_compose_online_quantization(
quant_cls.from_config(hf_quant_config)
)
return make_checkpoint_config(hf_quant_config)
# if hf_quant_config is None, we will try to get config from
# hf_overrides
@@ -376,14 +385,14 @@ def get_quant_config(
if model_config.quantization in ("modelopt", "modelopt_mixed"):
if config.get("producer", {}).get("name") == "modelopt":
return maybe_compose_online_quantization(quant_cls.from_config(config))
return make_checkpoint_config(config)
else:
raise ValueError(
f"Unsupported quantization config"
f" found for {model_config.quantization} in {f}."
)
return maybe_compose_online_quantization(quant_cls.from_config(config))
return make_checkpoint_config(config)
def get_sparse_attention_config(