[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
@@ -28,6 +28,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
import vllm.version
from vllm.config import ModelConfig
from vllm.config.utils import normalize_value
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.weight_cache.utils import (
format_socket_role_suffix,
@@ -227,12 +228,17 @@ def verify_peer_is_owner(conn: socket.socket) -> None:
raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}")
def _hash_quant_config(quant_config: Any) -> str:
if quant_config is None:
return ""
def _hash_quant_config(quant_config: Any, runtime_quant_config: Any = None) -> str:
if hasattr(quant_config, "to_dict"):
quant_config = quant_config.to_dict()
payload = json.dumps(quant_config, sort_keys=True, default=str)
payload = json.dumps(
{
"checkpoint": quant_config,
"runtime": normalize_value(runtime_quant_config),
},
sort_keys=True,
default=str,
)
return safe_hash(payload.encode(), usedforsecurity=False).hexdigest()
@@ -333,7 +339,10 @@ class WeightCacheKey:
pp_rank=pp_rank,
dtype=str(model_config.dtype),
quantization=model_config.quantization,
quant_config_hash=_hash_quant_config(quant_config),
quant_config_hash=_hash_quant_config(
quant_config=quant_config,
runtime_quant_config=model_config.quantization_config,
),
revision=model_config.revision,
vllm_version=vllm.version.__version__,
is_draft=is_draft,