[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)
Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
OpenAI Codex
parent
b803edbc9f
commit
f38c6778ce
@@ -28,6 +28,7 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
import vllm.version
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.utils import normalize_value
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.weight_cache.utils import (
|
||||
format_socket_role_suffix,
|
||||
@@ -227,12 +228,17 @@ def verify_peer_is_owner(conn: socket.socket) -> None:
|
||||
raise PermissionError(f"Rejecting weight cache connection from uid {peer_uid}")
|
||||
|
||||
|
||||
def _hash_quant_config(quant_config: Any) -> str:
|
||||
if quant_config is None:
|
||||
return ""
|
||||
def _hash_quant_config(quant_config: Any, runtime_quant_config: Any = None) -> str:
|
||||
if hasattr(quant_config, "to_dict"):
|
||||
quant_config = quant_config.to_dict()
|
||||
payload = json.dumps(quant_config, sort_keys=True, default=str)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"checkpoint": quant_config,
|
||||
"runtime": normalize_value(runtime_quant_config),
|
||||
},
|
||||
sort_keys=True,
|
||||
default=str,
|
||||
)
|
||||
return safe_hash(payload.encode(), usedforsecurity=False).hexdigest()
|
||||
|
||||
|
||||
@@ -333,7 +339,10 @@ class WeightCacheKey:
|
||||
pp_rank=pp_rank,
|
||||
dtype=str(model_config.dtype),
|
||||
quantization=model_config.quantization,
|
||||
quant_config_hash=_hash_quant_config(quant_config),
|
||||
quant_config_hash=_hash_quant_config(
|
||||
quant_config=quant_config,
|
||||
runtime_quant_config=model_config.quantization_config,
|
||||
),
|
||||
revision=model_config.revision,
|
||||
vllm_version=vllm.version.__version__,
|
||||
is_draft=is_draft,
|
||||
|
||||
Reference in New Issue
Block a user