[Quantization] Select per-token NVFP4 MoE backends explicitly (#57176)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matej Sirovatka
2026-09-23 07:30:58 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex aoshen02 Claude Opus 5.5
parent 0f2a15c927
commit 9127170b88
6 changed files with 97 additions and 21 deletions
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_weight_quant,
kNvfp4Dynamic,
kNvfp4DynamicToken,
kNvfp4Static,
weight_amax,
)
@@ -100,7 +100,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=kNvfp4Dynamic,
activation_key=kNvfp4DynamicToken,
)
def process_weights_after_loading(self, layer: Module) -> None:
@@ -208,6 +208,9 @@ kNvfp4DynamicGroupScale = ScaleDesc(FP8_DTYPE, False, GroupShape(1, 16))
kNvfp4Dynamic = QuantKey(
FP4_DTYPE, scale=kNvfp4DynamicGroupScale, scale2=kStaticTensorScale
)
kNvfp4DynamicToken = QuantKey(
FP4_DTYPE, scale=kNvfp4DynamicGroupScale, scale2=kDynamicTokenScale
)
kNvfp4StaticGroupScale = ScaleDesc(FP8_DTYPE, True, GroupShape(1, 16))
kNvfp4Static = QuantKey(