[Quantization] Select per-token NVFP4 MoE backends explicitly (#57176)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
aoshen02
Claude Opus 5.5
parent
0f2a15c927
commit
9127170b88
@@ -24,7 +24,7 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_weight_quant,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4DynamicToken,
|
||||
kNvfp4Static,
|
||||
weight_amax,
|
||||
)
|
||||
@@ -100,7 +100,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=kNvfp4Dynamic,
|
||||
activation_key=kNvfp4DynamicToken,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
|
||||
@@ -208,6 +208,9 @@ kNvfp4DynamicGroupScale = ScaleDesc(FP8_DTYPE, False, GroupShape(1, 16))
|
||||
kNvfp4Dynamic = QuantKey(
|
||||
FP4_DTYPE, scale=kNvfp4DynamicGroupScale, scale2=kStaticTensorScale
|
||||
)
|
||||
kNvfp4DynamicToken = QuantKey(
|
||||
FP4_DTYPE, scale=kNvfp4DynamicGroupScale, scale2=kDynamicTokenScale
|
||||
)
|
||||
|
||||
kNvfp4StaticGroupScale = ScaleDesc(FP8_DTYPE, True, GroupShape(1, 16))
|
||||
kNvfp4Static = QuantKey(
|
||||
|
||||
Reference in New Issue
Block a user