[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Misha Goin
parent
e2414ec1db
commit
9f3d70e081
@@ -9,6 +9,7 @@ import torch
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -42,6 +43,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
FP8 with block-32 scales) during weight loading.
|
||||
"""
|
||||
|
||||
activation_quant_key: QuantKey | None = None
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
Reference in New Issue
Block a user