[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
e229fdbc87
commit
fcde8e1460
@@ -32,4 +32,4 @@ quack-kernels>=0.6.1 # Required for CUTLASS DSL 4.6 by MSA
|
||||
tokenspeed-mla==0.1.8; platform_system == "Linux"
|
||||
|
||||
# Humming kernels for quantization gemm
|
||||
humming-kernels[cu13]==0.1.10
|
||||
humming-kernels[cu13]==0.1.12
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
|
||||
accuracy_threshold: 0.29
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -1,5 +1,5 @@
|
||||
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
|
||||
accuracy_threshold: 0.85
|
||||
accuracy_threshold: 0.78
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
|
||||
@@ -11,3 +11,4 @@ Qwen3-30B-A3B-NVFP4-humming.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-FP8-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming.yaml
|
||||
NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml
|
||||
|
||||
@@ -1265,11 +1265,10 @@ def _make_humming_indexed_experts(activation: MoEActivation):
|
||||
layer.intermediate_size_per_partition = intermediate_size
|
||||
quant_config = humming_utils.get_humming_moe_quant_config(layer)
|
||||
experts = HummingIndexedExperts(
|
||||
layer,
|
||||
moe_config,
|
||||
quant_config,
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
return experts
|
||||
return experts, layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
@@ -1328,8 +1327,7 @@ def test_humming_delegates_to_instance_activation():
|
||||
def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
|
||||
from vllm.utils import humming
|
||||
|
||||
experts = _make_humming_indexed_experts(activation)
|
||||
layer = experts.layer
|
||||
experts, layer = _make_humming_indexed_experts(activation)
|
||||
moe_config = experts.moe_config
|
||||
top_k = moe_config.experts_per_token
|
||||
num_experts = moe_config.num_experts
|
||||
@@ -1337,7 +1335,7 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
|
||||
intermediate_size = moe_config.intermediate_size
|
||||
gate_up_size = intermediate_size * 2 if activation.is_gated else intermediate_size
|
||||
|
||||
w13_meta, w2_meta = (layer.humming_metas[name] for name in ("w13", "w2"))
|
||||
w13_meta, w2_meta = (experts.humming_configs[name] for name in ("w13", "w2"))
|
||||
for meta in (w13_meta, w2_meta):
|
||||
assert meta.a_dtype == humming.dtypes.bfloat16
|
||||
assert meta.b_dtype == humming.dtypes.float4e2m1
|
||||
@@ -1364,7 +1362,7 @@ def test_humming_indexed_writes_supplied_output_buffer():
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
activation = MoEActivation.SILU
|
||||
experts = _make_humming_indexed_experts(activation)
|
||||
experts, layer = _make_humming_indexed_experts(activation)
|
||||
moe_config = experts.moe_config
|
||||
num_tokens = 1
|
||||
top_k = moe_config.experts_per_token
|
||||
@@ -1382,7 +1380,7 @@ def test_humming_indexed_writes_supplied_output_buffer():
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
dtype = experts.layer.params_dtype
|
||||
dtype = layer.params_dtype
|
||||
workspace13 = torch.empty(workspace13_shape, dtype=dtype, device=device)
|
||||
workspace2 = torch.empty(workspace2_shape, dtype=dtype, device=device)
|
||||
hidden_states = torch.ones((num_tokens, hidden_size), dtype=dtype, device=device)
|
||||
@@ -1394,14 +1392,12 @@ def test_humming_indexed_writes_supplied_output_buffer():
|
||||
device=device,
|
||||
)
|
||||
topk_ids = torch.arange(top_k, dtype=torch.int32, device=device).unsqueeze(0)
|
||||
unused = torch.empty((num_experts, 0), device=device)
|
||||
|
||||
with set_forward_context(None, vllm_config, num_tokens=num_tokens):
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=unused,
|
||||
w2=unused,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
|
||||
@@ -1489,7 +1489,6 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
mxfp4_backend=backend,
|
||||
experts_cls=experts_cls,
|
||||
routing_tables=None,
|
||||
layer=None,
|
||||
)
|
||||
|
||||
# Create inputs
|
||||
|
||||
@@ -161,7 +161,6 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
|
||||
assert method.moe_kernel is kernel
|
||||
assert captured["backend"] == WNA16MoEBackend.HUMMING
|
||||
assert captured["layer"] is layer
|
||||
|
||||
|
||||
def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
|
||||
@@ -198,7 +197,9 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
humming_utils,
|
||||
"get_humming_moe_quant_config",
|
||||
lambda actual_layer: quant_config if actual_layer is layer else None,
|
||||
lambda actual_layer, *args, **kwargs: (
|
||||
quant_config if actual_layer is layer else None
|
||||
),
|
||||
)
|
||||
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config
|
||||
|
||||
@@ -28,7 +28,8 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
|
||||
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
|
||||
@@ -46,9 +47,11 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
|
||||
prepare_humming_layer(
|
||||
self.layer_config = prepare_humming_linear_layer_config(
|
||||
layer, quant_config, input_quant_config=input_quant_config
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -56,12 +59,14 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
)
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -5,11 +5,14 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
@@ -26,6 +29,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -56,7 +62,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -64,12 +72,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -4,10 +4,13 @@
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
@@ -22,6 +25,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -44,7 +50,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -52,12 +60,10 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -5,9 +5,12 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_layer,
|
||||
apply_humming_linear,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
|
||||
@@ -24,6 +27,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -54,7 +60,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
1.0 / layer.weight_global_scale, requires_grad=False
|
||||
)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -62,12 +70,10 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -5,10 +5,13 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
@@ -30,6 +33,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -58,9 +64,6 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
scale_group_shape = self.config.weight_quant_key.scale.group_shape
|
||||
if scale_group_shape.is_per_tensor():
|
||||
quant_config["weight_scale_type"] = "tensor"
|
||||
if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
|
||||
del name_map["weight_scale"]
|
||||
name_map["global_scale"] = "weight_scale"
|
||||
elif scale_group_shape.is_per_channel():
|
||||
quant_config["weight_scale_type"] = "channel"
|
||||
elif scale_group_shape.is_per_group():
|
||||
@@ -76,7 +79,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
name_map["weight_scale"] = "weight_scale_inv"
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -84,15 +89,13 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
@@ -105,7 +108,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
raise NotImplementedError(
|
||||
"HummingFP8ScaledMMLinearKernel uses apply_weights directly"
|
||||
)
|
||||
|
||||
|
||||
class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
@@ -118,6 +123,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -137,7 +145,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
weight.data = weight.data + 128
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -145,12 +155,10 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import json
|
||||
import math
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -62,8 +62,15 @@ from vllm.utils.import_utils import has_humming
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
HummingMoEQuantConfig,
|
||||
)
|
||||
from vllm.utils.humming import (
|
||||
GemmType as HummingGemmType,
|
||||
)
|
||||
from vllm.utils.humming import (
|
||||
LayerConfig as HummingLayerConfig,
|
||||
)
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -88,15 +95,20 @@ def get_humming_moe_gemm_type() -> str:
|
||||
class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
def __init__(
|
||||
self,
|
||||
layer: "RoutedExperts",
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
self.layer = layer
|
||||
self.num_experts = self.layer.local_num_experts
|
||||
self.global_num_experts = self.layer.global_num_experts
|
||||
humming_quant_config = cast("HummingMoEQuantConfig", quant_config)
|
||||
self.humming_configs: dict[str, HummingLayerConfig] = {
|
||||
"w13": humming_quant_config.w1_humming_config,
|
||||
"w2": humming_quant_config.w2_humming_config,
|
||||
}
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
|
||||
self.num_experts = moe_config.num_local_experts
|
||||
self.global_num_experts = moe_config.num_experts
|
||||
self.quant_config = quant_config
|
||||
self.init_humming_moe()
|
||||
|
||||
if self.is_batched():
|
||||
@@ -111,31 +123,73 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
def init_humming_moe(self):
|
||||
from vllm.utils.humming import HummingMethod
|
||||
from vllm.utils.humming import get_heuristics_config
|
||||
|
||||
self.compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": self.humming_gemm_type().value,
|
||||
}
|
||||
self.w13_tuning_config = HummingMethod.get_default_tuning_configs(
|
||||
layer=self.layer,
|
||||
self.w13_tuning_config = get_heuristics_config(
|
||||
layer_config=self.humming_configs["w13"],
|
||||
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
|
||||
gemm_type=self.humming_gemm_type(),
|
||||
sublayer_name="w13",
|
||||
)
|
||||
self.w2_tuning_config = HummingMethod.get_default_tuning_configs(
|
||||
layer=self.layer,
|
||||
self.w2_tuning_config = get_heuristics_config(
|
||||
layer_config=self.humming_configs["w2"],
|
||||
use_f16_accum=envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
use_batch_invariant=envs.VLLM_BATCH_INVARIANT,
|
||||
gemm_type=self.humming_gemm_type(),
|
||||
sublayer_name="w2",
|
||||
)
|
||||
self.compute_config_str = json.dumps(self.compute_config)
|
||||
self.w13_tuning_config_str = json.dumps(self.w13_tuning_config)
|
||||
self.w2_tuning_config_str = json.dumps(self.w2_tuning_config)
|
||||
|
||||
def quantize_input(
|
||||
self,
|
||||
sublayer_name: str,
|
||||
inputs: torch.Tensor,
|
||||
quanted_input: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
from vllm.utils.humming import may_quant_input
|
||||
|
||||
return may_quant_input(
|
||||
self.humming_configs[sublayer_name],
|
||||
inputs=inputs,
|
||||
quanted_input=quanted_input,
|
||||
)
|
||||
|
||||
def humming_forward(
|
||||
self,
|
||||
sublayer_name: str,
|
||||
inputs: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
input_scale: torch.Tensor | None,
|
||||
outputs: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import humming_forward
|
||||
|
||||
is_w13 = sublayer_name == "w13"
|
||||
return humming_forward(
|
||||
self.humming_configs[sublayer_name],
|
||||
inputs=inputs,
|
||||
weight=weight,
|
||||
weight_scale=(
|
||||
self.quant_config.w1_scale if is_w13 else self.quant_config.w2_scale
|
||||
),
|
||||
zero_point=(self.quant_config.w1_zp if is_w13 else self.quant_config.w2_zp),
|
||||
bias=(self.quant_config.w1_bias if is_w13 else self.quant_config.w2_bias),
|
||||
weight_scale_2=(
|
||||
self.quant_config.g1_alphas if is_w13 else self.quant_config.g2_alphas
|
||||
),
|
||||
input_scale=input_scale,
|
||||
outputs=outputs,
|
||||
locks=self.locks,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
@@ -218,8 +272,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""
|
||||
Humming kernels handle input quantization internally via
|
||||
HummingMethod.may_quant_input() in the apply() method.
|
||||
Humming kernels handle input quantization internally in apply().
|
||||
|
||||
This property tells the prepare/finalize step to skip input
|
||||
quantization (by setting defer_input_quant=True) and pass
|
||||
@@ -255,6 +308,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_batch_invariance() -> bool:
|
||||
return True
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
@@ -262,10 +319,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
from vllm.utils.humming import HummingLayerMeta
|
||||
|
||||
meta1: HummingLayerMeta = self.layer.humming_metas["w13"]
|
||||
meta2: HummingLayerMeta = self.layer.humming_metas["w2"]
|
||||
meta1 = self.humming_configs["w13"]
|
||||
meta2 = self.humming_configs["w2"]
|
||||
|
||||
assert meta1.num_experts == meta2.num_experts
|
||||
|
||||
@@ -282,25 +337,25 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
assert a1.size(0) == num_experts
|
||||
num_tokens = a1.size(1)
|
||||
|
||||
return (
|
||||
meta1.num_experts,
|
||||
num_tokens,
|
||||
# Logical intermediate width for both gated and non-gated activations
|
||||
self.layer.intermediate_size_per_partition,
|
||||
meta1.shape_k,
|
||||
top_k,
|
||||
)
|
||||
intermediate_dim = meta2.shape_k - meta2.pad_shape_k
|
||||
hidden_dim = meta1.shape_k - meta1.pad_shape_k
|
||||
return meta1.num_experts, num_tokens, intermediate_dim, hidden_dim, top_k
|
||||
|
||||
def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
from vllm.utils.humming import dtypes
|
||||
|
||||
num_experts = self.num_experts
|
||||
N = self.layer.intermediate_size_per_partition
|
||||
K = self.layer.hidden_size
|
||||
w13_config = self.humming_configs["w13"]
|
||||
w2_config = self.humming_configs["w2"]
|
||||
gate_up_dim = w13_config.shape_n - w13_config.pad_shape_n
|
||||
intermediate_dim = w2_config.shape_k - w2_config.pad_shape_k
|
||||
K = w13_config.shape_k - w13_config.pad_shape_k
|
||||
assert isinstance(num_experts, int)
|
||||
assert isinstance(N, int)
|
||||
assert isinstance(gate_up_dim, int)
|
||||
assert isinstance(intermediate_dim, int)
|
||||
assert isinstance(K, int)
|
||||
assert intermediate_dim == self.adjust_N_for_activation(gate_up_dim, activation)
|
||||
|
||||
# hidden_states
|
||||
# (-> quanted_gate_up_input) (if not BF16/FP16 activation)
|
||||
@@ -310,7 +365,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
# -> down_output
|
||||
# (-> output) (if not is_batched)
|
||||
# Neighboring nodes are required to utilize distinct workspaces.
|
||||
# The output must be derived from workspace1.
|
||||
# The final output buffer is supplied by the modular kernel and may
|
||||
# alias workspace1.
|
||||
|
||||
output_shape: tuple[int, ...]
|
||||
if self.is_batched():
|
||||
@@ -327,10 +383,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
real_shape_m = M * topk
|
||||
output_shape = (M, K)
|
||||
|
||||
gate_up_size = N * (2 if activation.is_gated else 1)
|
||||
down_input_size = N
|
||||
a_dtype = self.layer.humming_metas["w13"].a_dtype
|
||||
c_dtype = self.layer.humming_metas["w13"].c_dtype
|
||||
a_dtype = self.humming_configs["w13"].a_dtype
|
||||
c_dtype = self.humming_configs["w13"].c_dtype
|
||||
num_bits = a_dtype.num_bits
|
||||
torch_dtype_map = {
|
||||
dtypes.float16: torch.float16,
|
||||
@@ -348,15 +402,15 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
"dtype": torch_dtype_map[a_dtype],
|
||||
},
|
||||
"gate_up_output": {
|
||||
"shape": (real_shape_m, gate_up_size),
|
||||
"shape": (real_shape_m, gate_up_dim),
|
||||
"dtype": torch_dtype_map[c_dtype],
|
||||
},
|
||||
"activation_output": {
|
||||
"shape": (real_shape_m, down_input_size),
|
||||
"shape": (real_shape_m, intermediate_dim),
|
||||
"dtype": torch_dtype_map[c_dtype],
|
||||
},
|
||||
"quanted_down_input": {
|
||||
"shape": (real_shape_m, down_input_size),
|
||||
"shape": (real_shape_m, intermediate_dim),
|
||||
"dtype": torch_dtype_map[a_dtype],
|
||||
},
|
||||
"down_output": {
|
||||
@@ -414,7 +468,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
output_key = "down_output" if self.is_batched() else "output"
|
||||
output_shape = buffer_metas[output_key]["shape"]
|
||||
elem_size = self.layer.params_dtype.itemsize
|
||||
elem_size = self.moe_config.in_dtype.itemsize
|
||||
|
||||
return (
|
||||
(workspace1_nbytes // elem_size,),
|
||||
@@ -438,7 +492,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
def make_workspaces(self, M: int, topk: int, activation: MoEActivation):
|
||||
shapes = self._workspace_shapes(M, topk, activation)
|
||||
workspace1_shape, workspace2_shape, output_shape = shapes
|
||||
torch_dtype = self.layer.params_dtype
|
||||
torch_dtype = self.moe_config.in_dtype
|
||||
workspace1, workspace2 = current_workspace_manager().get_simultaneous(
|
||||
(workspace1_shape, torch_dtype),
|
||||
(workspace2_shape, torch_dtype),
|
||||
@@ -600,12 +654,11 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
"""
|
||||
Standard apply implementation for Humming indexed experts.
|
||||
|
||||
Note: Humming kernels handle weights and quantization internally through
|
||||
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
|
||||
The output is written into workspace13 via the buffer management.
|
||||
Humming performs activation quantization internally and consumes the
|
||||
weights supplied by the modular kernel interface.
|
||||
Results are written to the output buffer supplied by the modular
|
||||
kernel, which may alias workspace13.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
|
||||
@@ -616,6 +669,7 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
topk_ids.size(1),
|
||||
activation,
|
||||
)
|
||||
buffers["output"] = output
|
||||
|
||||
moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs(
|
||||
topk_ids=topk_ids,
|
||||
@@ -623,19 +677,18 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
layer=self.layer,
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w13",
|
||||
inputs=hidden_states,
|
||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
HummingMethod.forward_layer(
|
||||
layer=self.layer,
|
||||
self.humming_forward(
|
||||
"w13",
|
||||
inputs=inputs,
|
||||
weight=w1,
|
||||
input_scale=input_scale,
|
||||
outputs=buffers["gate_up_output"],
|
||||
sublayer_name="w13",
|
||||
**moe_kwargs1,
|
||||
)
|
||||
|
||||
@@ -645,19 +698,18 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
layer=self.layer,
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w2",
|
||||
inputs=buffers["activation_output"],
|
||||
quanted_input=buffers.get("quanted_down_input", None),
|
||||
sublayer_name="w2",
|
||||
)
|
||||
|
||||
HummingMethod.forward_layer(
|
||||
layer=self.layer,
|
||||
self.humming_forward(
|
||||
"w2",
|
||||
inputs=inputs,
|
||||
weight=w2,
|
||||
input_scale=input_scale,
|
||||
outputs=buffers["down_output"].view(-1, hidden_states.size(-1)),
|
||||
sublayer_name="w2",
|
||||
**moe_kwargs2,
|
||||
)
|
||||
|
||||
@@ -705,12 +757,11 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
"""
|
||||
Standard apply implementation for Humming grouped experts.
|
||||
|
||||
Note: Humming kernels handle weights and quantization internally through
|
||||
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
|
||||
The output is written into workspace13 via the buffer management.
|
||||
Humming performs activation quantization internally and consumes the
|
||||
weights supplied by the modular kernel interface.
|
||||
Results are written to the output buffer supplied by the modular
|
||||
kernel, which may alias workspace13.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
||||
@@ -722,6 +773,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
topk_ids.size(1),
|
||||
activation,
|
||||
)
|
||||
buffers["output"] = output
|
||||
|
||||
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
@@ -733,23 +785,22 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
scratch=self._get_permute_scratch(),
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
layer=self.layer,
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w13",
|
||||
inputs=hidden_states,
|
||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
HummingMethod.forward_layer(
|
||||
layer=self.layer,
|
||||
self.humming_forward(
|
||||
"w13",
|
||||
inputs=inputs,
|
||||
weight=w1,
|
||||
input_scale=input_scale,
|
||||
outputs=buffers["gate_up_output"],
|
||||
valid_shape_m=valid_shape_m,
|
||||
expert_layout=expert_first_token_offset,
|
||||
compute_config=self.compute_config_str,
|
||||
tuning_config=self.w13_tuning_config_str,
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
self.apply_activation(
|
||||
@@ -758,23 +809,22 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
layer=self.layer,
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w2",
|
||||
inputs=buffers["activation_output"],
|
||||
quanted_input=buffers.get("quanted_down_input", None),
|
||||
sublayer_name="w2",
|
||||
)
|
||||
|
||||
HummingMethod.forward_layer(
|
||||
layer=self.layer,
|
||||
self.humming_forward(
|
||||
"w2",
|
||||
inputs=inputs,
|
||||
weight=w2,
|
||||
input_scale=input_scale,
|
||||
outputs=buffers["down_output"],
|
||||
valid_shape_m=valid_shape_m,
|
||||
expert_layout=expert_first_token_offset,
|
||||
compute_config=self.compute_config_str,
|
||||
tuning_config=self.w2_tuning_config_str,
|
||||
sublayer_name="w2",
|
||||
)
|
||||
|
||||
moe_unpermute(
|
||||
@@ -821,12 +871,11 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
"""
|
||||
Standard apply implementation for Humming batched grouped experts.
|
||||
|
||||
Note: Humming kernels handle weights and quantization internally through
|
||||
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
|
||||
The output is written into workspace13 via the buffer management.
|
||||
Humming performs activation quantization internally and consumes the
|
||||
weights supplied by the modular kernel interface.
|
||||
Results are written to the output buffer supplied by the modular
|
||||
kernel, which may alias workspace13.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
assert not apply_router_weight_on_input
|
||||
assert expert_tokens_meta is not None
|
||||
|
||||
@@ -841,24 +890,24 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
topk_ids.size(1),
|
||||
activation,
|
||||
)
|
||||
buffers["down_output"] = output
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
layer=self.layer,
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w13",
|
||||
inputs=hidden_states,
|
||||
quanted_input=buffers.get("quanted_gate_up_input", None),
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
HummingMethod.forward_layer(
|
||||
layer=self.layer,
|
||||
self.humming_forward(
|
||||
"w13",
|
||||
inputs=inputs,
|
||||
weight=w1,
|
||||
input_scale=input_scale,
|
||||
outputs=buffers["gate_up_output"],
|
||||
valid_shape_m=valid_shape_m,
|
||||
expert_layout=expert_num_tokens,
|
||||
compute_config=self.compute_config_str,
|
||||
tuning_config=self.w13_tuning_config_str,
|
||||
sublayer_name="w13",
|
||||
)
|
||||
|
||||
self.apply_activation(
|
||||
@@ -867,21 +916,20 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
layer=self.layer,
|
||||
inputs, input_scale = self.quantize_input(
|
||||
"w2",
|
||||
inputs=buffers["activation_output"],
|
||||
quanted_input=buffers.get("quanted_down_input", None),
|
||||
sublayer_name="w2",
|
||||
)
|
||||
|
||||
HummingMethod.forward_layer(
|
||||
layer=self.layer,
|
||||
self.humming_forward(
|
||||
"w2",
|
||||
inputs=inputs,
|
||||
weight=w2,
|
||||
input_scale=input_scale,
|
||||
outputs=output.view(-1, hidden_states.size(-1)),
|
||||
valid_shape_m=valid_shape_m,
|
||||
expert_layout=expert_num_tokens,
|
||||
compute_config=self.compute_config_str,
|
||||
tuning_config=self.w2_tuning_config_str,
|
||||
sublayer_name="w2",
|
||||
)
|
||||
|
||||
@@ -619,7 +619,12 @@ def make_fp8_moe_quant_config(
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
# Flashinfer CUTLASS or HPC per-tensor uses single dq scale
|
||||
# (alpha = w_scale * a_scale) and inverse a2 scale.
|
||||
@@ -694,7 +699,6 @@ def make_fp8_moe_kernel(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
fp8_backend: Fp8MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
@@ -708,11 +712,6 @@ def make_fp8_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if fp8_backend == Fp8MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -722,13 +721,11 @@ def make_fp8_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -195,7 +195,12 @@ def make_int8_moe_quant_config(
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
if scales_absent and not per_act_token_quant:
|
||||
return int8_w8a16_moe_quant_config(
|
||||
@@ -251,9 +256,9 @@ def convert_to_int8_moe_kernel_format(
|
||||
)
|
||||
|
||||
assert layer is not None
|
||||
# Humming reads canonical CT scales (w*_weight_scale) from the layer.
|
||||
# Online int8 produces per-channel (E, N) w*_scale; expose them as the
|
||||
# (E, N, 1) w*_weight_scale humming's loader expects.
|
||||
# The Humming conversion expects canonical CT scale parameter names.
|
||||
# Online int8 produces per-channel (E, N) w*_scale; expose them as
|
||||
# (E, N, 1) w*_weight_scale tensors before conversion.
|
||||
for sub in ("w13", "w2"):
|
||||
if hasattr(layer, f"{sub}_weight_scale"):
|
||||
continue
|
||||
@@ -279,7 +284,6 @@ def make_int8_moe_kernel(
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
@@ -293,11 +297,6 @@ def make_int8_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if int8_backend == Int8MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -307,13 +306,11 @@ def make_int8_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -358,7 +358,6 @@ def make_wna16_moe_kernel(
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
|
||||
layer: torch.nn.Module | None = None,
|
||||
is_k_full: bool = False,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
@@ -410,10 +409,7 @@ def make_wna16_moe_kernel(
|
||||
logger.info_once("Using %s", experts_cls.__name__, scope="local")
|
||||
|
||||
extra_args: dict[str, Any] = {}
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_args = {"layer": layer}
|
||||
elif issubclass(experts_cls, MarlinExpertsBase):
|
||||
if issubclass(experts_cls, MarlinExpertsBase):
|
||||
extra_args = {
|
||||
"w13_g_idx": w13_g_idx,
|
||||
"w2_g_idx": w2_g_idx,
|
||||
|
||||
@@ -1788,7 +1788,6 @@ def make_mxfp4_moe_kernel(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: RoutedExperts | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Create a FusedMoEKernel for the given MXFP4 backend."""
|
||||
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
|
||||
@@ -1805,11 +1804,6 @@ def make_mxfp4_moe_kernel(
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
logger.info_once("Using %s", experts_cls.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs["layer"] = layer
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -1819,13 +1813,11 @@ def make_mxfp4_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_kwargs,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -398,10 +398,10 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
a2_scale = None
|
||||
w13 = layer.w13_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
w13_scale_2 = getattr(layer, "w13_global_scale", None)
|
||||
w13_scale_2 = getattr(layer, "w13_weight_scale_2", None)
|
||||
w2 = layer.w2_weight
|
||||
w2_scale = layer.w2_weight_scale
|
||||
w2_scale_2 = getattr(layer, "w2_global_scale", None)
|
||||
w2_scale_2 = getattr(layer, "w2_weight_scale_2", None)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.MARLIN:
|
||||
a13_scale = None
|
||||
a2_scale = None
|
||||
@@ -485,7 +485,12 @@ def make_nvfp4_moe_quant_config(
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif backend == NvFp4MoeBackend.MARLIN:
|
||||
return nvfp4_w4a16_moe_quant_config(
|
||||
g1_alphas=w13_scale_2,
|
||||
@@ -544,7 +549,6 @@ def make_nvfp4_moe_kernel(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
backend: NvFp4MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
per_token_activation: bool = False,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
@@ -560,9 +564,6 @@ def make_nvfp4_moe_kernel(
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if backend == NvFp4MoeBackend.HUMMING:
|
||||
assert layer is not None
|
||||
extra_kwargs = {"layer": layer}
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation:
|
||||
extra_kwargs["per_token_activation"] = True
|
||||
|
||||
|
||||
@@ -751,7 +751,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
@@ -766,7 +765,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
|
||||
@@ -772,7 +772,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
@@ -787,7 +786,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
gptq_marlin_moe_quant_config,
|
||||
|
||||
-1
@@ -251,7 +251,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
-1
@@ -339,7 +339,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
|
||||
-1
@@ -159,7 +159,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
-1
@@ -139,7 +139,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -679,7 +679,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
get_humming_linear_compute_config,
|
||||
get_humming_moe_quant_config,
|
||||
input_schema_to_quant_key,
|
||||
make_humming_moe_kernel,
|
||||
@@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
if name != "bias":
|
||||
delattr(layer, name)
|
||||
delattr(layer, "locks")
|
||||
self.__class__ = UnquantizedLinearMethod # type: ignore
|
||||
tensor = torch.empty(
|
||||
(
|
||||
@@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
param = prepare_param(tensor, name, extra_attrs)
|
||||
setattr(layer, name, param)
|
||||
|
||||
locks = torch.zeros(1024, dtype=torch.int32)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
if self.force_input_schema is not None:
|
||||
self.input_schema = self.force_input_schema
|
||||
|
||||
@@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
self.layer_config = _hm.prepare_layer_config(
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
weight_schema=self.weight_schema,
|
||||
@@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
)
|
||||
tensors = _hm.transform_humming_tensors(
|
||||
self.layer_config, dict(layer.named_parameters())
|
||||
)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# preprocess weight for inference
|
||||
_hm.HummingMethod.transform_humming_layer(layer)
|
||||
|
||||
# compute_config: kernel configs that do not directly affect weights
|
||||
# but significantly impact kernel behavior or computation precision.
|
||||
# see https://github.com/inclusionAI/humming/blob/main/docs/config.md
|
||||
compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
self.compute_config = json.dumps(compute_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = _hm.HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
flatten_inputs = x.reshape(-1, x.size(-1))
|
||||
output = _hm.humming_forward(
|
||||
self.layer_config,
|
||||
inputs=flatten_inputs,
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
locks=self.locks,
|
||||
compute_config=self.compute_config,
|
||||
)
|
||||
output = output.view(*x.shape[:-1], output.size(-1))
|
||||
@@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# sublayer: a layer contains multiple sets of weights for quantized GEMM
|
||||
# (e.g., weight, weight_scale, etc.).
|
||||
# The weight names of sublayer start with the prefix "{sublayer_name}_"
|
||||
w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards
|
||||
layer.sublayer_configs = {
|
||||
"w13": {
|
||||
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
"shape_n": w13_output_size,
|
||||
"shape_k": hidden_size,
|
||||
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
|
||||
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
shape_n=w13_output_size,
|
||||
shape_k=hidden_size,
|
||||
num_experts=num_experts,
|
||||
param_dtype=params_dtype,
|
||||
@@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
if self.force_input_schema is not None:
|
||||
self.input_schema = self.force_input_schema
|
||||
|
||||
locks = torch.zeros(1024, dtype=torch.int32)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
self.humming_configs,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if getattr(self, "processed", False):
|
||||
@@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
self.processed = True
|
||||
|
||||
# Convert weights to Humming kernel format
|
||||
convert_to_humming_moe_kernel_format(
|
||||
self.humming_configs = convert_to_humming_moe_kernel_format(
|
||||
layer=layer,
|
||||
sublayer_configs=layer.sublayer_configs,
|
||||
weight_schema=self.weight_schema,
|
||||
@@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
self.moe_quant_config,
|
||||
self.moe,
|
||||
self.experts_cls,
|
||||
layer=layer,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
@@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
3. Finalization (weight application & reduction - no-op for Humming
|
||||
since it's already done internally)
|
||||
|
||||
Humming handles all quantization, weight application, and reduction
|
||||
internally in the experts.apply() method via HummingMethod calls.
|
||||
Humming handles quantization, weight application, and reduction
|
||||
internally in the experts implementation.
|
||||
|
||||
Note: Although w1/w2 weights are passed to the kernel for interface
|
||||
consistency, Humming's experts.apply() reads weights directly from
|
||||
the layer object via HummingMethod.forward_layer() and ignores the
|
||||
w1/w2 parameters.
|
||||
Humming experts consume the weights and quantization tensors passed
|
||||
through the standard modular kernel interface.
|
||||
"""
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -886,7 +886,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -1579,7 +1578,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -2058,7 +2056,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
|
||||
@@ -370,7 +370,12 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
has_zp = self.quant_config.has_zp
|
||||
return make_wna16_moe_quant_config(
|
||||
@@ -391,7 +396,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
|
||||
@@ -382,7 +382,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -773,7 +772,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def _convert_k3_situ_weight_to_kernel_format(
|
||||
|
||||
@@ -516,7 +516,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -117,7 +117,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -172,7 +172,6 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
per_token_activation=True,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -804,7 +804,6 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -1646,7 +1645,6 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
@@ -47,11 +48,19 @@ if TYPE_CHECKING:
|
||||
GPTQWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
LayerConfig,
|
||||
)
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class HummingMoEQuantConfig(FusedMoEQuantConfig):
|
||||
w1_humming_config: "LayerConfig"
|
||||
w2_humming_config: "LayerConfig"
|
||||
|
||||
|
||||
if has_humming():
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
@@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard(
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_layer(
|
||||
def prepare_humming_linear_layer_config(
|
||||
layer: LinearBase,
|
||||
quant_config: dict,
|
||||
input_quant_config: dict | None = None,
|
||||
):
|
||||
) -> "LayerConfig":
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingMethod,
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
@@ -480,18 +490,8 @@ def prepare_humming_layer(
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
if isinstance(tensor, torch.nn.Parameter):
|
||||
tensor = tensor.data
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
# Step 2: transform weight (humming standard format) for forwarding.
|
||||
config = prepare_layer_config(
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
@@ -501,20 +501,49 @@ def prepare_humming_layer(
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.params_dtype,
|
||||
)
|
||||
tensors = transform_humming_tensors(config, tensors)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
return config
|
||||
|
||||
compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
|
||||
layer.compute_config = json.dumps(compute_config)
|
||||
def get_humming_linear_compute_config() -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def apply_humming_linear(
|
||||
layer: LinearBase,
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
layer_config: "LayerConfig",
|
||||
compute_config: str,
|
||||
locks: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import humming_forward
|
||||
|
||||
flatten_inputs = x.reshape(-1, x.size(-1))
|
||||
output = humming_forward(
|
||||
layer_config,
|
||||
inputs=flatten_inputs,
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
locks=locks,
|
||||
compute_config=compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
|
||||
def make_humming_moe_quant_config(
|
||||
@@ -532,7 +561,9 @@ def make_humming_moe_quant_config(
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
humming_configs: dict[str, "LayerConfig"] | None = None,
|
||||
) -> HummingMoEQuantConfig:
|
||||
assert humming_configs is not None
|
||||
if quant_dtype is None:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
else:
|
||||
@@ -557,7 +588,7 @@ def make_humming_moe_quant_config(
|
||||
bias=w2_bias,
|
||||
)
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
return HummingMoEQuantConfig(
|
||||
_a1=a_quant_desc,
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
@@ -565,15 +596,20 @@ def make_humming_moe_quant_config(
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
w1_humming_config=humming_configs["w13"],
|
||||
w2_humming_config=humming_configs["w2"],
|
||||
)
|
||||
|
||||
|
||||
def get_humming_moe_quant_config(
|
||||
layer: "RoutedExperts",
|
||||
humming_configs: dict[str, "LayerConfig"] | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
):
|
||||
if humming_configs is None:
|
||||
humming_configs = layer.humming_configs
|
||||
input_schema = layer.input_schemas["w13"]
|
||||
weight_schema = layer.weight_schemas["w13"]
|
||||
|
||||
@@ -600,13 +636,17 @@ def get_humming_moe_quant_config(
|
||||
weight_dtype=str(weight_schema.b_dtype),
|
||||
weight_group_shape=weight_group_shape,
|
||||
w1_scale=getattr(layer, "w13_weight_scale", None),
|
||||
w1_gscale=getattr(layer, "w13_global_scale", None),
|
||||
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
|
||||
w1_zp=getattr(layer, "w13_zero_point", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_scale=getattr(layer, "w2_weight_scale", None),
|
||||
w2_gscale=getattr(layer, "w2_global_scale", None),
|
||||
w2_gscale=getattr(layer, "w2_weight_scale_2", None),
|
||||
w2_zp=getattr(layer, "w2_zero_point", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
humming_configs=humming_configs,
|
||||
)
|
||||
|
||||
|
||||
@@ -677,7 +717,6 @@ def make_humming_moe_kernel(
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
layer: "RoutedExperts",
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
@@ -692,8 +731,6 @@ def make_humming_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_args: dict[str, Any] = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -703,13 +740,11 @@ def make_humming_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_args,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_args,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
@@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer(
|
||||
has_bias: bool,
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""
|
||||
Prepare layer metadata and transform weights for a sublayer.
|
||||
) -> "LayerConfig":
|
||||
"""Prepare Humming configuration and transform one sublayer's tensors."""
|
||||
from vllm.utils.humming import (
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
)
|
||||
|
||||
This calls Humming's prepare_layer_meta and transform_humming_layer.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
config = prepare_layer_config(
|
||||
shape_n=shape_n,
|
||||
shape_k=shape_k,
|
||||
pad_n_to_multiple=256,
|
||||
@@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer(
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
torch_dtype=param_dtype,
|
||||
sublayer_name=sublayer_name,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
|
||||
tensors = transform_humming_tensors(
|
||||
config,
|
||||
_extract_sublayer_tensors(layer, sublayer_name),
|
||||
)
|
||||
_replace_layer_parameters(layer, sublayer_name, tensors)
|
||||
return config
|
||||
|
||||
|
||||
def _process_single_sublayer(
|
||||
@@ -858,7 +894,7 @@ def _process_single_sublayer(
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
) -> tuple[Any, Any, "LayerConfig"]:
|
||||
"""
|
||||
Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
|
||||
@@ -878,7 +914,7 @@ def _process_single_sublayer(
|
||||
force_weight_schema: Optional schema to force requantization to
|
||||
|
||||
Returns:
|
||||
Tuple of (final_weight_schema, final_input_schema)
|
||||
Tuple of the final weight schema, input schema, and Humming layer config.
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
@@ -910,7 +946,7 @@ def _process_single_sublayer(
|
||||
del tensors
|
||||
|
||||
# Step 3: Prepare layer metadata and transform weights
|
||||
_prepare_and_transform_sublayer(
|
||||
config = _prepare_and_transform_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
shape_n=shape_n,
|
||||
@@ -922,7 +958,7 @@ def _process_single_sublayer(
|
||||
param_dtype=param_dtype,
|
||||
)
|
||||
|
||||
return current_weight_schema, current_input_schema
|
||||
return current_weight_schema, current_input_schema, config
|
||||
|
||||
|
||||
def convert_to_humming_moe_kernel_format(
|
||||
@@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
weight_schema: Any | None = None,
|
||||
input_schema: Any | None = None,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> None:
|
||||
) -> dict[str, "LayerConfig"]:
|
||||
"""
|
||||
Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
|
||||
@@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
Side effects:
|
||||
- Modifies layer parameters in place
|
||||
- Sets layer.weight_schemas and layer.input_schemas
|
||||
- Sets layer.humming_configs for quant config construction
|
||||
"""
|
||||
|
||||
# Build schemas from quant_config if not provided
|
||||
@@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format(
|
||||
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
humming_configs = {}
|
||||
|
||||
for sublayer_name, configs in sublayer_configs.items():
|
||||
final_weight_schema, final_input_schema = _process_single_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
param_dtype=param_dtype,
|
||||
force_weight_schema=force_weight_schema,
|
||||
final_weight_schema, final_input_schema, humming_config = (
|
||||
_process_single_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
param_dtype=param_dtype,
|
||||
force_weight_schema=force_weight_schema,
|
||||
)
|
||||
)
|
||||
|
||||
layer.weight_schemas[sublayer_name] = final_weight_schema
|
||||
layer.input_schemas[sublayer_name] = final_input_schema
|
||||
humming_configs[sublayer_name] = humming_config
|
||||
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.w13_weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
layer.humming_configs = humming_configs
|
||||
return humming_configs
|
||||
|
||||
@@ -15,9 +15,13 @@ _EXPORTS: dict[str, str] = {
|
||||
"dtypes": "humming.dtypes",
|
||||
"DataType": "humming.dtypes:DataType",
|
||||
"GemmType": "humming.config:GemmType",
|
||||
"LayerConfig": "humming.config:LayerConfig",
|
||||
"WeightScaleType": "humming.config:WeightScaleType",
|
||||
"HummingMethod": "humming.layer:HummingMethod",
|
||||
"HummingLayerMeta": "humming.layer:HummingLayerMeta",
|
||||
"humming_forward": "humming.forward:humming_forward",
|
||||
"may_quant_input": "humming.forward:may_quant_input",
|
||||
"prepare_layer_config": "humming.transform:prepare_layer_config",
|
||||
"transform_humming_tensors": "humming.transform:transform_humming_tensors",
|
||||
"get_heuristics_config": "humming.tune:get_heuristics_config",
|
||||
"BaseInputSchema": "humming.schema:BaseInputSchema",
|
||||
"BaseWeightSchema": "humming.schema:BaseWeightSchema",
|
||||
"HummingInputSchema": "humming.schema:HummingInputSchema",
|
||||
|
||||
Reference in New Issue
Block a user