[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
e229fdbc87
commit
fcde8e1460
@@ -751,7 +751,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
@@ -766,7 +765,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
|
||||
@@ -772,7 +772,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
@@ -787,7 +786,12 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
gptq_marlin_moe_quant_config,
|
||||
|
||||
-1
@@ -251,7 +251,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
-1
@@ -339,7 +339,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
|
||||
-1
@@ -159,7 +159,6 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
-1
@@ -139,7 +139,6 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -679,7 +679,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -32,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
get_humming_linear_compute_config,
|
||||
get_humming_moe_quant_config,
|
||||
input_schema_to_quant_key,
|
||||
make_humming_moe_kernel,
|
||||
@@ -425,7 +425,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
if name != "bias":
|
||||
delattr(layer, name)
|
||||
delattr(layer, "locks")
|
||||
self.__class__ = UnquantizedLinearMethod # type: ignore
|
||||
tensor = torch.empty(
|
||||
(
|
||||
@@ -513,9 +512,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
param = prepare_param(tensor, name, extra_attrs)
|
||||
setattr(layer, name, param)
|
||||
|
||||
locks = torch.zeros(1024, dtype=torch.int32)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
if self.force_input_schema is not None:
|
||||
self.input_schema = self.force_input_schema
|
||||
|
||||
@@ -574,9 +570,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
self.layer_config = _hm.prepare_layer_config(
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
weight_schema=self.weight_schema,
|
||||
@@ -586,19 +580,17 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.param_dtype,
|
||||
)
|
||||
tensors = _hm.transform_humming_tensors(
|
||||
self.layer_config, dict(layer.named_parameters())
|
||||
)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# preprocess weight for inference
|
||||
_hm.HummingMethod.transform_humming_layer(layer)
|
||||
|
||||
# compute_config: kernel configs that do not directly affect weights
|
||||
# but significantly impact kernel behavior or computation precision.
|
||||
# see https://github.com/inclusionAI/humming/blob/main/docs/config.md
|
||||
compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
self.compute_config = json.dumps(compute_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -606,10 +598,16 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = _hm.HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
flatten_inputs = x.reshape(-1, x.size(-1))
|
||||
output = _hm.humming_forward(
|
||||
self.layer_config,
|
||||
inputs=flatten_inputs,
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
locks=self.locks,
|
||||
compute_config=self.compute_config,
|
||||
)
|
||||
output = output.view(*x.shape[:-1], output.size(-1))
|
||||
@@ -726,12 +724,13 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
# sublayer: a layer contains multiple sets of weights for quantized GEMM
|
||||
# (e.g., weight, weight_scale, etc.).
|
||||
# The weight names of sublayer start with the prefix "{sublayer_name}_"
|
||||
w13_output_size = intermediate_size_per_partition * self.moe.w13_num_shards
|
||||
layer.sublayer_configs = {
|
||||
"w13": {
|
||||
"shape_n": intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
"shape_n": w13_output_size,
|
||||
"shape_k": hidden_size,
|
||||
"tensors_attrs": self.weight_schema.get_padded_tensors_attrs(
|
||||
shape_n=intermediate_size_per_partition * self.moe.w13_num_shards,
|
||||
shape_n=w13_output_size,
|
||||
shape_k=hidden_size,
|
||||
num_experts=num_experts,
|
||||
param_dtype=params_dtype,
|
||||
@@ -763,11 +762,14 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
if self.force_input_schema is not None:
|
||||
self.input_schema = self.force_input_schema
|
||||
|
||||
locks = torch.zeros(1024, dtype=torch.int32)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
self.humming_configs,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if getattr(self, "processed", False):
|
||||
@@ -775,7 +777,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
self.processed = True
|
||||
|
||||
# Convert weights to Humming kernel format
|
||||
convert_to_humming_moe_kernel_format(
|
||||
self.humming_configs = convert_to_humming_moe_kernel_format(
|
||||
layer=layer,
|
||||
sublayer_configs=layer.sublayer_configs,
|
||||
weight_schema=self.weight_schema,
|
||||
@@ -791,7 +793,6 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
self.moe_quant_config,
|
||||
self.moe,
|
||||
self.experts_cls,
|
||||
layer=layer,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
@@ -814,13 +815,11 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
3. Finalization (weight application & reduction - no-op for Humming
|
||||
since it's already done internally)
|
||||
|
||||
Humming handles all quantization, weight application, and reduction
|
||||
internally in the experts.apply() method via HummingMethod calls.
|
||||
Humming handles quantization, weight application, and reduction
|
||||
internally in the experts implementation.
|
||||
|
||||
Note: Although w1/w2 weights are passed to the kernel for interface
|
||||
consistency, Humming's experts.apply() reads weights directly from
|
||||
the layer object via HummingMethod.forward_layer() and ignores the
|
||||
w1/w2 parameters.
|
||||
Humming experts consume the weights and quantization tensors passed
|
||||
through the standard modular kernel interface.
|
||||
"""
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -886,7 +886,6 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -1579,7 +1578,6 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -2058,7 +2056,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
|
||||
@@ -370,7 +370,12 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
|
||||
has_zp = self.quant_config.has_zp
|
||||
return make_wna16_moe_quant_config(
|
||||
@@ -391,7 +396,6 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
|
||||
@@ -382,7 +382,6 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -773,7 +772,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def _convert_k3_situ_weight_to_kernel_format(
|
||||
|
||||
@@ -516,7 +516,6 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -117,7 +117,6 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@@ -200,7 +200,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -172,7 +172,6 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
per_token_activation=True,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -804,7 +804,6 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -1646,7 +1645,6 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
@@ -47,11 +48,19 @@ if TYPE_CHECKING:
|
||||
GPTQWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingWeightSchema,
|
||||
LayerConfig,
|
||||
)
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class HummingMoEQuantConfig(FusedMoEQuantConfig):
|
||||
w1_humming_config: "LayerConfig"
|
||||
w2_humming_config: "LayerConfig"
|
||||
|
||||
|
||||
if has_humming():
|
||||
from vllm.utils.humming import dtypes as humming_dtypes
|
||||
|
||||
@@ -435,16 +444,17 @@ def convert_linear_layer_to_humming_standard(
|
||||
setattr(layer, name, param)
|
||||
|
||||
|
||||
def prepare_humming_layer(
|
||||
def prepare_humming_linear_layer_config(
|
||||
layer: LinearBase,
|
||||
quant_config: dict,
|
||||
input_quant_config: dict | None = None,
|
||||
):
|
||||
) -> "LayerConfig":
|
||||
from vllm.utils.humming import (
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
HummingMethod,
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
@@ -480,18 +490,8 @@ def prepare_humming_layer(
|
||||
|
||||
layer.weight_schema = weight_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
if isinstance(tensor, torch.nn.Parameter):
|
||||
tensor = tensor.data
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
# Step 2: transform weight (humming standard format) for forwarding.
|
||||
config = prepare_layer_config(
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
@@ -501,20 +501,49 @@ def prepare_humming_layer(
|
||||
has_bias=layer.has_bias,
|
||||
torch_dtype=layer.params_dtype,
|
||||
)
|
||||
tensors = transform_humming_tensors(config, tensors)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer)
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
return config
|
||||
|
||||
compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
|
||||
layer.compute_config = json.dumps(compute_config)
|
||||
def get_humming_linear_compute_config() -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
"gemm_type": "dense",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def apply_humming_linear(
|
||||
layer: LinearBase,
|
||||
x: torch.Tensor,
|
||||
*,
|
||||
layer_config: "LayerConfig",
|
||||
compute_config: str,
|
||||
locks: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import humming_forward
|
||||
|
||||
flatten_inputs = x.reshape(-1, x.size(-1))
|
||||
output = humming_forward(
|
||||
layer_config,
|
||||
inputs=flatten_inputs,
|
||||
weight=layer.weight,
|
||||
weight_scale=getattr(layer, "weight_scale", None),
|
||||
zero_point=getattr(layer, "zero_point", None),
|
||||
bias=getattr(layer, "bias", None),
|
||||
weight_scale_2=getattr(layer, "weight_scale_2", None),
|
||||
locks=locks,
|
||||
compute_config=compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
|
||||
def make_humming_moe_quant_config(
|
||||
@@ -532,7 +561,9 @@ def make_humming_moe_quant_config(
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
humming_configs: dict[str, "LayerConfig"] | None = None,
|
||||
) -> HummingMoEQuantConfig:
|
||||
assert humming_configs is not None
|
||||
if quant_dtype is None:
|
||||
a_quant_desc = FusedMoEQuantDesc(dtype=None)
|
||||
else:
|
||||
@@ -557,7 +588,7 @@ def make_humming_moe_quant_config(
|
||||
bias=w2_bias,
|
||||
)
|
||||
|
||||
return FusedMoEQuantConfig(
|
||||
return HummingMoEQuantConfig(
|
||||
_a1=a_quant_desc,
|
||||
_a2=a_quant_desc,
|
||||
_w1=w1_quant_desc,
|
||||
@@ -565,15 +596,20 @@ def make_humming_moe_quant_config(
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
w1_humming_config=humming_configs["w13"],
|
||||
w2_humming_config=humming_configs["w2"],
|
||||
)
|
||||
|
||||
|
||||
def get_humming_moe_quant_config(
|
||||
layer: "RoutedExperts",
|
||||
humming_configs: dict[str, "LayerConfig"] | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
):
|
||||
if humming_configs is None:
|
||||
humming_configs = layer.humming_configs
|
||||
input_schema = layer.input_schemas["w13"]
|
||||
weight_schema = layer.weight_schemas["w13"]
|
||||
|
||||
@@ -600,13 +636,17 @@ def get_humming_moe_quant_config(
|
||||
weight_dtype=str(weight_schema.b_dtype),
|
||||
weight_group_shape=weight_group_shape,
|
||||
w1_scale=getattr(layer, "w13_weight_scale", None),
|
||||
w1_gscale=getattr(layer, "w13_global_scale", None),
|
||||
w1_gscale=getattr(layer, "w13_weight_scale_2", None),
|
||||
w1_zp=getattr(layer, "w13_zero_point", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_scale=getattr(layer, "w2_weight_scale", None),
|
||||
w2_gscale=getattr(layer, "w2_global_scale", None),
|
||||
w2_gscale=getattr(layer, "w2_weight_scale_2", None),
|
||||
w2_zp=getattr(layer, "w2_zero_point", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
humming_configs=humming_configs,
|
||||
)
|
||||
|
||||
|
||||
@@ -677,7 +717,6 @@ def make_humming_moe_kernel(
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
layer: "RoutedExperts",
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
# Create Prepare/Finalize.
|
||||
@@ -692,8 +731,6 @@ def make_humming_moe_kernel(
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
|
||||
extra_args: dict[str, Any] = {"layer": layer}
|
||||
|
||||
# Create Experts.
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
|
||||
@@ -703,13 +740,11 @@ def make_humming_moe_kernel(
|
||||
quant_config=moe_quant_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
**extra_args,
|
||||
)
|
||||
else:
|
||||
experts = experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=moe_quant_config,
|
||||
**extra_args,
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
@@ -822,16 +857,14 @@ def _prepare_and_transform_sublayer(
|
||||
has_bias: bool,
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""
|
||||
Prepare layer metadata and transform weights for a sublayer.
|
||||
) -> "LayerConfig":
|
||||
"""Prepare Humming configuration and transform one sublayer's tensors."""
|
||||
from vllm.utils.humming import (
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
)
|
||||
|
||||
This calls Humming's prepare_layer_meta and transform_humming_layer.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
config = prepare_layer_config(
|
||||
shape_n=shape_n,
|
||||
shape_k=shape_k,
|
||||
pad_n_to_multiple=256,
|
||||
@@ -841,10 +874,13 @@ def _prepare_and_transform_sublayer(
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
torch_dtype=param_dtype,
|
||||
sublayer_name=sublayer_name,
|
||||
)
|
||||
|
||||
HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name)
|
||||
tensors = transform_humming_tensors(
|
||||
config,
|
||||
_extract_sublayer_tensors(layer, sublayer_name),
|
||||
)
|
||||
_replace_layer_parameters(layer, sublayer_name, tensors)
|
||||
return config
|
||||
|
||||
|
||||
def _process_single_sublayer(
|
||||
@@ -858,7 +894,7 @@ def _process_single_sublayer(
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> tuple[Any, Any]:
|
||||
) -> tuple[Any, Any, "LayerConfig"]:
|
||||
"""
|
||||
Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
|
||||
@@ -878,7 +914,7 @@ def _process_single_sublayer(
|
||||
force_weight_schema: Optional schema to force requantization to
|
||||
|
||||
Returns:
|
||||
Tuple of (final_weight_schema, final_input_schema)
|
||||
Tuple of the final weight schema, input schema, and Humming layer config.
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
@@ -910,7 +946,7 @@ def _process_single_sublayer(
|
||||
del tensors
|
||||
|
||||
# Step 3: Prepare layer metadata and transform weights
|
||||
_prepare_and_transform_sublayer(
|
||||
config = _prepare_and_transform_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
shape_n=shape_n,
|
||||
@@ -922,7 +958,7 @@ def _process_single_sublayer(
|
||||
param_dtype=param_dtype,
|
||||
)
|
||||
|
||||
return current_weight_schema, current_input_schema
|
||||
return current_weight_schema, current_input_schema, config
|
||||
|
||||
|
||||
def convert_to_humming_moe_kernel_format(
|
||||
@@ -932,7 +968,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
weight_schema: Any | None = None,
|
||||
input_schema: Any | None = None,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> None:
|
||||
) -> dict[str, "LayerConfig"]:
|
||||
"""
|
||||
Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
|
||||
@@ -959,6 +995,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
Side effects:
|
||||
- Modifies layer parameters in place
|
||||
- Sets layer.weight_schemas and layer.input_schemas
|
||||
- Sets layer.humming_configs for quant config construction
|
||||
"""
|
||||
|
||||
# Build schemas from quant_config if not provided
|
||||
@@ -1005,25 +1042,27 @@ def convert_to_humming_moe_kernel_format(
|
||||
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
humming_configs = {}
|
||||
|
||||
for sublayer_name, configs in sublayer_configs.items():
|
||||
final_weight_schema, final_input_schema = _process_single_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
param_dtype=param_dtype,
|
||||
force_weight_schema=force_weight_schema,
|
||||
final_weight_schema, final_input_schema, humming_config = (
|
||||
_process_single_sublayer(
|
||||
layer=layer,
|
||||
sublayer_name=sublayer_name,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
weight_schema=weight_schema,
|
||||
input_schema=input_schema,
|
||||
has_bias=has_bias,
|
||||
num_experts=num_experts,
|
||||
param_dtype=param_dtype,
|
||||
force_weight_schema=force_weight_schema,
|
||||
)
|
||||
)
|
||||
|
||||
layer.weight_schemas[sublayer_name] = final_weight_schema
|
||||
layer.input_schemas[sublayer_name] = final_input_schema
|
||||
humming_configs[sublayer_name] = humming_config
|
||||
|
||||
if not hasattr(layer, "locks"):
|
||||
device = layer.w13_weight.device
|
||||
locks = torch.zeros(1024, dtype=torch.int32, device=device)
|
||||
layer.register_buffer("locks", locks)
|
||||
layer.humming_configs = humming_configs
|
||||
return humming_configs
|
||||
|
||||
Reference in New Issue
Block a user