[CT] Support Humming for WNA16 MoE (#48918)
Signed-off-by: yiliu30 <yi4.liu@intel.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
Misha Goin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c676232313
commit
54dd98be28
@@ -40,6 +40,9 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
INT4_DTYPE,
|
||||
INT8_DTYPE,
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
@@ -58,6 +61,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType
|
||||
from vllm.utils.import_utils import has_humming
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
@@ -76,6 +80,29 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _is_supported_wna16_weight_key(weight_key: QuantKey | None) -> bool:
|
||||
if weight_key is None or weight_key.scale2 is not None:
|
||||
return False
|
||||
|
||||
group_shape = weight_key.scale.group_shape
|
||||
if not (
|
||||
group_shape == GroupShape.PER_CHANNEL
|
||||
or (group_shape.row == 1 and group_shape.col > 0)
|
||||
):
|
||||
return False
|
||||
|
||||
dtype = weight_key.dtype
|
||||
if dtype in (INT4_DTYPE, INT8_DTYPE, torch.uint8):
|
||||
return True
|
||||
|
||||
return (
|
||||
isinstance(dtype, ScalarType)
|
||||
and dtype.is_integer()
|
||||
and not dtype.is_signed()
|
||||
and 2 <= dtype.size_bits <= 8
|
||||
)
|
||||
|
||||
|
||||
def get_humming_moe_gemm_type() -> str:
|
||||
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
|
||||
gemm_type = "indexed"
|
||||
@@ -267,7 +294,10 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
# mxfp8 (compressed-tensors / modelopt / online)
|
||||
(kMxfp8Static, kMxfp8Dynamic),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A or (
|
||||
activation_key in (None, kFp8DynamicTokenSym)
|
||||
and _is_supported_wna16_weight_key(weight_key)
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
|
||||
@@ -1104,9 +1104,21 @@ def _humming_wna16_weight_schema(
|
||||
"desc_act": quant_config.desc_act,
|
||||
"sym": quant_config.is_sym,
|
||||
}
|
||||
if isinstance(quant_config, QuantizationArgs):
|
||||
quant_type = getattr(quant_config.type, "value", quant_config.type)
|
||||
quant_strategy = getattr(quant_config.strategy, "value", quant_config.strategy)
|
||||
return {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "pack-quantized",
|
||||
"type": str(quant_type),
|
||||
"num_bits": quant_config.num_bits,
|
||||
"strategy": str(quant_strategy),
|
||||
"group_size": quant_config.group_size,
|
||||
"symmetric": quant_config.symmetric,
|
||||
}
|
||||
raise TypeError(
|
||||
"Humming WNA16 checkpoint schema requires AutoAWQConfig or "
|
||||
"AutoGPTQConfig, "
|
||||
"Humming WNA16 checkpoint schema requires AutoAWQConfig, "
|
||||
"AutoGPTQConfig or QuantizationArgs, "
|
||||
f"got {type(quant_config).__name__}."
|
||||
)
|
||||
|
||||
|
||||
+42
-14
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -40,7 +42,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_make_workspace_new,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
kInt4Static32GroupScale,
|
||||
kInt4StaticGroupScale,
|
||||
kInt8StaticGroupScale,
|
||||
@@ -64,7 +68,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
# Extract properties from weight_quant
|
||||
self.symmetric = weight_quant.symmetric
|
||||
self.num_bits = weight_quant.num_bits
|
||||
self.packed_factor = 32 // weight_quant.num_bits
|
||||
self.packed_factor = Fraction(32, weight_quant.num_bits)
|
||||
self.strategy = weight_quant.strategy
|
||||
self.group_size = weight_quant.group_size
|
||||
self.actorder = weight_quant.actorder
|
||||
@@ -84,8 +88,14 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
elif self.num_bits == 8:
|
||||
scale = kInt8StaticGroupScale
|
||||
else:
|
||||
raise ValueError(
|
||||
"CompressedTensorsWNA16MoEMethod only supports int4 and int8 now."
|
||||
scale = ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=(
|
||||
GroupShape.PER_CHANNEL
|
||||
if self.group_size == -1
|
||||
else GroupShape(row=1, col=self.group_size)
|
||||
),
|
||||
)
|
||||
|
||||
weight_key = QuantKey(self.quant_type, scale, symmetric=self.symmetric)
|
||||
@@ -109,7 +119,10 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
]
|
||||
self.is_transposed = self.wna16_backend != WNA16MoEBackend.FLASHINFER_TRTLLM
|
||||
self.is_transposed = self.wna16_backend not in (
|
||||
WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
WNA16MoEBackend.HUMMING,
|
||||
)
|
||||
|
||||
if self.is_marlin:
|
||||
assert check_moe_marlin_supports_config(
|
||||
@@ -125,6 +138,9 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
# Non-Marlin WNA16 always uses bf16/fp16 inputs
|
||||
self.input_dtype = torch.bfloat16
|
||||
|
||||
def _packed_dim(self, dim: int) -> int:
|
||||
return math.ceil(dim * self.num_bits / 32)
|
||||
|
||||
def get_weight_shape(
|
||||
self,
|
||||
weight_name: str,
|
||||
@@ -149,16 +165,16 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
"num_groups_w2 must be provided for weight scales/zero_points"
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
shape_map = {
|
||||
shape_map: dict[str, dict[str, tuple[int, int | None, int | None]]] = {
|
||||
"w13_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // self.packed_factor,
|
||||
self._packed_dim(hidden_size),
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
hidden_size // self.packed_factor,
|
||||
self._packed_dim(hidden_size),
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
),
|
||||
},
|
||||
@@ -178,20 +194,18 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards
|
||||
* intermediate_size_per_partition
|
||||
// self.packed_factor,
|
||||
self._packed_dim(w13_num_shards * intermediate_size_per_partition),
|
||||
),
|
||||
},
|
||||
"w2_weight": {
|
||||
"Flashinfer": (
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
self._packed_dim(intermediate_size_per_partition),
|
||||
),
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
self._packed_dim(intermediate_size_per_partition),
|
||||
hidden_size,
|
||||
),
|
||||
},
|
||||
@@ -203,12 +217,14 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
"Marlin": (
|
||||
num_experts,
|
||||
num_groups_w2,
|
||||
hidden_size // self.packed_factor,
|
||||
self._packed_dim(hidden_size),
|
||||
),
|
||||
},
|
||||
}
|
||||
backend_key = "Marlin" if self.is_transposed else "Flashinfer"
|
||||
return shape_map[weight_name][backend_key]
|
||||
shape = shape_map[weight_name][backend_key]
|
||||
assert shape[1] is not None and shape[2] is not None
|
||||
return shape[0], shape[1], shape[2]
|
||||
|
||||
@staticmethod
|
||||
def _w2_scale_sharding(
|
||||
@@ -483,6 +499,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
**marlin_args,
|
||||
)
|
||||
@@ -586,6 +603,17 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.wna16_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
)
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
|
||||
@@ -73,6 +73,9 @@ if has_humming():
|
||||
humming_dtypes.uint8: INT8_DTYPE,
|
||||
humming_dtypes.uint2: torch.uint8,
|
||||
humming_dtypes.uint3: torch.uint8,
|
||||
humming_dtypes.uint5: torch.uint8,
|
||||
humming_dtypes.uint6: torch.uint8,
|
||||
humming_dtypes.uint7: torch.uint8,
|
||||
}
|
||||
|
||||
_HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = {
|
||||
|
||||
@@ -25,6 +25,13 @@ INT4_DTYPE = scalar_types.uint4b8
|
||||
INT8_DTYPE = scalar_types.uint8b128
|
||||
|
||||
|
||||
def _dtype_abbr(dtype: torch.dtype | ScalarType) -> str:
|
||||
"""Return a stable short name for torch and ScalarType dtypes."""
|
||||
if isinstance(dtype, ScalarType):
|
||||
return str(dtype)
|
||||
return fx.graph.dtype_abbrs[dtype]
|
||||
|
||||
|
||||
def weight_amax(
|
||||
weight: torch.Tensor, *, dim: int | None = None, keepdim: bool = False
|
||||
) -> torch.Tensor:
|
||||
@@ -152,7 +159,7 @@ class ScaleDesc:
|
||||
group_shape = d.get(self.group_shape, str(self.group_shape))
|
||||
|
||||
return (
|
||||
f"{fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"{_dtype_abbr(self.dtype)},"
|
||||
f"{'static' if self.static else 'dynamic'},{group_shape}"
|
||||
)
|
||||
|
||||
@@ -178,13 +185,8 @@ class QuantKey:
|
||||
|
||||
def __str__(self):
|
||||
scale2_str = f"scale2({self.scale2})," if self.scale2 else ""
|
||||
dtype_description = (
|
||||
fx.graph.dtype_abbrs[self.dtype]
|
||||
if isinstance(self.dtype, torch.dtype)
|
||||
else self.dtype
|
||||
)
|
||||
return (
|
||||
f"QuantKey({dtype_description},"
|
||||
f"QuantKey({_dtype_abbr(self.dtype)},"
|
||||
f"scale({self.scale}),{scale2_str}"
|
||||
f"{'a' if not self.symmetric else ''}symmetric)"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user