[CT] Support Humming for WNA16 MoE (#48918)

Signed-off-by: yiliu30 <yi4.liu@intel.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Yi Liu
2026-08-19 12:31:32 -04:00
committed by GitHub
co-authored by OpenAI Codex OpenAI Codex Misha Goin mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c676232313
commit 54dd98be28
7 changed files with 253 additions and 24 deletions
@@ -43,8 +43,14 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
find_matched_target,
)
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
ScaleDesc,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
# AITER only supports per-channel-per-channel INT8 gemm
@@ -971,6 +977,44 @@ def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected)
assert result == expected
@pytest.mark.parametrize("num_bits", range(2, 9))
def test_humming_supports_compressed_tensors_wna16_quant_key(num_bits):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
)
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
WNA16_SUPPORTED_TYPES_MAP,
)
weight_key = QuantKey(
dtype=WNA16_SUPPORTED_TYPES_MAP[num_bits],
scale=ScaleDesc(
dtype=torch.float16,
static=True,
group_shape=GroupShape(row=1, col=128),
),
symmetric=True,
)
assert HummingExpertsBase._supports_quant_scheme(weight_key, None)
def test_quant_key_str_supports_scalar_type_dtypes():
quant_key = QuantKey(
dtype=scalar_types.uint2b2,
scale=ScaleDesc(
dtype=torch.float16,
static=True,
group_shape=GroupShape(row=1, col=128),
),
symmetric=True,
)
assert str(quant_key) == (
"QuantKey(uint2b2,scale(f16,static,GroupShape(row=1, col=128)),symmetric)"
)
@pytest.mark.skipif(
not current_platform.is_cuda() or not current_platform.has_device_capability(80),
reason="MXFP4 requires ampere or newer",
+110
View File
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.platforms import current_platform
def test_map_wna16_backend_supports_triton():
@@ -203,3 +204,112 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
)
assert method.get_fused_moe_quant_config(layer) is quant_config
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
)
@pytest.mark.parametrize("num_bits", [3, 5, 6, 7])
def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
num_bits,
):
pytest.importorskip("humming")
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
quant_args = QuantizationArgs(
num_bits=num_bits,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
)
moe_config = make_dummy_moe_config(
num_experts=2,
hidden_dim=256,
intermediate_size=512,
)
moe_config.moe_backend = "humming"
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
layer = torch.nn.Module()
method.create_weights(
layer,
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=512,
params_dtype=torch.float16,
intermediate_size_full=512,
)
packed_hidden = (256 * num_bits + 31) // 32
packed_intermediate = (512 * num_bits + 31) // 32
assert method.wna16_backend == WNA16MoEBackend.HUMMING
assert layer.w13_weight_packed.shape == (2, 1024, packed_hidden)
assert layer.w2_weight_packed.shape == (2, 256, packed_intermediate)
assert layer.w13_weight_scale.shape == (2, 1024, 2)
assert layer.w2_weight_scale.shape == (2, 256, 4)
assert layer.w13_weight_packed.dtype is torch.int32
assert layer.w2_weight_scale.dtype is torch.float16
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
)
def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
pytest.importorskip("humming")
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
quant_args = QuantizationArgs(
num_bits=3,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
)
moe_config = make_dummy_moe_config(
num_experts=2,
hidden_dim=256,
intermediate_size=512,
)
moe_config.moe_backend = "humming"
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
layer = torch.nn.Module()
layer.moe_config = moe_config
layer.params_dtype = torch.bfloat16
layer.layer_name = "test.humming_moe"
layer._expert_routing_tables = lambda: (None, None, None)
method.create_weights(
layer,
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=512,
intermediate_size_full=512,
params_dtype=torch.bfloat16,
)
layer.cuda()
for parameter in layer.parameters():
parameter.data.zero_()
method.process_weights_after_loading(layer)
assert method.wna16_backend == WNA16MoEBackend.HUMMING
assert method.moe_kernel is not None
assert set(layer.weight_schemas) == {"w13", "w2"}
assert set(layer.humming_configs) == {"w13", "w2"}
assert not hasattr(layer, "w13_weight_packed")
assert not hasattr(layer, "w2_weight_packed")
assert layer.w13_weight.dtype is torch.int32
assert layer.w2_weight.dtype is torch.int32