[CT] Support Humming for WNA16 MoE (#48918)
Signed-off-by: yiliu30 <yi4.liu@intel.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
Misha Goin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c676232313
commit
54dd98be28
@@ -43,8 +43,14 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
find_matched_target,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
|
||||
# AITER only supports per-channel-per-channel INT8 gemm
|
||||
@@ -971,6 +977,44 @@ def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_bits", range(2, 9))
|
||||
def test_humming_supports_compressed_tensors_wna16_quant_key(num_bits):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
|
||||
weight_key = QuantKey(
|
||||
dtype=WNA16_SUPPORTED_TYPES_MAP[num_bits],
|
||||
scale=ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=GroupShape(row=1, col=128),
|
||||
),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
assert HummingExpertsBase._supports_quant_scheme(weight_key, None)
|
||||
|
||||
|
||||
def test_quant_key_str_supports_scalar_type_dtypes():
|
||||
quant_key = QuantKey(
|
||||
dtype=scalar_types.uint2b2,
|
||||
scale=ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=GroupShape(row=1, col=128),
|
||||
),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
assert str(quant_key) == (
|
||||
"QuantKey(uint2b2,scale(f16,static,GroupShape(row=1, col=128)),symmetric)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(80),
|
||||
reason="MXFP4 requires ampere or newer",
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_map_wna16_backend_supports_triton():
|
||||
@@ -203,3 +204,112 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
|
||||
)
|
||||
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
|
||||
)
|
||||
@pytest.mark.parametrize("num_bits", [3, 5, 6, 7])
|
||||
def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
|
||||
num_bits,
|
||||
):
|
||||
pytest.importorskip("humming")
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
quant_args = QuantizationArgs(
|
||||
num_bits=num_bits,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
group_size=128,
|
||||
)
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
)
|
||||
moe_config.moe_backend = "humming"
|
||||
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
|
||||
layer = torch.nn.Module()
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=512,
|
||||
params_dtype=torch.float16,
|
||||
intermediate_size_full=512,
|
||||
)
|
||||
|
||||
packed_hidden = (256 * num_bits + 31) // 32
|
||||
packed_intermediate = (512 * num_bits + 31) // 32
|
||||
assert method.wna16_backend == WNA16MoEBackend.HUMMING
|
||||
assert layer.w13_weight_packed.shape == (2, 1024, packed_hidden)
|
||||
assert layer.w2_weight_packed.shape == (2, 256, packed_intermediate)
|
||||
assert layer.w13_weight_scale.shape == (2, 1024, 2)
|
||||
assert layer.w2_weight_scale.shape == (2, 256, 4)
|
||||
assert layer.w13_weight_packed.dtype is torch.int32
|
||||
assert layer.w2_weight_scale.dtype is torch.float16
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Compressed-tensors Humming WNA16 MoE requires CUDA",
|
||||
)
|
||||
def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
|
||||
pytest.importorskip("humming")
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
quant_args = QuantizationArgs(
|
||||
num_bits=3,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
group_size=128,
|
||||
)
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
)
|
||||
moe_config.moe_backend = "humming"
|
||||
method = CompressedTensorsWNA16MoEMethod(quant_args, None, moe_config)
|
||||
layer = torch.nn.Module()
|
||||
layer.moe_config = moe_config
|
||||
layer.params_dtype = torch.bfloat16
|
||||
layer.layer_name = "test.humming_moe"
|
||||
layer._expert_routing_tables = lambda: (None, None, None)
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=512,
|
||||
intermediate_size_full=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
layer.cuda()
|
||||
for parameter in layer.parameters():
|
||||
parameter.data.zero_()
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert method.wna16_backend == WNA16MoEBackend.HUMMING
|
||||
assert method.moe_kernel is not None
|
||||
assert set(layer.weight_schemas) == {"w13", "w2"}
|
||||
assert set(layer.humming_configs) == {"w13", "w2"}
|
||||
assert not hasattr(layer, "w13_weight_packed")
|
||||
assert not hasattr(layer, "w2_weight_packed")
|
||||
assert layer.w13_weight.dtype is torch.int32
|
||||
assert layer.w2_weight.dtype is torch.int32
|
||||
|
||||
Reference in New Issue
Block a user