[CT] Support Humming for WNA16 MoE (#48918)
Signed-off-by: yiliu30 <yi4.liu@intel.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
OpenAI Codex
Misha Goin
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c676232313
commit
54dd98be28
@@ -43,8 +43,14 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
find_matched_target,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.v1.attention.backends.fa_utils import get_flash_attn_version
|
||||
|
||||
# AITER only supports per-channel-per-channel INT8 gemm
|
||||
@@ -971,6 +977,44 @@ def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_bits", range(2, 9))
|
||||
def test_humming_supports_compressed_tensors_wna16_quant_key(num_bits):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( # noqa: E501
|
||||
WNA16_SUPPORTED_TYPES_MAP,
|
||||
)
|
||||
|
||||
weight_key = QuantKey(
|
||||
dtype=WNA16_SUPPORTED_TYPES_MAP[num_bits],
|
||||
scale=ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=GroupShape(row=1, col=128),
|
||||
),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
assert HummingExpertsBase._supports_quant_scheme(weight_key, None)
|
||||
|
||||
|
||||
def test_quant_key_str_supports_scalar_type_dtypes():
|
||||
quant_key = QuantKey(
|
||||
dtype=scalar_types.uint2b2,
|
||||
scale=ScaleDesc(
|
||||
dtype=torch.float16,
|
||||
static=True,
|
||||
group_shape=GroupShape(row=1, col=128),
|
||||
),
|
||||
symmetric=True,
|
||||
)
|
||||
|
||||
assert str(quant_key) == (
|
||||
"QuantKey(uint2b2,scale(f16,static,GroupShape(row=1, col=128)),symmetric)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(80),
|
||||
reason="MXFP4 requires ampere or newer",
|
||||
|
||||
Reference in New Issue
Block a user