add 2/3/5/6/7 CUDA support in AutoRound format (#52890)
Signed-off-by: Wenhua Cheng <wenhua.cheng@intel.com>
This commit is contained in:
@@ -666,6 +666,92 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
|
|||||||
assert isinstance(scheme, INCWna16Scheme)
|
assert isinstance(scheme, INCWna16Scheme)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bits", [2, 3])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"packing_format", ["auto_round:auto_gptq", "auto_round:auto_awq"]
|
||||||
|
)
|
||||||
|
def test_wna16_cuda_low_bit_linear_routes_to_humming(
|
||||||
|
monkeypatch, bits, packing_format
|
||||||
|
) -> None:
|
||||||
|
expected_method = object()
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||||
|
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
|
||||||
|
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||||
|
"inc_wna16_scheme._build_humming_linear_method",
|
||||||
|
lambda layer_config: captured.update({"layer_config": layer_config})
|
||||||
|
or expected_method,
|
||||||
|
)
|
||||||
|
|
||||||
|
layer_config = make_layer_config(bits=bits, packing_format=packing_format)
|
||||||
|
method = INCWna16Scheme().get_linear_method(
|
||||||
|
make_config(), object(), "model.layers.0.mlp.down_proj", layer_config
|
||||||
|
)
|
||||||
|
|
||||||
|
assert method is expected_method
|
||||||
|
assert captured["layer_config"] is layer_config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bits", [2, 3])
|
||||||
|
def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
|
||||||
|
expected_method = object()
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
class DummyMoeConfig:
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||||
|
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||||
|
"inc_wna16_scheme._build_humming_moe_method",
|
||||||
|
lambda layer, layer_config: captured.update(
|
||||||
|
{"layer": layer, "layer_config": layer_config}
|
||||||
|
)
|
||||||
|
or expected_method,
|
||||||
|
)
|
||||||
|
|
||||||
|
layer = object.__new__(RoutedExperts)
|
||||||
|
layer.moe_config = DummyMoeConfig()
|
||||||
|
layer_config = make_layer_config(bits=bits)
|
||||||
|
method = INCWna16Scheme().get_moe_method(
|
||||||
|
make_config(), layer, "model.layers.0.mlp", layer_config
|
||||||
|
)
|
||||||
|
|
||||||
|
assert method is expected_method
|
||||||
|
assert captured["layer"] is layer
|
||||||
|
assert captured["layer_config"] is layer_config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("bits", [4, 8])
|
||||||
|
def test_wna16_cuda_high_bit_skips_humming(monkeypatch, bits) -> None:
|
||||||
|
"""4/8-bit int stays on the Marlin/GPTQ/AWQ path even on CUDA so a single
|
||||||
|
model can mix high-bit Marlin and low-bit humming layers."""
|
||||||
|
called = {"humming": False}
|
||||||
|
|
||||||
|
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||||
|
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
|
||||||
|
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||||
|
"inc_wna16_scheme._build_humming_linear_method",
|
||||||
|
lambda layer_config: called.update({"humming": True}),
|
||||||
|
)
|
||||||
|
|
||||||
|
method = INCWna16Scheme().get_linear_method(
|
||||||
|
make_config(),
|
||||||
|
object(),
|
||||||
|
"model.layers.0.mlp.down_proj",
|
||||||
|
make_layer_config(bits=bits),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert called["humming"] is False
|
||||||
|
assert isinstance(method, INCLinearMethod)
|
||||||
|
|
||||||
|
|
||||||
def test_inc_config_accepts_mxfp_family_llm_compressor() -> None:
|
def test_inc_config_accepts_mxfp_family_llm_compressor() -> None:
|
||||||
config = INCConfig.from_config(
|
config = INCConfig.from_config(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -34,9 +34,8 @@ class INCConfig(QuantizationConfig):
|
|||||||
Repo: https://github.com/intel/neural-compressor
|
Repo: https://github.com/intel/neural-compressor
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
|
||||||
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
|
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
|
||||||
|
|
||||||
SUPPORTED_BITS = {2, 3, 4, 8}
|
|
||||||
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
|
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
|
||||||
SUPPORTED_FORMATS = {
|
SUPPORTED_FORMATS = {
|
||||||
"auto_round:auto_gptq",
|
"auto_round:auto_gptq",
|
||||||
|
|||||||
@@ -21,6 +21,11 @@ if TYPE_CHECKING:
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
XPU_WNA16_SUPPORTED_BITS = {2, 4}
|
XPU_WNA16_SUPPORTED_BITS = {2, 4}
|
||||||
|
# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
|
||||||
|
# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
|
||||||
|
# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
|
||||||
|
# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
|
||||||
|
CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
|
||||||
|
|
||||||
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
|
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
|
||||||
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
|
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
|
||||||
@@ -133,6 +138,15 @@ class INCWna16Scheme(INCScheme):
|
|||||||
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
||||||
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
|
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
|
||||||
|
|
||||||
|
# CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
|
||||||
|
# so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
|
||||||
|
# layers.
|
||||||
|
if (
|
||||||
|
current_platform.is_cuda()
|
||||||
|
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
|
||||||
|
):
|
||||||
|
return _build_humming_linear_method(layer_config)
|
||||||
|
|
||||||
from .inc_wna16_linear import INCWNA16LinearScheme
|
from .inc_wna16_linear import INCWNA16LinearScheme
|
||||||
|
|
||||||
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
||||||
@@ -152,6 +166,12 @@ class INCWna16Scheme(INCScheme):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||||
|
# CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
|
||||||
|
if (
|
||||||
|
current_platform.is_cuda()
|
||||||
|
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
|
||||||
|
):
|
||||||
|
return _build_humming_moe_method(layer, layer_config)
|
||||||
if layer_config.is_gptq:
|
if layer_config.is_gptq:
|
||||||
return _resolve_gptq_moe(layer, layer_config)
|
return _resolve_gptq_moe(layer, layer_config)
|
||||||
if layer_config.is_awq:
|
if layer_config.is_awq:
|
||||||
@@ -251,3 +271,48 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
return MoeWNA16Method(moe_config, layer.moe_config)
|
return MoeWNA16Method(moe_config, layer.moe_config)
|
||||||
|
|
||||||
|
|
||||||
|
def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
|
||||||
|
"""Build the humming weight-schema config for a WNA16 int checkpoint."""
|
||||||
|
if layer_config.is_gptq:
|
||||||
|
return {
|
||||||
|
"quant_method": "gptq",
|
||||||
|
"bits": layer_config.bits,
|
||||||
|
"group_size": layer_config.group_size,
|
||||||
|
"desc_act": False,
|
||||||
|
"sym": layer_config.sym,
|
||||||
|
}
|
||||||
|
if layer_config.is_awq:
|
||||||
|
return {
|
||||||
|
"quant_method": "awq",
|
||||||
|
"bits": layer_config.bits,
|
||||||
|
"group_size": layer_config.group_size,
|
||||||
|
"zero_point": not layer_config.sym,
|
||||||
|
}
|
||||||
|
raise NotImplementedError(
|
||||||
|
"INC humming dispatch only supports gptq/awq packed int checkpoints, "
|
||||||
|
f"but found {layer_config}."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_humming_quant_config(layer_config: "INCLayerConfig"):
|
||||||
|
from vllm.model_executor.layers.quantization.humming import (
|
||||||
|
HummingLayerQuantizationConfig,
|
||||||
|
)
|
||||||
|
from vllm.utils.humming import BaseWeightSchema
|
||||||
|
|
||||||
|
weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
|
||||||
|
return HummingLayerQuantizationConfig(weight_schema=weight_schema)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_humming_linear_method(layer_config: "INCLayerConfig"):
|
||||||
|
from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
|
||||||
|
|
||||||
|
return HummingLinearMethod(_build_humming_quant_config(layer_config))
|
||||||
|
|
||||||
|
|
||||||
|
def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||||
|
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
|
||||||
|
|
||||||
|
return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)
|
||||||
|
|||||||
Reference in New Issue
Block a user