[Misc] Update w4a16 compressed-tensors support to include w8a16 (#5794)
This commit is contained in:
@@ -8,9 +8,9 @@ import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||
CompressedTensorsLinearMethod, CompressedTensorsW4A16,
|
||||
CompressedTensorsW4A16Sparse24, CompressedTensorsW8A8DynamicToken,
|
||||
CompressedTensorsW8A8StaticTensor)
|
||||
CompressedTensorsLinearMethod, CompressedTensorsW4A16Sparse24,
|
||||
CompressedTensorsW8A8DynamicToken, CompressedTensorsW8A8StaticTensor,
|
||||
CompressedTensorsWNA16)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_args", [
|
||||
@@ -74,26 +74,27 @@ def test_compressed_tensors_w8a8_dynanmic_per_token(vllm_runner, model_args):
|
||||
assert qkv_proj.weight.dtype is torch.int8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("w4a16_args", [
|
||||
("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None),
|
||||
("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128),
|
||||
])
|
||||
def test_compressed_tensors_w4a16(vllm_runner, w4a16_args):
|
||||
model, strategy, group = w4a16_args
|
||||
@pytest.mark.parametrize(
|
||||
"wNa16_args",
|
||||
[("nm-testing/tinyllama-oneshot-w4a16-channel-v2", "channel", None, 8),
|
||||
("nm-testing/tinyllama-oneshot-w4a16-group128-v2", "group", 128, 8),
|
||||
("nm-testing/tinyllama-oneshot-w8a16-per-channel", "channel", None, 4)])
|
||||
def test_compressed_tensors_w4a16(vllm_runner, wNa16_args):
|
||||
model, strategy, group, pack_factor = wNa16_args
|
||||
with vllm_runner(model) as llm:
|
||||
model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
|
||||
layer = model.model.layers[0]
|
||||
|
||||
qkv_proj = layer.self_attn.qkv_proj
|
||||
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
|
||||
assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16)
|
||||
assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
|
||||
|
||||
assert qkv_proj.scheme.strategy == strategy
|
||||
assert qkv_proj.scheme.group_size == group
|
||||
|
||||
assert qkv_proj.weight_packed.dtype is torch.int32
|
||||
assert qkv_proj.weight_scale.dtype is torch.float16
|
||||
assert qkv_proj.weight_packed.pack_factor == 8
|
||||
assert qkv_proj.weight_packed.pack_factor == pack_factor
|
||||
|
||||
|
||||
def test_compressed_tensors_w4a16_marlin24(vllm_runner):
|
||||
|
||||
Reference in New Issue
Block a user