[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Luke Alonso <lalonso@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Luke Alonso
parent
ac71b17517
commit
6849ca696a
@@ -179,7 +179,14 @@ def _make_b12x_moe_kernel(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=hidden_states.shape[1],
|
||||
intermediate_size=w2.shape[2] * 2,
|
||||
intermediate_size=w2.shape[2]
|
||||
* (
|
||||
32
|
||||
if quant_config.weight_quant_dtype == "q8_0"
|
||||
else 256
|
||||
if quant_config.weight_quant_dtype in ("iq2_xs", "iq2_xxs")
|
||||
else 2
|
||||
),
|
||||
in_dtype=hidden_states.dtype,
|
||||
activation=activation,
|
||||
)
|
||||
@@ -1272,3 +1279,78 @@ def test_b12x_moe_cuda_graph_replay(
|
||||
assert torch.isfinite(expected).all()
|
||||
assert torch.isfinite(actual).all()
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _has_b12x_moe(), reason="requires b12x MoE on SM120")
|
||||
@pytest.mark.parametrize("codec", ["iq2_xxs", "q8_0"])
|
||||
@pytest.mark.parametrize("tokens", [1, 2, 4])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_block_quant_relu2_moe_graph(codec, tokens, workspace_init):
|
||||
from b12x.testing.iq2_xs_reference import moe_reference_iq2_xs
|
||||
|
||||
from tests.model_executor.kernels.test_b12x_linear import _block_weights
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
torch.manual_seed(71)
|
||||
e, h, i, topk = 4, 256, 256, 2
|
||||
raw1, raw2 = _block_weights(e * i, h, codec), _block_weights(e * h, i, codec)
|
||||
w1 = raw1.view(e, i, *raw1.shape[1:]).cuda()
|
||||
w2 = raw2.view(e, h, *raw2.shape[1:]).cuda()
|
||||
x = torch.randn(tokens, h, device="cuda", dtype=torch.bfloat16) * 0.125
|
||||
scores = torch.randn(tokens, e, device="cuda", dtype=torch.bfloat16)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantDesc
|
||||
|
||||
quant = FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(),
|
||||
_a2=FusedMoEQuantDesc(),
|
||||
_w1=FusedMoEQuantDesc(dtype=codec),
|
||||
_w2=FusedMoEQuantDesc(dtype=codec),
|
||||
)
|
||||
activation = MoEActivation.RELU2_NO_MUL
|
||||
kernel = _make_b12x_moe_kernel(x, w1, w2, topk, activation, quant)
|
||||
weights, ids, _ = fused_topk(x, scores, topk, renormalize=False)
|
||||
|
||||
def apply():
|
||||
return kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=weights,
|
||||
topk_ids=ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
apply()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = apply()
|
||||
for _ in range(2):
|
||||
x.normal_(std=0.125)
|
||||
scores.normal_()
|
||||
new_weights, new_ids, _ = fused_topk(x, scores, topk, renormalize=False)
|
||||
weights.copy_(new_weights)
|
||||
ids.copy_(new_ids)
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
expected = moe_reference_iq2_xs(
|
||||
x,
|
||||
raw1.view(e, i, *raw1.shape[1:]),
|
||||
raw2.view(e, h, *raw2.shape[1:]),
|
||||
ids,
|
||||
weights,
|
||||
activation="relu2",
|
||||
)
|
||||
assert torch.isfinite(output).all()
|
||||
relative_l2 = (
|
||||
output.float() - expected.float()
|
||||
).norm() / expected.float().norm()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), expected.float().flatten(), dim=0
|
||||
)
|
||||
assert cosine >= 0.999 and relative_l2 <= 0.01
|
||||
graph.reset()
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
B12xMxFp4LinearKernel,
|
||||
B12xMxfp8LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Mxfp8LinearLayerConfig,
|
||||
@@ -782,7 +783,7 @@ def test_b12x_nvfp4_can_implement_supported_config() -> None:
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
def test_b12x_backend_selects_w4a16(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear as linear_mod
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
@@ -792,17 +793,17 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
linear_mod._POSSIBLE_NVFP4_KERNELS,
|
||||
PlatformEnum.CUDA,
|
||||
[B12xNvFp4LinearKernel, MarlinNvFp4LinearKernel],
|
||||
[B12xNvFp4LinearKernel, B12xNvFp4W4A16LinearKernel, MarlinNvFp4LinearKernel],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MarlinNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
"is_supported",
|
||||
classmethod(lambda cls, compute_capability=None: (True, None)),
|
||||
)
|
||||
|
||||
kernel = init_nvfp4_linear_kernel(use_a16=True)
|
||||
|
||||
assert isinstance(kernel, MarlinNvFp4LinearKernel)
|
||||
assert isinstance(kernel, B12xNvFp4W4A16LinearKernel)
|
||||
|
||||
|
||||
def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
|
||||
@@ -859,3 +860,227 @@ def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
|
||||
layer.alpha,
|
||||
)
|
||||
assert kwargs == {"out_dtype": torch.bfloat16}
|
||||
|
||||
|
||||
def test_b12x_nvfp4_w4a16_uses_weight_only_scale_and_a16_dispatch(monkeypatch):
|
||||
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
|
||||
|
||||
calls = []
|
||||
packed_weight = object()
|
||||
|
||||
def pack_weight(weight, scale, *, recipe, global_scale):
|
||||
assert weight is layer.weight
|
||||
assert scale is layer.weight_scale
|
||||
assert recipe == "nvfp4"
|
||||
assert global_scale is layer.weight_global_scale
|
||||
return packed_weight
|
||||
|
||||
def mm(source, weight, *, required_mode):
|
||||
assert weight is packed_weight
|
||||
assert required_mode == "a16"
|
||||
assert source.dtype == torch.bfloat16
|
||||
assert source.is_contiguous()
|
||||
calls.append(source)
|
||||
return torch.full((source.shape[0], 48), 3.0, dtype=source.dtype)
|
||||
|
||||
def reject_activation_quantization(*args, **kwargs):
|
||||
pytest.fail("W4A16 must not quantize activations")
|
||||
|
||||
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_blockscaled",
|
||||
lambda: types.SimpleNamespace(pack_weight=pack_weight, mm=mm),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_intrinsics",
|
||||
lambda: types.SimpleNamespace(swizzle_block_scale=lambda scale: scale),
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((48, 64), dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.empty((48, 8), dtype=torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_global_scale = torch.tensor([0.125])
|
||||
kernel = object.__new__(B12xNvFp4W4A16LinearKernel)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.b12x_warmup_provider is kernel
|
||||
kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16).compile()
|
||||
assert [source.shape[0] for source in calls] == [1, 8]
|
||||
|
||||
x = torch.randn((2, 3, 256), dtype=torch.bfloat16)[..., ::2]
|
||||
output = kernel.apply_weights(layer, x, torch.ones(48, dtype=torch.bfloat16))
|
||||
torch.testing.assert_close(calls[-1], x.reshape(6, 128))
|
||||
torch.testing.assert_close(output, torch.full((2, 3, 48), 4.0, dtype=x.dtype))
|
||||
with pytest.raises(ValueError, match="requires BF16 input"):
|
||||
kernel.apply_weights(layer, x.float())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", [(2, 136, 96), (17, 256, 256)])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_nvfp4_w4a16_preserves_bf16_activations_under_graph_replay(
|
||||
monkeypatch, default_vllm_config, m, n, k
|
||||
) -> None:
|
||||
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
|
||||
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
def reject_activation_quantization(*args, **kwargs):
|
||||
pytest.fail("W4A16 must not quantize activations")
|
||||
|
||||
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
|
||||
torch.manual_seed(17)
|
||||
codes = torch.randint(0, 16, (n, k), device="cuda")
|
||||
scales = (2.0 ** torch.randint(-2, 2, (n, k // 16), device="cuda")).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
(codes[:, ::2] | codes[:, 1::2] << 4).to(torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
torch.tensor([0.125], device="cuda"), requires_grad=False
|
||||
)
|
||||
lut = torch.tensor(
|
||||
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
|
||||
device="cuda",
|
||||
)
|
||||
decoded = lut[codes] * scales.float().repeat_interleave(16, dim=1) * 0.125
|
||||
x_storage = torch.randn(1, m, k * 2, device="cuda", dtype=torch.bfloat16)
|
||||
x = x_storage[..., ::2]
|
||||
bias = torch.randn(n, device="cuda", dtype=torch.bfloat16)
|
||||
kernel = B12xNvFp4W4A16LinearKernel(None)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
kernel.get_b12x_warmup_unit(layer, (1, 2, 4, 32), x.dtype).compile()
|
||||
run = torch.compile(
|
||||
lambda inputs: kernel.apply_weights(layer, inputs, bias),
|
||||
fullgraph=True,
|
||||
dynamic=True,
|
||||
)
|
||||
run(x)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = run(x)
|
||||
for _ in range(2):
|
||||
x_storage.normal_()
|
||||
graph.replay()
|
||||
expected = (x.double() @ decoded.double().T).bfloat16() + bias
|
||||
torch.testing.assert_close(actual, expected, atol=0.005, rtol=0.01)
|
||||
|
||||
|
||||
def _block_weights(n, k, codec):
|
||||
block_size, block_bytes = (
|
||||
(32, 34) if codec == "q8_0" else (256, 66 if codec == "iq2_xxs" else 74)
|
||||
)
|
||||
raw = torch.randint(0, 256, (n, k // block_size, block_bytes), dtype=torch.uint8)
|
||||
raw[..., :2] = torch.full(
|
||||
(n, k // block_size, 1), 1 / 128, dtype=torch.float16
|
||||
).view(torch.uint8)
|
||||
return raw
|
||||
|
||||
|
||||
@pytest.mark.parametrize("codec", ["iq2_xs", "iq2_xxs", "q8_0"])
|
||||
@pytest.mark.parametrize("profile_first", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_block_quant_dense_compiled_graph(
|
||||
default_vllm_config, codec, profile_first
|
||||
):
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
from b12x.testing.iq2_xs_reference import dequantize_blocks
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
|
||||
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
|
||||
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
|
||||
method = ModelOptBlockQuantLinearMethod(codec)
|
||||
raw = _block_weights(128, 256, codec)
|
||||
decoded = dequantize_blocks(raw).bfloat16().cuda()
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
|
||||
method.process_weights_after_loading(layer)
|
||||
assert layer.b12x_block_plan.prepared is None
|
||||
if profile_first:
|
||||
method.apply(layer, torch.zeros(32, 256, device="cuda", dtype=torch.bfloat16))
|
||||
layer.b12x_warmup_provider.get_b12x_warmup_unit(
|
||||
layer, (1, 4, 32), torch.bfloat16
|
||||
).compile()
|
||||
assert layer.b12x_block_plan.prepared is not None
|
||||
run = torch.compile(lambda x: method.apply(layer, x), fullgraph=True, dynamic=True)
|
||||
for rows in (1, 4, 17):
|
||||
x = torch.randn(rows, 256, device="cuda", dtype=torch.bfloat16)
|
||||
run(x)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = run(x)
|
||||
for _ in range(2):
|
||||
x.normal_()
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
expected = (x.float() @ decoded.float().T).bfloat16()
|
||||
torch.testing.assert_close(output, expected, rtol=0.015, atol=0.03125)
|
||||
graph.reset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
|
||||
@pytest.mark.parametrize("profile_first", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_q8_embedding_graph(default_vllm_config, id_dtype, profile_first):
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
from b12x.testing.q8_0_reference import dequantize_blocks
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
|
||||
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
|
||||
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
|
||||
method = ModelOptBlockQuantLinearMethod("q8_0")
|
||||
method.is_embedding = True
|
||||
raw = _block_weights(128, 256, "q8_0")
|
||||
decoded = dequantize_blocks(raw).bfloat16().cuda()
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
|
||||
method.process_weights_after_loading(layer)
|
||||
assert all(plan.prepared is None for plan in layer.b12x_embedding_plans.values())
|
||||
if profile_first:
|
||||
method.embedding(layer, torch.zeros(32, device="cuda", dtype=id_dtype))
|
||||
layer.b12x_warmup_provider.get_b12x_warmup_unit(
|
||||
layer, (1, 4, 32), torch.bfloat16
|
||||
).compile()
|
||||
assert all(
|
||||
plan.prepared is not None for plan in layer.b12x_embedding_plans.values()
|
||||
)
|
||||
run = torch.compile(
|
||||
lambda x: method.embedding(layer, x), fullgraph=True, dynamic=True
|
||||
)
|
||||
for count in (7, 9, 70):
|
||||
ids = torch.arange(count, device="cuda", dtype=id_dtype)
|
||||
torch._dynamo.mark_dynamic(ids, 0)
|
||||
torch.testing.assert_close(run(ids), decoded[ids.long()], rtol=0, atol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = run(ids)
|
||||
ids.add_(20)
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
torch.testing.assert_close(output, decoded[ids.long()], rtol=0, atol=0)
|
||||
graph.reset()
|
||||
|
||||
@@ -25,6 +25,60 @@ def test_b12x_warmup_token_counts_cover_serving_regimes() -> None:
|
||||
) == (1, 2, 8, 128, 2048)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_embedding", [False, True])
|
||||
def test_b12x_block_quant_warmup_covers_each_layers_plans(monkeypatch, is_embedding):
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.warmup.b12x_warmup import _collect_warmup_units
|
||||
|
||||
method = object.__new__(ModelOptBlockQuantLinearMethod)
|
||||
method.codec = "q8_0"
|
||||
method.is_embedding = is_embedding
|
||||
model = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
method, "apply", lambda layer, x: calls.append((layer, x.shape, x.dtype))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"embedding",
|
||||
lambda layer, ids: calls.append((layer, ids.shape, ids.dtype)),
|
||||
)
|
||||
for index, layer in enumerate(model):
|
||||
layer.b12x_warmup_provider = method
|
||||
if is_embedding:
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((8, 8, 34), dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.b12x_embedding_plans = {
|
||||
dtype: SimpleNamespace(
|
||||
handle=2 * index + offset, query=SimpleNamespace(max_rows=32)
|
||||
)
|
||||
for offset, dtype in enumerate((torch.int32, torch.int64))
|
||||
}
|
||||
else:
|
||||
layer.b12x_block_weight = SimpleNamespace(
|
||||
in_features=256, values=torch.empty(1)
|
||||
)
|
||||
layer.b12x_block_plan = SimpleNamespace(
|
||||
handle=index, token_counts=(1, 4, 32)
|
||||
)
|
||||
|
||||
units = list(_collect_warmup_units(model, (1, 17, 32), torch.bfloat16))
|
||||
assert len(units) == 2
|
||||
for unit in units:
|
||||
unit.compile()
|
||||
expected = (
|
||||
[((32,), torch.int32), ((32,), torch.int64)]
|
||||
if is_embedding
|
||||
else [((rows, 256), torch.bfloat16) for rows in (1, 4, 17, 32)]
|
||||
)
|
||||
assert calls == [
|
||||
(layer, shape, dtype) for layer in model for shape, dtype in expected
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_cls", "module_name", "call_name", "layer", "name"),
|
||||
[
|
||||
|
||||
@@ -7,6 +7,7 @@ Run `pytest tests/quantization/test_modelopt.py`.
|
||||
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, NoReturn
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
@@ -1027,3 +1028,248 @@ def test_modelopt_fp8_pb_wo_rejects_non_128_input():
|
||||
scheme.create_weights(
|
||||
torch.nn.Module(), mo.WEIGHT, mo.CkptCtx(), shapes, Mock()
|
||||
)
|
||||
|
||||
|
||||
def _iq2_xs_moe_config(tp_size=1, tp_rank=0):
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
return SimpleNamespace(
|
||||
moe_backend="b12x",
|
||||
in_dtype=torch.bfloat16,
|
||||
activation=MoEActivation.SILU,
|
||||
has_bias=False,
|
||||
swiglu_limit=None,
|
||||
swiglu_alpha=None,
|
||||
swiglu_beta=None,
|
||||
hidden_dim=256,
|
||||
intermediate_size_per_partition=512 // tp_size,
|
||||
tp_rank=tp_rank,
|
||||
moe_parallel_config=SimpleNamespace(
|
||||
use_ep=False,
|
||||
ep_size=1,
|
||||
use_all2all_kernels=False,
|
||||
enable_eplb=False,
|
||||
tp_size=tp_size,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"codec,block_size,block_bytes",
|
||||
[("iq2_xs", 256, 74), ("iq2_xxs", 256, 66), ("q8_0", 32, 34)],
|
||||
)
|
||||
def test_modelopt_mixed_precision_dispatches_iq2_xs_experts(
|
||||
monkeypatch, codec, block_size, block_bytes
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.b12x import B12xExperts
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantMoEMethod,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(B12xExperts, "_supports_current_device", lambda: True)
|
||||
config = _mixed_precision_config(
|
||||
{
|
||||
"model.language_model.layers.0.mlp.experts": {
|
||||
"quant_algo": codec.upper(),
|
||||
"group_size": block_size,
|
||||
"block_payload_bytes": block_bytes,
|
||||
"packing": "ggml",
|
||||
},
|
||||
}
|
||||
)
|
||||
layer = MagicMock(spec=RoutedExperts)
|
||||
layer.moe_config = _iq2_xs_moe_config()
|
||||
method = config.get_quant_method(
|
||||
layer,
|
||||
"language_model.model.layers.0.mlp.experts",
|
||||
)
|
||||
assert isinstance(method, ModelOptBlockQuantMoEMethod)
|
||||
quant = method.get_fused_moe_quant_config(layer)
|
||||
assert quant.weight_quant_dtype == codec
|
||||
assert quant.quant_dtype is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("group_size", 0),
|
||||
("block_payload_bytes", 75),
|
||||
("packing", "unknown"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"codec,block_size,block_bytes",
|
||||
[("iq2_xs", 256, 74), ("iq2_xxs", 256, 66), ("q8_0", 32, 34)],
|
||||
)
|
||||
def test_modelopt_block_quant_rejects_incompatible_block_metadata(
|
||||
field, value, codec, block_size, block_bytes
|
||||
):
|
||||
recipe = {
|
||||
"quant_algo": codec.upper(),
|
||||
"group_size": block_size,
|
||||
"block_payload_bytes": block_bytes,
|
||||
"packing": "ggml",
|
||||
field: value,
|
||||
}
|
||||
with pytest.raises(ValueError, match=f"unsupported {codec.upper()} block contract"):
|
||||
_mixed_precision_config({"model.layers.0.mlp.experts": recipe})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parallel", ["row", "column"])
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize(
|
||||
"codec,block_size,block_bytes",
|
||||
[("iq2_xs", 256, 74), ("iq2_xxs", 256, 66), ("q8_0", 32, 34)],
|
||||
)
|
||||
def test_modelopt_block_quant_dense_loading_preserves_whole_blocks(
|
||||
monkeypatch, parallel, tp_rank, codec, block_size, block_bytes
|
||||
):
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
import vllm.utils.b12x as b12x_utils
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
b12x_utils,
|
||||
"get_b12x_blockscaled",
|
||||
lambda: SimpleNamespace(
|
||||
BlockQuantLinearWeight=object,
|
||||
is_supported=lambda: True,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_rank", lambda: tp_rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: 2
|
||||
)
|
||||
prefix = "model.layers.1.mixer.shared_experts.up_proj"
|
||||
config = _mixed_precision_config(
|
||||
{
|
||||
prefix: {
|
||||
"quant_algo": codec.upper(),
|
||||
"group_size": block_size,
|
||||
"block_payload_bytes": block_bytes,
|
||||
"packing": "ggml",
|
||||
}
|
||||
}
|
||||
)
|
||||
method = config.get_quant_method(MagicMock(spec=LinearBase), prefix)
|
||||
assert isinstance(method, ModelOptBlockQuantLinearMethod)
|
||||
layer = torch.nn.Module()
|
||||
n, k = (128, 256) if parallel == "row" else (64, 512)
|
||||
method.create_weights(
|
||||
layer, k, [n], 512, 128, torch.bfloat16, weight_loader=lambda *args: None
|
||||
)
|
||||
source = torch.randint(
|
||||
0, 256, (128, 512 // block_size, block_bytes), dtype=torch.uint8
|
||||
)
|
||||
if parallel == "row":
|
||||
layer.weight.load_row_parallel_weight(source)
|
||||
expected = source[
|
||||
:, tp_rank * (256 // block_size) : (tp_rank + 1) * (256 // block_size)
|
||||
]
|
||||
else:
|
||||
layer.weight.load_column_parallel_weight(source)
|
||||
expected = source[tp_rank * n : (tp_rank + 1) * n]
|
||||
torch.testing.assert_close(layer.weight, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size,tp_rank", [(1, 0), (2, 0), (2, 1)])
|
||||
@pytest.mark.parametrize("gated", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"codec,block_size,block_bytes",
|
||||
[("iq2_xs", 256, 74), ("iq2_xxs", 256, 66), ("q8_0", 32, 34)],
|
||||
)
|
||||
def test_modelopt_block_quant_loads_expert_blocks_with_aligned_tp_slices(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
tp_rank,
|
||||
gated,
|
||||
codec,
|
||||
block_size,
|
||||
block_bytes,
|
||||
):
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
from vllm.model_executor.layers.fused_moe.b12x import B12xExperts
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantMoEMethod,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(B12xExperts, "_supports_current_device", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_rank", lambda: tp_rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
config = _iq2_xs_moe_config(tp_size, tp_rank)
|
||||
if not gated:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
config.activation = MoEActivation.RELU2_NO_MUL
|
||||
method = ModelOptBlockQuantMoEMethod(config, codec=codec)
|
||||
layer = torch.nn.Module()
|
||||
layer.layer_name = "model.layers.0.mlp.experts"
|
||||
layer.is_fused_checkpoint_transposed = False
|
||||
layer._orient_fused_weight = RoutedExperts._orient_fused_weight
|
||||
layer.get_expert_mapping = lambda **kwargs: (
|
||||
RoutedExperts.build_expert_params_mapping(
|
||||
"gate_proj",
|
||||
"down_proj",
|
||||
"up_proj",
|
||||
num_experts=2,
|
||||
routed_experts_prefix="",
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
local_i = 512 // tp_size
|
||||
method.create_weights(layer, 2, 256, local_i, torch.bfloat16)
|
||||
assert layer.w13_weight.shape[1] == local_i * (2 if gated else 1)
|
||||
layer.w13_weight.data.zero_()
|
||||
layer.w2_weight.data.zero_()
|
||||
for shard, name in (
|
||||
("w1", "w13_weight"),
|
||||
("w3", "w13_weight"),
|
||||
("w2", "w2_weight"),
|
||||
):
|
||||
if shard == "w3" and not gated:
|
||||
continue
|
||||
shape = (
|
||||
(256, 512 // block_size, block_bytes)
|
||||
if shard == "w2"
|
||||
else (512, 256 // block_size, block_bytes)
|
||||
)
|
||||
source = torch.randint(1, 256, shape, dtype=torch.uint8)
|
||||
param = getattr(layer, name)
|
||||
projection = {"w1": "gate", "w3": "up", "w2": "down"}[shard]
|
||||
loaded = list(
|
||||
RoutedExperts.load_weights(
|
||||
layer,
|
||||
[(f"1.{projection}_proj.weight", source)],
|
||||
)
|
||||
)
|
||||
assert loaded == [name]
|
||||
if shard == "w2":
|
||||
expected = source[
|
||||
:,
|
||||
tp_rank * (local_i // block_size) : (tp_rank + 1)
|
||||
* (local_i // block_size),
|
||||
]
|
||||
actual = param[1]
|
||||
else:
|
||||
expected = source[tp_rank * local_i : (tp_rank + 1) * local_i]
|
||||
offset = 0 if shard == "w1" else local_i
|
||||
actual = param[1, offset : offset + local_i]
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
assert param[0].count_nonzero() == 0
|
||||
with pytest.raises(ValueError, match="invalid packed tensor"):
|
||||
layer.w2_weight.weight_loader(
|
||||
layer.w2_weight,
|
||||
torch.zeros(256, 512),
|
||||
"w2_weight",
|
||||
"w2",
|
||||
0,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user