[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Luke Alonso <lalonso@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Luke Alonso
parent
ac71b17517
commit
6849ca696a
@@ -21,6 +21,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
B12xMxFp4LinearKernel,
|
||||
B12xMxfp8LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Mxfp8LinearLayerConfig,
|
||||
@@ -782,7 +783,7 @@ def test_b12x_nvfp4_can_implement_supported_config() -> None:
|
||||
assert reason is None
|
||||
|
||||
|
||||
def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
def test_b12x_backend_selects_w4a16(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear as linear_mod
|
||||
|
||||
monkeypatch.setattr(linear_mod.current_platform, "_enum", PlatformEnum.CUDA)
|
||||
@@ -792,17 +793,17 @@ def test_b12x_backend_preserves_w4a16_fallback(monkeypatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
linear_mod._POSSIBLE_NVFP4_KERNELS,
|
||||
PlatformEnum.CUDA,
|
||||
[B12xNvFp4LinearKernel, MarlinNvFp4LinearKernel],
|
||||
[B12xNvFp4LinearKernel, B12xNvFp4W4A16LinearKernel, MarlinNvFp4LinearKernel],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
MarlinNvFp4LinearKernel,
|
||||
B12xNvFp4W4A16LinearKernel,
|
||||
"is_supported",
|
||||
classmethod(lambda cls, compute_capability=None: (True, None)),
|
||||
)
|
||||
|
||||
kernel = init_nvfp4_linear_kernel(use_a16=True)
|
||||
|
||||
assert isinstance(kernel, MarlinNvFp4LinearKernel)
|
||||
assert isinstance(kernel, B12xNvFp4W4A16LinearKernel)
|
||||
|
||||
|
||||
def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
|
||||
@@ -859,3 +860,227 @@ def test_b12x_nvfp4_apply_calls_native_blockscaled_gemm(monkeypatch) -> None:
|
||||
layer.alpha,
|
||||
)
|
||||
assert kwargs == {"out_dtype": torch.bfloat16}
|
||||
|
||||
|
||||
def test_b12x_nvfp4_w4a16_uses_weight_only_scale_and_a16_dispatch(monkeypatch):
|
||||
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
|
||||
|
||||
calls = []
|
||||
packed_weight = object()
|
||||
|
||||
def pack_weight(weight, scale, *, recipe, global_scale):
|
||||
assert weight is layer.weight
|
||||
assert scale is layer.weight_scale
|
||||
assert recipe == "nvfp4"
|
||||
assert global_scale is layer.weight_global_scale
|
||||
return packed_weight
|
||||
|
||||
def mm(source, weight, *, required_mode):
|
||||
assert weight is packed_weight
|
||||
assert required_mode == "a16"
|
||||
assert source.dtype == torch.bfloat16
|
||||
assert source.is_contiguous()
|
||||
calls.append(source)
|
||||
return torch.full((source.shape[0], 48), 3.0, dtype=source.dtype)
|
||||
|
||||
def reject_activation_quantization(*args, **kwargs):
|
||||
pytest.fail("W4A16 must not quantize activations")
|
||||
|
||||
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_blockscaled",
|
||||
lambda: types.SimpleNamespace(pack_weight=pack_weight, mm=mm),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_intrinsics",
|
||||
lambda: types.SimpleNamespace(swizzle_block_scale=lambda scale: scale),
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((48, 64), dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.empty((48, 8), dtype=torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_global_scale = torch.tensor([0.125])
|
||||
kernel = object.__new__(B12xNvFp4W4A16LinearKernel)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.b12x_warmup_provider is kernel
|
||||
kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16).compile()
|
||||
assert [source.shape[0] for source in calls] == [1, 8]
|
||||
|
||||
x = torch.randn((2, 3, 256), dtype=torch.bfloat16)[..., ::2]
|
||||
output = kernel.apply_weights(layer, x, torch.ones(48, dtype=torch.bfloat16))
|
||||
torch.testing.assert_close(calls[-1], x.reshape(6, 128))
|
||||
torch.testing.assert_close(output, torch.full((2, 3, 48), 4.0, dtype=x.dtype))
|
||||
with pytest.raises(ValueError, match="requires BF16 input"):
|
||||
kernel.apply_weights(layer, x.float())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", [(2, 136, 96), (17, 256, 256)])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_nvfp4_w4a16_preserves_bf16_activations_under_graph_replay(
|
||||
monkeypatch, default_vllm_config, m, n, k
|
||||
) -> None:
|
||||
import vllm.model_executor.kernels.linear.nvfp4.b12x as b12x_mod
|
||||
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
def reject_activation_quantization(*args, **kwargs):
|
||||
pytest.fail("W4A16 must not quantize activations")
|
||||
|
||||
monkeypatch.setattr(b12x_mod, "scaled_fp4_quant", reject_activation_quantization)
|
||||
torch.manual_seed(17)
|
||||
codes = torch.randint(0, 16, (n, k), device="cuda")
|
||||
scales = (2.0 ** torch.randint(-2, 2, (n, k // 16), device="cuda")).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
(codes[:, ::2] | codes[:, 1::2] << 4).to(torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
torch.tensor([0.125], device="cuda"), requires_grad=False
|
||||
)
|
||||
lut = torch.tensor(
|
||||
[0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
|
||||
device="cuda",
|
||||
)
|
||||
decoded = lut[codes] * scales.float().repeat_interleave(16, dim=1) * 0.125
|
||||
x_storage = torch.randn(1, m, k * 2, device="cuda", dtype=torch.bfloat16)
|
||||
x = x_storage[..., ::2]
|
||||
bias = torch.randn(n, device="cuda", dtype=torch.bfloat16)
|
||||
kernel = B12xNvFp4W4A16LinearKernel(None)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
kernel.get_b12x_warmup_unit(layer, (1, 2, 4, 32), x.dtype).compile()
|
||||
run = torch.compile(
|
||||
lambda inputs: kernel.apply_weights(layer, inputs, bias),
|
||||
fullgraph=True,
|
||||
dynamic=True,
|
||||
)
|
||||
run(x)
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = run(x)
|
||||
for _ in range(2):
|
||||
x_storage.normal_()
|
||||
graph.replay()
|
||||
expected = (x.double() @ decoded.double().T).bfloat16() + bias
|
||||
torch.testing.assert_close(actual, expected, atol=0.005, rtol=0.01)
|
||||
|
||||
|
||||
def _block_weights(n, k, codec):
|
||||
block_size, block_bytes = (
|
||||
(32, 34) if codec == "q8_0" else (256, 66 if codec == "iq2_xxs" else 74)
|
||||
)
|
||||
raw = torch.randint(0, 256, (n, k // block_size, block_bytes), dtype=torch.uint8)
|
||||
raw[..., :2] = torch.full(
|
||||
(n, k // block_size, 1), 1 / 128, dtype=torch.float16
|
||||
).view(torch.uint8)
|
||||
return raw
|
||||
|
||||
|
||||
@pytest.mark.parametrize("codec", ["iq2_xs", "iq2_xxs", "q8_0"])
|
||||
@pytest.mark.parametrize("profile_first", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_block_quant_dense_compiled_graph(
|
||||
default_vllm_config, codec, profile_first
|
||||
):
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
from b12x.testing.iq2_xs_reference import dequantize_blocks
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
|
||||
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
|
||||
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
|
||||
method = ModelOptBlockQuantLinearMethod(codec)
|
||||
raw = _block_weights(128, 256, codec)
|
||||
decoded = dequantize_blocks(raw).bfloat16().cuda()
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
|
||||
method.process_weights_after_loading(layer)
|
||||
assert layer.b12x_block_plan.prepared is None
|
||||
if profile_first:
|
||||
method.apply(layer, torch.zeros(32, 256, device="cuda", dtype=torch.bfloat16))
|
||||
layer.b12x_warmup_provider.get_b12x_warmup_unit(
|
||||
layer, (1, 4, 32), torch.bfloat16
|
||||
).compile()
|
||||
assert layer.b12x_block_plan.prepared is not None
|
||||
run = torch.compile(lambda x: method.apply(layer, x), fullgraph=True, dynamic=True)
|
||||
for rows in (1, 4, 17):
|
||||
x = torch.randn(rows, 256, device="cuda", dtype=torch.bfloat16)
|
||||
run(x)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = run(x)
|
||||
for _ in range(2):
|
||||
x.normal_()
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
expected = (x.float() @ decoded.float().T).bfloat16()
|
||||
torch.testing.assert_close(output, expected, rtol=0.015, atol=0.03125)
|
||||
graph.reset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
|
||||
@pytest.mark.parametrize("profile_first", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_q8_embedding_graph(default_vllm_config, id_dtype, profile_first):
|
||||
supported, reason = B12xNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
from b12x.testing.q8_0_reference import dequantize_blocks
|
||||
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.scheduler_config.max_num_batched_tokens = 32
|
||||
default_vllm_config.scheduler_config.max_num_scheduled_tokens = 32
|
||||
default_vllm_config.compilation_config.cudagraph_capture_sizes = [1, 4]
|
||||
method = ModelOptBlockQuantLinearMethod("q8_0")
|
||||
method.is_embedding = True
|
||||
raw = _block_weights(128, 256, "q8_0")
|
||||
decoded = dequantize_blocks(raw).bfloat16().cuda()
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(raw.cuda(), requires_grad=False)
|
||||
method.process_weights_after_loading(layer)
|
||||
assert all(plan.prepared is None for plan in layer.b12x_embedding_plans.values())
|
||||
if profile_first:
|
||||
method.embedding(layer, torch.zeros(32, device="cuda", dtype=id_dtype))
|
||||
layer.b12x_warmup_provider.get_b12x_warmup_unit(
|
||||
layer, (1, 4, 32), torch.bfloat16
|
||||
).compile()
|
||||
assert all(
|
||||
plan.prepared is not None for plan in layer.b12x_embedding_plans.values()
|
||||
)
|
||||
run = torch.compile(
|
||||
lambda x: method.embedding(layer, x), fullgraph=True, dynamic=True
|
||||
)
|
||||
for count in (7, 9, 70):
|
||||
ids = torch.arange(count, device="cuda", dtype=id_dtype)
|
||||
torch._dynamo.mark_dynamic(ids, 0)
|
||||
torch.testing.assert_close(run(ids), decoded[ids.long()], rtol=0, atol=0)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = run(ids)
|
||||
ids.add_(20)
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
torch.testing.assert_close(output, decoded[ids.long()], rtol=0, atol=0)
|
||||
graph.reset()
|
||||
|
||||
Reference in New Issue
Block a user