[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
-294
@@ -1,294 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
from compressed_tensors.quantization import (
|
||||
QuantizationArgs,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
int4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _pack_shuffled_int8_to_packed_int4_no_perm(x_shuf_i8: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack a preshuffled int8 tensor (values in [-8, 7]) into packed int4 bytes.
|
||||
Each contiguous 8-value block [v0..v7] -> 4 bytes:
|
||||
b0=(v4<<4)|v0, b1=(v5<<4)|v1, b2=(v6<<4)|v2, b3=(v7<<4)|v3.
|
||||
This matches the 7-op in-kernel unpack sequence and avoids any v_perm.
|
||||
"""
|
||||
flat = x_shuf_i8.contiguous().view(-1).to(torch.int16)
|
||||
assert flat.numel() % 8 == 0
|
||||
u = (flat & 0xF).to(torch.uint8).view(-1, 8)
|
||||
out = torch.empty((u.shape[0], 4), device=u.device, dtype=torch.uint8)
|
||||
out[:, 0] = u[:, 0] | (u[:, 4] << 4)
|
||||
out[:, 1] = u[:, 1] | (u[:, 5] << 4)
|
||||
out[:, 2] = u[:, 2] | (u[:, 6] << 4)
|
||||
out[:, 3] = u[:, 3] | (u[:, 7] << 4)
|
||||
return out.view(-1).to(torch.int8)
|
||||
|
||||
|
||||
def _unpack_gptq_int32_to_signed_int4(w_int32):
|
||||
"""Unpack GPTQ int32 [E, K//8, N] to signed int4 values [E, N, K] (as int8).
|
||||
Shared by both the packed-int4 and bf16-dequant paths.
|
||||
"""
|
||||
E = w_int32.shape[0]
|
||||
# [E, K//8, N] -> transpose -> [E, N, K//8]
|
||||
w = w_int32.transpose(1, 2).contiguous()
|
||||
N = w.shape[1]
|
||||
K_div8 = w.shape[2]
|
||||
K = K_div8 * 8
|
||||
|
||||
# Unpack int32 -> 8 x uint4 values along K
|
||||
w_expanded = w.unsqueeze(-1).expand(E, N, K_div8, 8) # [E, N, K//8, 8]
|
||||
shifts = torch.arange(8, device=w.device) * 4 # [0, 4, 8, ..., 28]
|
||||
nibbles = ((w_expanded >> shifts) & 0xF).to(torch.int8) # [E, N, K//8, 8]
|
||||
nibbles = nibbles.reshape(E, N, K) # [E, N, K] unsigned int4 as int8
|
||||
|
||||
# Convert unsigned [0,15] to signed [-8,7]
|
||||
signed = nibbles.to(torch.int16) - 8
|
||||
signed = signed.to(torch.int8) # [E, N, K] signed int4 as int8
|
||||
return signed
|
||||
|
||||
|
||||
def _gptq_int32_to_flydsl_packed(w_int32):
|
||||
"""Convert GPTQ int32 [E, K//8, N] to FlyDSL shuffled packed int4 [E, N, K//2].
|
||||
Steps:
|
||||
1. Unpack int32 to individual signed int4 values (as int8)
|
||||
2. Apply FlyDSL preshuffle (on individual int8 values)
|
||||
3. Pack with FlyDSL's interleaved int4 packing
|
||||
"""
|
||||
signed = _unpack_gptq_int32_to_signed_int4(w_int32)
|
||||
E, N, K = signed.shape
|
||||
|
||||
# FlyDSL preshuffle (operates on individual values)
|
||||
shuffled = shuffle_weight(signed, layout=(16, 16))
|
||||
|
||||
# FlyDSL interleaved int4 packing
|
||||
packed = _pack_shuffled_int8_to_packed_int4_no_perm(shuffled).contiguous()
|
||||
return packed.view(E, N, K // 2)
|
||||
|
||||
|
||||
class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant: QuantizationArgs,
|
||||
input_quant: QuantizationArgs | None,
|
||||
moe: FusedMoEConfig,
|
||||
layer_name: str | None = None,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.weight_quant = weight_quant
|
||||
self.input_quant = input_quant
|
||||
# Extract properties from weight_quant
|
||||
assert weight_quant.num_bits == 4
|
||||
self.num_bits = weight_quant.num_bits
|
||||
self.packed_factor = 32 // weight_quant.num_bits
|
||||
self.strategy = weight_quant.strategy
|
||||
# channelwise is not supported by this kernel
|
||||
assert weight_quant.strategy == "group"
|
||||
assert weight_quant.group_size == 32
|
||||
self.group_size = weight_quant.group_size
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
self.num_experts = num_experts
|
||||
self.inter_dim = intermediate_size_per_partition
|
||||
# Will transpose the loaded weight along the
|
||||
# intermediate and hidden dim sizes. Will
|
||||
# shard for TP along the transposed dims
|
||||
extra_weight_attrs.update(
|
||||
{"is_transposed": True, "quant_method": self.strategy}
|
||||
)
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size // self.packed_factor,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
intermediate_size_per_partition // self.packed_factor,
|
||||
hidden_size,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w2_scales_size = intermediate_size_per_partition
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
self.group_size = -1
|
||||
else:
|
||||
num_groups_w2 = w2_scales_size // self.group_size
|
||||
num_groups_w13 = hidden_size // self.group_size
|
||||
|
||||
w13_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
num_groups_w13,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_scale)
|
||||
set_weight_attrs(w13_scale, extra_weight_attrs)
|
||||
|
||||
w2_scale = torch.nn.Parameter(
|
||||
torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_scale)
|
||||
set_weight_attrs(w2_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
layer.register_parameter("w2_weight_shape", w2_weight_shape)
|
||||
set_weight_attrs(w2_weight_shape, extra_weight_attrs)
|
||||
w13_weight_shape = torch.nn.Parameter(
|
||||
torch.empty(num_experts, 2), requires_grad=False
|
||||
)
|
||||
|
||||
layer.register_parameter("w13_weight_shape", w13_weight_shape)
|
||||
set_weight_attrs(w13_weight_shape, extra_weight_attrs)
|
||||
|
||||
layer.a13_scale = None
|
||||
layer.a2_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Reconfigure packed weights and scales to match flydsl_w4a16 format
|
||||
|
||||
# Convert w13 weights
|
||||
w13 = layer.w13_weight_packed.data
|
||||
w13 = _gptq_int32_to_flydsl_packed(w13)
|
||||
w13 = w13.view(-1).contiguous()
|
||||
layer.w13_weight_packed = torch.nn.Parameter(w13, requires_grad=False)
|
||||
|
||||
# Convert w2 weights
|
||||
w2 = layer.w2_weight_packed.data
|
||||
w2 = _gptq_int32_to_flydsl_packed(w2)
|
||||
w2 = w2.view(-1).contiguous()
|
||||
layer.w2_weight_packed = torch.nn.Parameter(w2, requires_grad=False)
|
||||
|
||||
# Convert scales for FlyDSL:
|
||||
# per-row: [E, 1, N] -> squeeze -> [E, N]
|
||||
# groupwise: [E, K//gs, N] -> keep as-is (Opt 0: cache-friendly layout)
|
||||
w13_scale = layer.w13_weight_scale.data
|
||||
if self.group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
|
||||
E, G, N = w13_scale.shape
|
||||
w13_scale = (
|
||||
w13_scale.view(E, G // 2, 2, N)
|
||||
.permute(0, 1, 3, 2)
|
||||
.contiguous()
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
|
||||
# Per-row: squeeze [E, 1, N] -> [E, N]
|
||||
w13_scale = w13_scale.squeeze(1)
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
w13_scale.contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
w2_scale = layer.w2_weight_scale.data
|
||||
if self.group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
|
||||
E, G, N = w2_scale.shape
|
||||
w2_scale = (
|
||||
w2_scale.view(E, G // 2, 2, N)
|
||||
.permute(0, 1, 3, 2)
|
||||
.contiguous()
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
|
||||
# Per-row: squeeze [E, 1, N] -> [E, N]
|
||||
w2_scale = w2_scale.squeeze(1)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
w2_scale.contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
layer.w13_weight_packed.is_shuffled = True
|
||||
layer.w2_weight_packed.is_shuffled = True
|
||||
layer.is_aiter_converted = True
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
assert self.num_bits == 4
|
||||
return int4_w4a16_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
block_shape=[0, self.group_size],
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import (
|
||||
fused_flydsl_moe,
|
||||
)
|
||||
|
||||
assert self.moe_quant_config is not None
|
||||
|
||||
return fused_flydsl_moe(
|
||||
x,
|
||||
layer.w13_weight_packed,
|
||||
layer.w2_weight_packed,
|
||||
self.num_experts,
|
||||
self.inter_dim,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
w1_scale=self.moe_quant_config.w1_scale,
|
||||
w2_scale=self.moe_quant_config.w2_scale,
|
||||
topk=topk_weights.shape[-1],
|
||||
group_size=self.group_size,
|
||||
doweight_stage1=layer.apply_router_weight_on_input,
|
||||
scale_is_bf16=True,
|
||||
)
|
||||
-272
@@ -1,272 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
MarlinExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
is_flashinfer_moe_ep_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
def __init__(self, moe):
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None = None
|
||||
|
||||
# Backend selection must match the weight preparation below: CUTLASS
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
|
||||
# repacks weights and scales, and the megakernels take the checkpoint
|
||||
# tensors as they are.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
|
||||
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
|
||||
assert experts_cls is not None
|
||||
self.experts_cls = experts_cls
|
||||
self.use_cutlass_mxfp4 = False
|
||||
elif self.use_cutlass_mxfp4:
|
||||
logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE")
|
||||
self.experts_cls = CutlassExpertsMxfp4
|
||||
elif current_platform.is_xpu():
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.XPU
|
||||
self.experts_cls = XPUExpertsMxFp4
|
||||
logger.info_once("Using XPUExpertsMxFp4 for MXFP4 MoE on XPU platform")
|
||||
else:
|
||||
logger.info_once("Using MarlinExperts for MXFP4 MoE")
|
||||
self.experts_cls = MarlinExperts
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.num_experts = num_experts
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // 2,
|
||||
requires_grad=False,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_packed", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
intermediate_size_per_partition // 2,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
self.moe.w13_num_shards * intermediate_size_per_partition,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
hidden_size // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
# 2 fp4 items are packed in the input dimension
|
||||
intermediate_size_per_partition // self.group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.use_cutlass_mxfp4:
|
||||
# W4A4: both weights and activations quantized to MXFP4
|
||||
return mxfp4_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
)
|
||||
else:
|
||||
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.w13_weight = torch.nn.Parameter(
|
||||
layer.w13_weight_packed.data, requires_grad=False
|
||||
)
|
||||
delattr(layer, "w13_weight_packed")
|
||||
|
||||
layer.w2_weight = torch.nn.Parameter(
|
||||
layer.w2_weight_packed.data, requires_grad=False
|
||||
)
|
||||
delattr(layer, "w2_weight_packed")
|
||||
|
||||
if self.use_cutlass_mxfp4:
|
||||
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
|
||||
# to CUTLASS tiled layout [E, numMTiles*numKTiles*512].
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
|
||||
E = layer.w13_weight_scale.shape[0]
|
||||
w13_N = layer.w13_weight_scale.shape[1]
|
||||
w13_scale_K = layer.w13_weight_scale.shape[2]
|
||||
w13_K = w13_scale_K * 32
|
||||
|
||||
w2_M = layer.w2_weight_scale.shape[1]
|
||||
w2_scale_N = layer.w2_weight_scale.shape[2]
|
||||
w2_N = w2_scale_N * 32
|
||||
|
||||
swizzled_w13 = []
|
||||
swizzled_w2 = []
|
||||
for e_idx in range(E):
|
||||
s13 = layer.w13_weight_scale[e_idx]
|
||||
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
|
||||
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
|
||||
s2 = layer.w2_weight_scale[e_idx]
|
||||
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
|
||||
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w13), requires_grad=False
|
||||
)
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
w13, w2, w13_scale, w2_scale, _, _ = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
w13_weight_scale=layer.w13_weight_scale,
|
||||
w2_weight_scale=layer.w2_weight_scale,
|
||||
)
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP4 computation "
|
||||
"but FP4 quantization is being used. Weight-only FP4 "
|
||||
"compression will be used leveraging the Marlin kernel. "
|
||||
"This may degrade performance for compute-heavy workloads."
|
||||
)
|
||||
prepare_moe_fp4_layer_for_marlin(layer)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
-34
@@ -1,34 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import Mxfp4MoEMethod
|
||||
|
||||
_PACKED_WEIGHTS = ("w13_weight", "w2_weight")
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Mxfp4MoEMethod(Mxfp4MoEMethod):
|
||||
"""MXFP4 weights with dynamic FP8 activations in compressed-tensors format.
|
||||
|
||||
Only the checkpoint naming and the declared activation scheme differ from
|
||||
the native MXFP4 path, so backend selection, weight conversion and
|
||||
execution are inherited.
|
||||
"""
|
||||
|
||||
# The packed -> unpacked rename means a processed state dict no longer
|
||||
# matches the parameters registered by create_weights.
|
||||
supports_pre_processed_weights = False
|
||||
|
||||
def create_weights(self, layer: RoutedExperts, *args, **kwargs):
|
||||
super().create_weights(layer, *args, **kwargs)
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, name)
|
||||
delattr(layer, name)
|
||||
layer.register_parameter(f"{name}_packed", param)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
for name in _PACKED_WEIGHTS:
|
||||
param = getattr(layer, f"{name}_packed")
|
||||
delattr(layer, f"{name}_packed")
|
||||
layer.register_parameter(name, param)
|
||||
super().process_weights_after_loading(layer)
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe import ( # noqa: E501
|
||||
CompressedTensorsMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
|
||||
class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
"""Compressed-tensors MoE method for pre-quantized MXFP8 (W8A8) checkpoints.
|
||||
|
||||
Loads FP8 (E4M3) weights with E8M0 uint8 per-group scales (group_size=32)
|
||||
from checkpoint. Activations are dynamically quantized to MXFP8 at runtime.
|
||||
Supports FlashInfer TRT-LLM and Marlin backends (auto-selected).
|
||||
"""
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
|
||||
self.fp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.num_experts = num_experts
|
||||
layer.params_dtype = params_dtype
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
extra_weight_attrs.update(
|
||||
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
|
||||
)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=self.fp8_backend,
|
||||
layer=layer,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_input_scale=layer.w13_input_scale,
|
||||
w2_input_scale=layer.w2_input_scale,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_fp8_moe_quant_config(
|
||||
fp8_backend=self.fp8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -5,8 +5,8 @@ from .compressed_tensors_scheme import CompressedTensorsScheme
|
||||
from .compressed_tensors_w4a4_mxfp4 import CompressedTensorsW4A4Mxfp4
|
||||
from .compressed_tensors_w4a4_nvfp4 import CompressedTensorsW4A4Fp4
|
||||
from .compressed_tensors_w4a8_fp8 import CompressedTensorsW4A8Fp8
|
||||
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
|
||||
from .compressed_tensors_w4a8_mxfp4 import CompressedTensorsW4A8Mxfp4
|
||||
from .compressed_tensors_w4a8_int import CompressedTensorsW4A8Int
|
||||
from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8
|
||||
from .compressed_tensors_w8a8_int8 import CompressedTensorsW8A8Int8
|
||||
from .compressed_tensors_w8a8_mxfp8 import CompressedTensorsW8A8Mxfp8
|
||||
@@ -23,12 +23,12 @@ __all__ = [
|
||||
"CompressedTensorsW8A16Fp8",
|
||||
"CompressedTensorsW8A8Int8",
|
||||
"CompressedTensorsW8A8Fp8",
|
||||
"CompressedTensorsW4A4Fp4",
|
||||
"CompressedTensorsW4A4Mxfp4",
|
||||
"CompressedTensorsW4A8Mxfp4",
|
||||
"CompressedTensorsW4A4Fp4",
|
||||
"CompressedTensorsW8A8Mxfp8",
|
||||
"CompressedTensorsW4A8Int",
|
||||
"CompressedTensorsW4A8Fp8",
|
||||
"CompressedTensorsW8A8Mxfp8",
|
||||
"CompressedTensorsWNA4Int",
|
||||
"CompressedTensorsWNA8Int",
|
||||
]
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TurboQuant: KV-cache quantization for vLLM.
|
||||
|
||||
Hadamard rotation + per-coordinate Lloyd-Max scalar quantization for
|
||||
keys, uniform quantization for values.
|
||||
|
||||
The core algorithmic pattern implemented for key quantization (Hadamard
|
||||
rotation followed by deterministic scalar quantization and
|
||||
re-normalization) was originally established in DRIVE (Vargaftik et al.,
|
||||
NeurIPS 2021) and EDEN (Vargaftik et al., ICML 2022). This formulation is
|
||||
also mathematically equivalent to the scalar case of the HIGGS
|
||||
quantization method (Malinovskii et al., "Pushing the Limits of Large
|
||||
Language Model Quantization via the Linearity Theorem", NAACL 2025;
|
||||
preprint arXiv:2411.17525), which subsequently generalized these concepts.
|
||||
|
||||
A first application of this approach to KV-cache compression is in "Cache
|
||||
Me If You Must: Adaptive Key-Value Quantization for Large Language Models"
|
||||
(Shutova et al., ICML 2025; preprint arXiv:2501.19392). All of these
|
||||
foundational and application references pre-date the TurboQuant paper
|
||||
(Zandieh et al., ICLR 2026).
|
||||
"""
|
||||
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import TurboQuantConfig
|
||||
|
||||
__all__ = ["TurboQuantConfig"]
|
||||
@@ -1,87 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lloyd-Max optimal scalar quantizer for TurboQuant.
|
||||
|
||||
After rotating a d-dimensional unit vector by a random orthogonal matrix,
|
||||
each coordinate approximately follows N(0, 1/d) for d >= 64.
|
||||
We solve the Lloyd-Max conditions to find optimal centroids.
|
||||
|
||||
Based on: turboquant-pytorch/lloyd_max.py (Zandieh et al.)
|
||||
"""
|
||||
|
||||
import math
|
||||
from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _gaussian_pdf(x: float, sigma2: float) -> float:
|
||||
return (1.0 / math.sqrt(2 * math.pi * sigma2)) * math.exp(-x * x / (2 * sigma2))
|
||||
|
||||
|
||||
def _trapz(f, a: float, b: float, n: int = 200) -> float:
|
||||
"""Trapezoidal numerical integration (replaces scipy.integrate.quad)."""
|
||||
h = (b - a) / n
|
||||
result = 0.5 * (f(a) + f(b))
|
||||
for i in range(1, n):
|
||||
result += f(a + i * h)
|
||||
return result * h
|
||||
|
||||
|
||||
def solve_lloyd_max(
|
||||
d: int,
|
||||
bits: int,
|
||||
max_iter: int = 200,
|
||||
tol: float = 1e-10,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Solve Lloyd-Max optimal quantizer for N(0, 1/d) distribution.
|
||||
|
||||
Args:
|
||||
d: Vector dimension (determines variance = 1/d).
|
||||
bits: Number of quantization bits.
|
||||
max_iter: Maximum Lloyd-Max iterations.
|
||||
tol: Convergence tolerance.
|
||||
|
||||
Returns:
|
||||
centroids: Sorted tensor of 2^bits optimal centroids.
|
||||
boundaries: Sorted tensor of 2^bits - 1 decision boundaries.
|
||||
|
||||
"""
|
||||
n_levels = 2**bits
|
||||
sigma2 = 1.0 / d
|
||||
sigma = math.sqrt(sigma2)
|
||||
|
||||
def pdf(x):
|
||||
return _gaussian_pdf(x, sigma2)
|
||||
|
||||
lo, hi = -3.5 * sigma, 3.5 * sigma
|
||||
centroids = [lo + (hi - lo) * (i + 0.5) / n_levels for i in range(n_levels)]
|
||||
|
||||
for _ in range(max_iter):
|
||||
boundaries = [
|
||||
(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)
|
||||
]
|
||||
edges = [lo * 3] + boundaries + [hi * 3]
|
||||
new_centroids = []
|
||||
for i in range(n_levels):
|
||||
a, b = edges[i], edges[i + 1]
|
||||
num = _trapz(lambda x: x * pdf(x), a, b)
|
||||
den = _trapz(pdf, a, b)
|
||||
new_centroids.append(num / den if den > 1e-15 else centroids[i])
|
||||
|
||||
if max(abs(new_centroids[i] - centroids[i]) for i in range(n_levels)) < tol:
|
||||
break
|
||||
centroids = new_centroids
|
||||
|
||||
boundaries = [(centroids[i] + centroids[i + 1]) / 2.0 for i in range(n_levels - 1)]
|
||||
return (
|
||||
torch.tensor(centroids, dtype=torch.float32),
|
||||
torch.tensor(boundaries, dtype=torch.float32),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def get_centroids(d: int, bits: int) -> torch.Tensor:
|
||||
"""Get precomputed Lloyd-Max centroids (cached)."""
|
||||
centroids, _ = solve_lloyd_max(d, bits)
|
||||
return centroids
|
||||
@@ -1,261 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TurboQuant configuration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Named TQ presets: each maps to frozen config parameters.
|
||||
# key_quant_bits: 8 = FP8 keys, 3-4 = MSE (Lloyd-Max) quantized keys.
|
||||
# value_quant_bits: 3-4 = uniform quantized values.
|
||||
TQ_PRESETS: dict[str, dict] = {
|
||||
"turboquant_k8v4": {
|
||||
"key_quant_bits": 8,
|
||||
"value_quant_bits": 4,
|
||||
"norm_correction": False,
|
||||
},
|
||||
"turboquant_4bit_nc": {
|
||||
"key_quant_bits": 4,
|
||||
"value_quant_bits": 4,
|
||||
"norm_correction": True,
|
||||
},
|
||||
"turboquant_k3v4_nc": {
|
||||
"key_quant_bits": 3,
|
||||
"value_quant_bits": 4,
|
||||
"norm_correction": True,
|
||||
},
|
||||
"turboquant_3bit_nc": {
|
||||
"key_quant_bits": 3,
|
||||
"value_quant_bits": 3,
|
||||
"norm_correction": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurboQuantConfig:
|
||||
"""Configuration for TurboQuant KV-cache quantization.
|
||||
|
||||
Applies Hadamard rotation followed by per-coordinate Lloyd-Max scalar
|
||||
quantization for keys, and uniform quantization for values.
|
||||
|
||||
Historical note: the core algorithmic pattern implemented for key
|
||||
quantization (Hadamard rotation followed by deterministic scalar
|
||||
quantization and re-normalization) was originally established in DRIVE
|
||||
(Vargaftik et al., NeurIPS 2021) and EDEN (Vargaftik et al., ICML
|
||||
2022). This formulation is also mathematically equivalent to the
|
||||
scalar case of the HIGGS quantization method (Malinovskii et al.,
|
||||
"Pushing the Limits of Large Language Model Quantization via the
|
||||
Linearity Theorem", NAACL 2025; preprint arXiv:2411.17525), which
|
||||
subsequently generalized these concepts.
|
||||
|
||||
A first application of this approach to KV-cache compression is in
|
||||
"Cache Me If You Must: Adaptive Key-Value Quantization for Large
|
||||
Language Models" (Shutova et al., ICML 2025; preprint
|
||||
arXiv:2501.19392). All of these foundational and application
|
||||
references pre-date the TurboQuant paper (Zandieh et al., ICLR 2026).
|
||||
|
||||
QJL is intentionally omitted: community consensus (5+ independent
|
||||
groups) found it hurts attention quality by amplifying variance
|
||||
through softmax.
|
||||
|
||||
Named presets (use via --kv-cache-dtype):
|
||||
turboquant_k8v4: FP8 keys + 4-bit values, 2.6x, +1.17% PPL
|
||||
turboquant_4bit_nc: 4-bit MSE keys + 4-bit values + NC, 3.8x, +2.71%
|
||||
turboquant_k3v4_nc: 3-bit MSE keys + 4-bit values + NC, ~3.5x, +10.63%
|
||||
turboquant_3bit_nc: 3-bit MSE keys + 3-bit values + NC, 4.9x, +20.59%
|
||||
|
||||
Args:
|
||||
head_dim: Attention head dimension (e.g. 64, 96, 128).
|
||||
key_quant_bits: Bits for key quantization. 8 = FP8 keys (no
|
||||
rotation/MSE). 3-4 = Lloyd-Max MSE quantized keys.
|
||||
value_quant_bits: Bits per value dimension for uniform quantization.
|
||||
3 = 8 levels, 4 = 16 levels (default).
|
||||
norm_correction: Re-normalize centroid vectors to unit norm before
|
||||
inverse rotation during dequant. Fixes quantization-induced norm
|
||||
distortion, improving PPL by ~0.8% at 4-bit.
|
||||
|
||||
"""
|
||||
|
||||
head_dim: int = 128
|
||||
key_quant_bits: int = 3 # 3-4 = MSE keys, 8 = FP8 keys
|
||||
value_quant_bits: int = 4 # 3-4 = uniform quantized values
|
||||
seed: int = 42 # kept for backward compatibility; no longer used internally
|
||||
norm_correction: bool = False
|
||||
|
||||
@property
|
||||
def key_fp8(self) -> bool:
|
||||
"""Whether keys are stored as FP8 — no rotation/quantization needed."""
|
||||
return self.key_quant_bits == 8
|
||||
|
||||
@property
|
||||
def mse_bits(self) -> int:
|
||||
"""MSE quantizer bit-width (determines centroid count: 2^mse_bits).
|
||||
|
||||
For MSE key modes, equals key_quant_bits.
|
||||
For FP8 key mode, falls back to value_quant_bits (centroids are still
|
||||
needed for continuation-prefill dequant and decode kernel params).
|
||||
"""
|
||||
if self.key_fp8:
|
||||
return self.value_quant_bits
|
||||
return self.key_quant_bits
|
||||
|
||||
@property
|
||||
def key_mse_bits(self) -> int:
|
||||
"""MSE bits actually used for key quantization (0 if FP8 keys)."""
|
||||
if self.key_fp8:
|
||||
return 0
|
||||
return self.key_quant_bits
|
||||
|
||||
@property
|
||||
def centroid_bits(self) -> int:
|
||||
"""Bits for centroid generation — always non-zero."""
|
||||
return self.mse_bits
|
||||
|
||||
@property
|
||||
def n_centroids(self) -> int:
|
||||
return 2**self.mse_bits
|
||||
|
||||
@property
|
||||
def key_packed_size(self) -> int:
|
||||
"""Packed bytes for a single KEY vector.
|
||||
|
||||
FP8 mode (key_quant_bits=8):
|
||||
head_dim bytes (1 byte per element, no overhead).
|
||||
|
||||
TQ mode:
|
||||
- MSE indices: ceil(head_dim * key_mse_bits / 8) bytes
|
||||
- vec_norm: 2 bytes (float16)
|
||||
"""
|
||||
if self.key_fp8:
|
||||
return self.head_dim # 1 byte per element
|
||||
mse_bytes = math.ceil(self.head_dim * self.key_mse_bits / 8)
|
||||
norm_bytes = 2 # vec_norm fp16
|
||||
return mse_bytes + norm_bytes
|
||||
|
||||
@property
|
||||
def effective_value_quant_bits(self) -> int:
|
||||
"""Actual bits used for value storage."""
|
||||
return self.value_quant_bits
|
||||
|
||||
@property
|
||||
def value_packed_size(self) -> int:
|
||||
"""Packed bytes for a single VALUE vector.
|
||||
|
||||
Uniform quantization: ceil(head_dim * bits / 8) + 4 bytes (scale + zero fp16).
|
||||
"""
|
||||
data_bytes = math.ceil(self.head_dim * self.value_quant_bits / 8)
|
||||
return data_bytes + 4 # +2 scale(fp16) +2 zero(fp16)
|
||||
|
||||
@property
|
||||
def slot_size(self) -> int:
|
||||
"""Total packed bytes per head per position (key + value combined).
|
||||
|
||||
Layout: [key_packed | value_packed]
|
||||
"""
|
||||
return self.key_packed_size + self.value_packed_size
|
||||
|
||||
@property
|
||||
def slot_size_aligned(self) -> int:
|
||||
"""Slot size rounded up to next even number.
|
||||
|
||||
Even-number is required so effective_head_size = slot_size_aligned // 2
|
||||
is integral.
|
||||
"""
|
||||
s = self.slot_size
|
||||
return s + (s % 2) # round up to even
|
||||
|
||||
@staticmethod
|
||||
def get_boundary_skip_layers(
|
||||
model_config: ModelConfig,
|
||||
n: int = 2,
|
||||
) -> list[str]:
|
||||
"""Layer indices to skip TQ compression (boundary protection).
|
||||
|
||||
For hybrid models (attention + Mamba/linear-attention), boundary
|
||||
protection is disabled — hybrids typically have only 8-12
|
||||
full-attention layers and a hard n=2 on each side would cover
|
||||
~40 % of them. The dense GSM8K baselines that motivate n=2
|
||||
don't apply to hybrids.
|
||||
|
||||
For dense models, skips first N and last N attention layers.
|
||||
Empirically required for aggressive presets (k3v4_nc, 3bit_nc)
|
||||
— without it GSM8K drops ~30 points on Qwen3-4B.
|
||||
"""
|
||||
if model_config.is_hybrid:
|
||||
attn_indices = _get_full_attention_layer_indices(model_config)
|
||||
if not attn_indices:
|
||||
raise NotImplementedError(
|
||||
"TurboQuant KV cache requires identifiable "
|
||||
"full-attention layers, but none were found in "
|
||||
"the hybrid model config."
|
||||
)
|
||||
logger.info("TQ hybrid: full-attention layers %s", attn_indices)
|
||||
return []
|
||||
|
||||
num_layers = model_config.hf_text_config.num_hidden_layers
|
||||
if n <= 0 or num_layers <= 0:
|
||||
return []
|
||||
n = min(n, num_layers // 2) # don't skip more than half
|
||||
first = list(range(n))
|
||||
last = list(range(num_layers - n, num_layers))
|
||||
# Deduplicate (if num_layers <= 2*n)
|
||||
indices = sorted(set(first + last))
|
||||
return [str(i) for i in indices]
|
||||
|
||||
@staticmethod
|
||||
def from_cache_dtype(cache_dtype: str, head_dim: int) -> TurboQuantConfig:
|
||||
"""Create config from a named preset.
|
||||
|
||||
Valid presets: turboquant_k8v4, turboquant_4bit_nc, etc.
|
||||
"""
|
||||
if cache_dtype not in TQ_PRESETS:
|
||||
valid = ", ".join(TQ_PRESETS.keys())
|
||||
raise ValueError(
|
||||
f"Unknown TurboQuant cache dtype: {cache_dtype!r}. "
|
||||
f"Valid presets: {valid}"
|
||||
)
|
||||
preset = TQ_PRESETS[cache_dtype]
|
||||
return TurboQuantConfig(
|
||||
head_dim=head_dim,
|
||||
key_quant_bits=preset["key_quant_bits"],
|
||||
value_quant_bits=preset["value_quant_bits"],
|
||||
norm_correction=preset["norm_correction"],
|
||||
)
|
||||
|
||||
|
||||
def _get_full_attention_layer_indices(model_config: ModelConfig) -> list[int]:
|
||||
"""Global indices of full-attention layers in a hybrid model.
|
||||
|
||||
Covers the conventions used across vLLM: ``layer_types`` (Qwen3.5/Next),
|
||||
``layers_block_type`` (Jamba/Zamba2), ``attn_type_list`` (Minimax).
|
||||
"""
|
||||
text_cfg = model_config.hf_text_config
|
||||
hf_cfg = model_config.hf_config
|
||||
|
||||
layer_types = getattr(text_cfg, "layer_types", None)
|
||||
if layer_types is not None:
|
||||
return [
|
||||
i for i, t in enumerate(layer_types) if t in ("full_attention", "attention")
|
||||
]
|
||||
|
||||
layers_block_type = getattr(text_cfg, "layers_block_type", None)
|
||||
if layers_block_type is not None:
|
||||
return [
|
||||
i for i, t in enumerate(layers_block_type) if t in ("attention", "hybrid")
|
||||
]
|
||||
|
||||
attn_type_list = getattr(hf_cfg, "attn_type_list", None)
|
||||
if attn_type_list is not None:
|
||||
return [i for i, t in enumerate(attn_type_list) if t == 1]
|
||||
|
||||
return []
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions used for tests and benchmarks."""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
|
||||
from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales, marlin_zero_points
|
||||
from .quant_utils import (
|
||||
get_pack_factor,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
)
|
||||
|
||||
|
||||
class MarlinWorkspace:
|
||||
def __init__(self, out_features, min_thread_n, max_parallel):
|
||||
assert out_features % min_thread_n == 0, (
|
||||
"out_features = {} is indivisible by min_thread_n = {}".format(
|
||||
out_features, min_thread_n
|
||||
)
|
||||
)
|
||||
|
||||
max_workspace_size = (out_features // min_thread_n) * max_parallel
|
||||
|
||||
self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda")
|
||||
|
||||
|
||||
def marlin_permute_weights(
|
||||
q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE, is_a_8bit=False
|
||||
):
|
||||
assert q_w.shape == (size_k, size_n)
|
||||
assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}"
|
||||
assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}"
|
||||
|
||||
if is_a_8bit:
|
||||
# Permute weights to 32x32 marlin tiles
|
||||
q_w = q_w.reshape((size_k // (tile * 2), tile * 2, size_n // tile, tile))
|
||||
else:
|
||||
# Permute weights to 16x64 marlin tiles
|
||||
q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile))
|
||||
q_w = q_w.permute((0, 2, 1, 3))
|
||||
q_w = q_w.reshape((size_k // tile, size_n * tile))
|
||||
|
||||
q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape)
|
||||
|
||||
return q_w
|
||||
|
||||
|
||||
def marlin_weights(q_w, size_k, size_n, num_bits, perm, is_a_8bit=False):
|
||||
# Permute
|
||||
q_w = marlin_permute_weights(q_w, size_k, size_n, perm, is_a_8bit=is_a_8bit)
|
||||
|
||||
# Pack
|
||||
pack_factor = get_pack_factor(num_bits)
|
||||
orig_device = q_w.device
|
||||
|
||||
q_w = q_w.cpu().numpy().astype(np.uint32)
|
||||
|
||||
q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32)
|
||||
for i in range(pack_factor):
|
||||
q_packed |= q_w[:, i::pack_factor] << num_bits * i
|
||||
|
||||
q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device)
|
||||
|
||||
return q_packed
|
||||
|
||||
|
||||
def get_weight_perm(num_bits: int, is_a_8bit: bool = False):
|
||||
perm_list: list[int] = []
|
||||
if is_a_8bit:
|
||||
for i in range(32):
|
||||
perm1 = []
|
||||
col = i // 4
|
||||
for block in [0, 1]:
|
||||
for row in [
|
||||
4 * (i % 4),
|
||||
4 * (i % 4) + 1,
|
||||
4 * (i % 4) + 2,
|
||||
4 * (i % 4) + 3,
|
||||
4 * (i % 4 + 4),
|
||||
4 * (i % 4 + 4) + 1,
|
||||
4 * (i % 4 + 4) + 2,
|
||||
4 * (i % 4 + 4) + 3,
|
||||
]:
|
||||
perm1.append(16 * row + col + 8 * block)
|
||||
for j in range(2):
|
||||
perm_list.extend([p + 512 * j for p in perm1])
|
||||
else:
|
||||
for i in range(32):
|
||||
perm1 = []
|
||||
col = i // 4
|
||||
for block in [0, 1]:
|
||||
for row in [
|
||||
2 * (i % 4),
|
||||
2 * (i % 4) + 1,
|
||||
2 * (i % 4 + 4),
|
||||
2 * (i % 4 + 4) + 1,
|
||||
]:
|
||||
perm1.append(16 * row + col + 8 * block)
|
||||
for j in range(4):
|
||||
perm_list.extend([p + 256 * j for p in perm1])
|
||||
|
||||
perm = np.array(perm_list)
|
||||
|
||||
if num_bits == 4:
|
||||
if is_a_8bit: # noqa: SIM108
|
||||
interleave = np.array([0, 4, 1, 5, 2, 6, 3, 7])
|
||||
else:
|
||||
interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
|
||||
elif num_bits == 8:
|
||||
if is_a_8bit: # noqa: SIM108
|
||||
interleave = np.array([0, 1, 2, 3])
|
||||
else:
|
||||
interleave = np.array([0, 2, 1, 3])
|
||||
else:
|
||||
raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
|
||||
|
||||
perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel()
|
||||
perm = torch.from_numpy(perm)
|
||||
return perm
|
||||
|
||||
|
||||
def marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
size_k, size_n = w.shape
|
||||
num_bits = quant_type.size_bits
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(num_bits, is_a_8bit)
|
||||
marlin_q_w = marlin_weights(
|
||||
q_w, size_k, size_n, num_bits, weight_perm, is_a_8bit=is_a_8bit
|
||||
)
|
||||
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
|
||||
|
||||
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4b8:
|
||||
ops.marlin_int4_fp8_preprocess(marlin_q_w, inplace=True)
|
||||
marlin_s = marlin_s * 512
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s]
|
||||
for i in range(len(res_list)):
|
||||
res_list[i] = res_list[i].to(w.device)
|
||||
|
||||
return res_list
|
||||
|
||||
|
||||
def awq_marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
size_k, size_n = w.shape
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
# Detect num groups
|
||||
assert size_k % group_size == 0
|
||||
num_groups = size_k // group_size
|
||||
|
||||
# Quantize with zp
|
||||
w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True)
|
||||
|
||||
if input_dtype == torch.float8_e4m3fn and quant_type == scalar_types.uint4:
|
||||
repeated_zp = zp.repeat_interleave(group_size, 0)
|
||||
q_w_old = q_w
|
||||
q_w = q_w_old - repeated_zp
|
||||
q_w[q_w < 0] = 15 - q_w_old[q_w < 0]
|
||||
s = s * 512
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w = marlin_weights(
|
||||
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit=is_a_8bit
|
||||
)
|
||||
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size, is_a_8bit=is_a_8bit)
|
||||
marlin_zp = marlin_zero_points(
|
||||
zp, num_groups, size_n, quant_type.size_bits, is_a_8bit=is_a_8bit
|
||||
)
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
|
||||
for i in range(len(res_list)):
|
||||
res_list[i] = res_list[i].to(w.device)
|
||||
|
||||
return res_list
|
||||
Reference in New Issue
Block a user