[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
parent
e1726d9142
commit
c9bfa574ce
@@ -11,6 +11,7 @@ from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
__all__ = [
|
||||
"break_fp4_bytes",
|
||||
"dequantize_nvfp4_moe_weights",
|
||||
"dequantize_to_dtype",
|
||||
"nvfp4_gathered_bias",
|
||||
"ref_nvfp4_quant",
|
||||
@@ -629,6 +630,58 @@ def dequantize_to_dtype(
|
||||
return out.to(dtype)
|
||||
|
||||
|
||||
def dequantize_nvfp4_moe_weights(
|
||||
w13_weight_packed: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w13_weight_global_scale: torch.Tensor,
|
||||
w2_weight_packed: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
w2_weight_global_scale: torch.Tensor,
|
||||
dtype: torch.dtype,
|
||||
group_size: int = 16,
|
||||
*,
|
||||
invert_global_scales: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Materialize serialized NVFP4 MoE weights for requantization.
|
||||
|
||||
Compressed-tensors stores global encode scales, while ModelOpt stores
|
||||
``weight_scale_2`` as the corresponding decode multiplier. Set
|
||||
``invert_global_scales`` according to the source checkpoint convention.
|
||||
"""
|
||||
if invert_global_scales:
|
||||
w13_weight_global_scale = 1.0 / w13_weight_global_scale
|
||||
w2_weight_global_scale = 1.0 / w2_weight_global_scale
|
||||
|
||||
w13_num_shards = w13_weight_global_scale.shape[1]
|
||||
w13_weights = torch.chunk(w13_weight_packed, w13_num_shards, dim=1)
|
||||
w13_scales = torch.chunk(w13_weight_scale, w13_num_shards, dim=1)
|
||||
w13 = torch.cat(
|
||||
[
|
||||
dequantize_to_dtype(
|
||||
weight,
|
||||
scale,
|
||||
w13_weight_global_scale[:, shard_idx].contiguous(),
|
||||
dtype,
|
||||
block_size=group_size,
|
||||
swizzle=False,
|
||||
)
|
||||
for shard_idx, (weight, scale) in enumerate(
|
||||
zip(w13_weights, w13_scales, strict=True)
|
||||
)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
w2 = dequantize_to_dtype(
|
||||
w2_weight_packed,
|
||||
w2_weight_scale,
|
||||
w2_weight_global_scale,
|
||||
dtype,
|
||||
block_size=group_size,
|
||||
swizzle=False,
|
||||
)
|
||||
return w13, w2
|
||||
|
||||
|
||||
def get_reciprocal(x):
|
||||
if isinstance(x, torch.Tensor):
|
||||
# torch.where yields operation not permitted when stream is capturing.
|
||||
|
||||
Reference in New Issue
Block a user