Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .layer_utils import replace_parameter, update_tensor_inplace
|
||||
|
||||
__all__ = ['update_tensor_inplace', 'replace_parameter']
|
||||
from .layer_utils import replace_parameter, update_tensor_inplace
|
||||
|
||||
__all__ = ["update_tensor_inplace", "replace_parameter"]
|
||||
|
||||
@@ -12,41 +12,56 @@ ALLSPARK_AMPERE_N_ALIGN = 16
|
||||
ALLSPARK_AMPERE_K_ALIGN = 16
|
||||
|
||||
|
||||
def check_allspark_supported_dtype_shape(input_size_per_partition: int,
|
||||
output_size_per_partition: int,
|
||||
group_size: int,
|
||||
weight_dtype: ScalarType,
|
||||
act_dtype: torch.dtype):
|
||||
def check_allspark_supported_dtype_shape(
|
||||
input_size_per_partition: int,
|
||||
output_size_per_partition: int,
|
||||
group_size: int,
|
||||
weight_dtype: ScalarType,
|
||||
act_dtype: torch.dtype,
|
||||
):
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (-1 if capability_tuple is None else
|
||||
capability_tuple.to_int())
|
||||
device_capability = -1 if capability_tuple is None else capability_tuple.to_int()
|
||||
|
||||
# For Ampere GPU
|
||||
if device_capability >= 80 and device_capability < 90:
|
||||
if group_size != -1:
|
||||
return False, \
|
||||
"For Ampere GPU, AllSpark does not support group_size "\
|
||||
f"= {group_size}. Only group_size = -1 are supported."
|
||||
return (
|
||||
False,
|
||||
"For Ampere GPU, AllSpark does not support group_size "
|
||||
f"= {group_size}. Only group_size = -1 are supported.",
|
||||
)
|
||||
|
||||
if weight_dtype not in ALLSPARK_SUPPORTED_QUANT_TYPES:
|
||||
return False, "For Ampere GPU, AllSpark does not support "\
|
||||
f"quant type ({weight_dtype}). Only quant type "\
|
||||
f"({ALLSPARK_SUPPORTED_QUANT_TYPES}) are supported."
|
||||
return (
|
||||
False,
|
||||
"For Ampere GPU, AllSpark does not support "
|
||||
f"quant type ({weight_dtype}). Only quant type "
|
||||
f"({ALLSPARK_SUPPORTED_QUANT_TYPES}) are supported.",
|
||||
)
|
||||
|
||||
if input_size_per_partition % ALLSPARK_AMPERE_K_ALIGN != 0 \
|
||||
or output_size_per_partition % ALLSPARK_AMPERE_N_ALIGN != 0:
|
||||
return False, \
|
||||
"AllSpark needs input_size_per_partition % "\
|
||||
f"{ALLSPARK_AMPERE_K_ALIGN} = 0 and "\
|
||||
f"output_size_per_partition % {ALLSPARK_AMPERE_N_ALIGN} = 0 "\
|
||||
"for Ampere GPU optimized kernels."
|
||||
if (
|
||||
input_size_per_partition % ALLSPARK_AMPERE_K_ALIGN != 0
|
||||
or output_size_per_partition % ALLSPARK_AMPERE_N_ALIGN != 0
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"AllSpark needs input_size_per_partition % "
|
||||
f"{ALLSPARK_AMPERE_K_ALIGN} = 0 and "
|
||||
f"output_size_per_partition % {ALLSPARK_AMPERE_N_ALIGN} = 0 "
|
||||
"for Ampere GPU optimized kernels.",
|
||||
)
|
||||
|
||||
if act_dtype != torch.float16 and act_dtype != torch.bfloat16:
|
||||
return False, \
|
||||
"AllSpark only supports act_dtype = float16 or bfloat16,"\
|
||||
f"for Ampere GPU, but got act_dtype = {act_dtype}."
|
||||
return (
|
||||
False,
|
||||
"AllSpark only supports act_dtype = float16 or bfloat16,"
|
||||
f"for Ampere GPU, but got act_dtype = {act_dtype}.",
|
||||
)
|
||||
else:
|
||||
return False, "AllSpark currently does not support "\
|
||||
f"device_capability = {device_capability}."
|
||||
return (
|
||||
False,
|
||||
"AllSpark currently does not support "
|
||||
f"device_capability = {device_capability}.",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
@@ -28,13 +28,14 @@ BITBLAS_SUPPORTED_SYM = [False, True]
|
||||
|
||||
# Determines the supported quantization types for BitBLAS based on the
|
||||
# device's capability and whether zero-point (zp) is used.
|
||||
def query_bitblas_supported_quant_types(has_zp: bool,
|
||||
device_capability: Optional[int] = None
|
||||
):
|
||||
def query_bitblas_supported_quant_types(
|
||||
has_zp: bool, device_capability: Optional[int] = None
|
||||
):
|
||||
if device_capability is None:
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (-1 if capability_tuple is None else
|
||||
capability_tuple.to_int())
|
||||
device_capability = (
|
||||
-1 if capability_tuple is None else capability_tuple.to_int()
|
||||
)
|
||||
|
||||
if device_capability < 70:
|
||||
return []
|
||||
@@ -50,97 +51,116 @@ def query_bitblas_supported_quant_types(has_zp: bool,
|
||||
|
||||
|
||||
def _check_bitblas_supported(
|
||||
quant_type: ScalarType,
|
||||
group_size: Optional[int],
|
||||
has_zp: bool,
|
||||
device_capability: Optional[int] = None) -> tuple[bool, Optional[str]]:
|
||||
|
||||
quant_type: ScalarType,
|
||||
group_size: Optional[int],
|
||||
has_zp: bool,
|
||||
device_capability: Optional[int] = None,
|
||||
) -> tuple[bool, Optional[str]]:
|
||||
if device_capability is None:
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (-1 if capability_tuple is None else
|
||||
capability_tuple.to_int())
|
||||
device_capability = (
|
||||
-1 if capability_tuple is None else capability_tuple.to_int()
|
||||
)
|
||||
|
||||
supported_types = query_bitblas_supported_quant_types(
|
||||
has_zp, device_capability)
|
||||
supported_types = query_bitblas_supported_quant_types(has_zp, device_capability)
|
||||
|
||||
if quant_type not in supported_types:
|
||||
return (False, f"BitBLAS does not support weight_bits = {quant_type}. "
|
||||
f"Only types = {supported_types} "
|
||||
f"are supported (for group_size = {group_size}, "
|
||||
f"device_capability = {device_capability}, zp = {has_zp}).")
|
||||
if (group_size is None or group_size not in BITBLAS_SUPPORTED_GROUP_SIZES):
|
||||
return (False, f"BitBLAS does not support group_size = {group_size}. "
|
||||
f"Only group_sizes = {BITBLAS_SUPPORTED_GROUP_SIZES} "
|
||||
"are supported.")
|
||||
return (
|
||||
False,
|
||||
f"BitBLAS does not support weight_bits = {quant_type}. "
|
||||
f"Only types = {supported_types} "
|
||||
f"are supported (for group_size = {group_size}, "
|
||||
f"device_capability = {device_capability}, zp = {has_zp}).",
|
||||
)
|
||||
if group_size is None or group_size not in BITBLAS_SUPPORTED_GROUP_SIZES:
|
||||
return (
|
||||
False,
|
||||
f"BitBLAS does not support group_size = {group_size}. "
|
||||
f"Only group_sizes = {BITBLAS_SUPPORTED_GROUP_SIZES} "
|
||||
"are supported.",
|
||||
)
|
||||
|
||||
# Finally, check if bitblas is installed
|
||||
try:
|
||||
import bitblas
|
||||
if version.parse(
|
||||
bitblas.__version__) < version.parse(MINIMUM_BITBLAS_VERSION):
|
||||
raise ImportError("bitblas version is wrong. Please "
|
||||
f"install bitblas>={MINIMUM_BITBLAS_VERSION}")
|
||||
|
||||
if version.parse(bitblas.__version__) < version.parse(MINIMUM_BITBLAS_VERSION):
|
||||
raise ImportError(
|
||||
"bitblas version is wrong. Please "
|
||||
f"install bitblas>={MINIMUM_BITBLAS_VERSION}"
|
||||
)
|
||||
except ImportError:
|
||||
return False, "BitBLAS is not installed."
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def check_bitblas_supported(quant_type: ScalarType,
|
||||
group_size: int,
|
||||
has_zp: bool = False,
|
||||
device_capability: Optional[int] = None) -> bool:
|
||||
cond, _ = _check_bitblas_supported(quant_type, group_size, has_zp,
|
||||
device_capability)
|
||||
def check_bitblas_supported(
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
has_zp: bool = False,
|
||||
device_capability: Optional[int] = None,
|
||||
) -> bool:
|
||||
cond, _ = _check_bitblas_supported(
|
||||
quant_type, group_size, has_zp, device_capability
|
||||
)
|
||||
return cond
|
||||
|
||||
|
||||
def verify_bitblas_supported(quant_type: ScalarType,
|
||||
group_size: int,
|
||||
has_zp: bool = False) -> None:
|
||||
def verify_bitblas_supported(
|
||||
quant_type: ScalarType, group_size: int, has_zp: bool = False
|
||||
) -> None:
|
||||
cond, err_msg = _check_bitblas_supported(quant_type, group_size, has_zp)
|
||||
if not cond:
|
||||
assert err_msg is not None
|
||||
raise ValueError(err_msg)
|
||||
|
||||
|
||||
def verify_bitblas_supports_shape(output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int, group_size: int) -> None:
|
||||
|
||||
def verify_bitblas_supports_shape(
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int,
|
||||
group_size: int,
|
||||
) -> None:
|
||||
# Validate output_size_per_partition
|
||||
if output_size_per_partition % BITBLAS_MIN_WEIGHT_SIZE_N != 0:
|
||||
raise ValueError(f"Weight output_size_per_partition = "
|
||||
f"{output_size_per_partition} is not divisible by "
|
||||
f" min_thread_n = {BITBLAS_MIN_WEIGHT_SIZE_N}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq.")
|
||||
raise ValueError(
|
||||
f"Weight output_size_per_partition = "
|
||||
f"{output_size_per_partition} is not divisible by "
|
||||
f" min_thread_n = {BITBLAS_MIN_WEIGHT_SIZE_N}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
# Validate input_size_per_partition
|
||||
if input_size_per_partition % BITBLAS_MIN_WEIGHT_SIZE_K != 0:
|
||||
raise ValueError(f"Weight input_size_per_partition = "
|
||||
f"{input_size_per_partition} is not divisible "
|
||||
f"by min_thread_k = {BITBLAS_MIN_WEIGHT_SIZE_K}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq.")
|
||||
raise ValueError(
|
||||
f"Weight input_size_per_partition = "
|
||||
f"{input_size_per_partition} is not divisible "
|
||||
f"by min_thread_k = {BITBLAS_MIN_WEIGHT_SIZE_K}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
if (group_size < input_size
|
||||
and input_size_per_partition % group_size != 0):
|
||||
if group_size < input_size and input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
f"Weight input_size_per_partition = {input_size_per_partition}"
|
||||
f" is not divisible by group_size = {group_size}."
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq.")
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
|
||||
def check_bitblas_supports_shape(output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int, group_size: int) \
|
||||
-> tuple[bool, Optional[str]]:
|
||||
def check_bitblas_supports_shape(
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int,
|
||||
group_size: int,
|
||||
) -> tuple[bool, Optional[str]]:
|
||||
try:
|
||||
verify_bitblas_supports_shape(output_size_per_partition,
|
||||
input_size_per_partition, input_size,
|
||||
group_size)
|
||||
verify_bitblas_supports_shape(
|
||||
output_size_per_partition, input_size_per_partition, input_size, group_size
|
||||
)
|
||||
except ValueError as e:
|
||||
return False, e.__str__()
|
||||
return True, None
|
||||
@@ -150,8 +170,9 @@ def bitblas_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
|
||||
return (not act_order) or (act_order and not is_row_parallel)
|
||||
|
||||
|
||||
def bitblas_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
|
||||
is_row_parallel: bool) -> bool:
|
||||
def bitblas_repeat_scales_on_all_ranks(
|
||||
act_order: bool, group_size: int, is_row_parallel: bool
|
||||
) -> bool:
|
||||
# Need to repeat scales on every rank if act_ordering or
|
||||
# channelwise and RowParallelLinear
|
||||
is_channelwise = group_size == -1
|
||||
@@ -159,17 +180,18 @@ def bitblas_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
|
||||
|
||||
|
||||
def bitblas_make_empty_g_idx(device: torch.device) -> torch.Tensor:
|
||||
return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
|
||||
requires_grad=False)
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def bitblas_make_empty_zp(device: torch.device) -> torch.Tensor:
|
||||
return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
|
||||
requires_grad=False)
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def bitblas_sort_g_idx(
|
||||
g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
def bitblas_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
|
||||
return g_idx[g_idx_sort_indices], g_idx_sort_indices
|
||||
|
||||
@@ -186,8 +208,7 @@ def unpack_gptq_qzeros(qzeros, bits, is_gptq_v2=False) -> torch.Tensor:
|
||||
|
||||
for col in range(unpacked_zeros.shape[1]):
|
||||
i = col % elems_per_int32
|
||||
unpacked_zeros[:, col] = (qzeros[:, col // elems_per_int32] >>
|
||||
(bits * i)) & 0xF
|
||||
unpacked_zeros[:, col] = (qzeros[:, col // elems_per_int32] >> (bits * i)) & 0xF
|
||||
if not is_gptq_v2:
|
||||
return unpacked_zeros + 1
|
||||
return unpacked_zeros
|
||||
@@ -204,7 +225,6 @@ def unpack_gptq_qweight(qweight, bits):
|
||||
)
|
||||
for col in range(unpacked_weight.shape[1]):
|
||||
i = col % elems_per_int8
|
||||
unpacked_weight[:, col] = (qweight[:, col // elems_per_int8] >>
|
||||
(bits * i))
|
||||
unpacked_weight[:, col] = qweight[:, col // elems_per_int8] >> (bits * i)
|
||||
|
||||
return torch.bitwise_and(unpacked_weight, 2**bits - 1)
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import (FusedMoEConfig,
|
||||
FusedMoEQuantConfig)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts)
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
|
||||
create_flashinfer_prepare_finalize)
|
||||
create_flashinfer_prepare_finalize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
|
||||
@@ -25,15 +30,17 @@ __all__ = [
|
||||
|
||||
def is_flashinfer_fp4_cutlass_moe_available() -> bool:
|
||||
"""Return ``True`` when FlashInfer CUTLASS NV-FP4 kernels can be used."""
|
||||
return (envs.VLLM_USE_FLASHINFER_MOE_FP4
|
||||
and has_flashinfer_cutlass_fused_moe()
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.is_device_capability(100))
|
||||
return (
|
||||
envs.VLLM_USE_FLASHINFER_MOE_FP4
|
||||
and has_flashinfer_cutlass_fused_moe()
|
||||
and current_platform.is_cuda()
|
||||
and current_platform.is_device_capability(100)
|
||||
)
|
||||
|
||||
|
||||
def reorder_w1w3_to_w3w1(weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
dim: int = -2) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
def reorder_w1w3_to_w3w1(
|
||||
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Re-order the concatenated `[w1, w3]` tensors to `[w3, w1]`"""
|
||||
size = weight.size(dim)
|
||||
assert size % 2 == 0, f"Expected even size in dim {dim}, got {size}"
|
||||
@@ -42,18 +49,21 @@ def reorder_w1w3_to_w3w1(weight: torch.Tensor,
|
||||
w1, w3 = weight.split(half, dim=dim)
|
||||
s1, s3 = scale.split(half, dim=dim)
|
||||
|
||||
return (torch.cat([w3, w1],
|
||||
dim=dim).contiguous(), torch.cat([s3, s1],
|
||||
dim=dim).contiguous())
|
||||
return (
|
||||
torch.cat([w3, w1], dim=dim).contiguous(),
|
||||
torch.cat([s3, s1], dim=dim).contiguous(),
|
||||
)
|
||||
|
||||
|
||||
def build_flashinfer_fp4_cutlass_moe_prepare_finalize(
|
||||
moe: FusedMoEConfig) -> mk.FusedMoEPrepareAndFinalize:
|
||||
moe: FusedMoEConfig,
|
||||
) -> mk.FusedMoEPrepareAndFinalize:
|
||||
"""Create a FlashInfer CUTLASS fused-MoE prepare finalize kernel"""
|
||||
use_dp = moe.moe_parallel_config.dp_size > 1
|
||||
enable_alltoallv = envs.VLLM_ALL2ALL_BACKEND == "flashinfer_all2allv"
|
||||
return create_flashinfer_prepare_finalize(
|
||||
use_dp=use_dp, use_nvfp4=True, enable_alltoallv=enable_alltoallv)
|
||||
use_dp=use_dp, use_nvfp4=True, enable_alltoallv=enable_alltoallv
|
||||
)
|
||||
|
||||
|
||||
def select_nvfp4_gemm_impl(
|
||||
@@ -76,4 +86,5 @@ def select_nvfp4_gemm_impl(
|
||||
# native cutlass experts currently don't support DP; TP case won't call this
|
||||
raise ValueError(
|
||||
"CutlassExpertsFp4 doesn't support DP. Use flashinfer CUTLASS "
|
||||
"Fused MoE backend instead (set VLLM_USE_FLASHINFER_MOE_FP4=1)")
|
||||
"Fused MoE backend instead (set VLLM_USE_FLASHINFER_MOE_FP4=1)"
|
||||
)
|
||||
|
||||
@@ -8,12 +8,16 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (FusedMoEConfig,
|
||||
FusedMoEQuantConfig)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts)
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_prepare_finalize import ( # noqa: E501
|
||||
create_flashinfer_prepare_finalize)
|
||||
create_flashinfer_prepare_finalize,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -24,7 +28,6 @@ class FlashinferMoeBackend(Enum):
|
||||
|
||||
|
||||
def calculate_tile_tokens_dim(num_tokens, top_k, num_experts):
|
||||
|
||||
# FlashInfer 0.2.10 has issues with larger tile sizes. Set to 8 for now.
|
||||
# TODO: Revert this to dynamic calculation once a new version of FlashInfer
|
||||
# with the necessary kernels is released.
|
||||
@@ -44,13 +47,16 @@ def calculate_tile_tokens_dim(num_tokens, top_k, num_experts):
|
||||
|
||||
|
||||
def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.reshape(-1, 2, x.shape[-2] // 2,
|
||||
x.shape[-1]).flip(dims=[1]).reshape(x.shape)
|
||||
return (
|
||||
x.reshape(-1, 2, x.shape[-2] // 2, x.shape[-1]).flip(dims=[1]).reshape(x.shape)
|
||||
)
|
||||
|
||||
|
||||
def rotate_flashinfer_fp8_moe_weights(gemm1_weights: torch.Tensor,
|
||||
gemm2_weights: torch.Tensor):
|
||||
def rotate_flashinfer_fp8_moe_weights(
|
||||
gemm1_weights: torch.Tensor, gemm2_weights: torch.Tensor
|
||||
):
|
||||
from flashinfer import reorder_rows_for_gated_act_gemm, shuffle_matrix_a
|
||||
|
||||
epilogue_tile_m = 128
|
||||
num_experts = gemm1_weights.shape[0]
|
||||
hidden_size = gemm1_weights.shape[-1]
|
||||
@@ -60,13 +66,13 @@ def rotate_flashinfer_fp8_moe_weights(gemm1_weights: torch.Tensor,
|
||||
gemm1_weights_fp8_interleaved = []
|
||||
for i in range(num_experts):
|
||||
gemm1_weights_fp8_interleaved.append(
|
||||
reorder_rows_for_gated_act_gemm(gemm1_weights[i]))
|
||||
reorder_rows_for_gated_act_gemm(gemm1_weights[i])
|
||||
)
|
||||
|
||||
# Stack weights and scales for all experts
|
||||
gemm1_weights_fp8_interleaved = torch.stack(
|
||||
gemm1_weights_fp8_interleaved).reshape(num_experts,
|
||||
2 * intermediate_size,
|
||||
hidden_size)
|
||||
gemm1_weights_fp8_interleaved = torch.stack(gemm1_weights_fp8_interleaved).reshape(
|
||||
num_experts, 2 * intermediate_size, hidden_size
|
||||
)
|
||||
|
||||
# Shuffle weights and scaling factors for transposed mma output
|
||||
gemm1_weights_fp8_shuffled = []
|
||||
@@ -74,18 +80,21 @@ def rotate_flashinfer_fp8_moe_weights(gemm1_weights: torch.Tensor,
|
||||
for i in range(num_experts):
|
||||
gemm1_weights_fp8_shuffled.append(
|
||||
shuffle_matrix_a(
|
||||
gemm1_weights_fp8_interleaved[i].view(torch.uint8),
|
||||
epilogue_tile_m))
|
||||
gemm1_weights_fp8_interleaved[i].view(torch.uint8), epilogue_tile_m
|
||||
)
|
||||
)
|
||||
|
||||
gemm2_weights_fp8_shuffled.append(
|
||||
shuffle_matrix_a(gemm2_weights[i].view(torch.uint8),
|
||||
epilogue_tile_m))
|
||||
shuffle_matrix_a(gemm2_weights[i].view(torch.uint8), epilogue_tile_m)
|
||||
)
|
||||
|
||||
# Stack weights for all experts
|
||||
gemm1_weights.data = torch.stack(gemm1_weights_fp8_shuffled).view(
|
||||
torch.float8_e4m3fn)
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
gemm2_weights.data = torch.stack(gemm2_weights_fp8_shuffled).view(
|
||||
torch.float8_e4m3fn)
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
|
||||
|
||||
def apply_flashinfer_per_tensor_scale_fp8(
|
||||
@@ -102,16 +111,22 @@ def apply_flashinfer_per_tensor_scale_fp8(
|
||||
from flashinfer.fused_moe import RoutingMethodType
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.flashinfer_trtllm_moe # noqa: E501, F401
|
||||
|
||||
assert layer.output1_scales_scalar is not None, (
|
||||
"Expected output1_scales_scalar to be initialized")
|
||||
"Expected output1_scales_scalar to be initialized"
|
||||
)
|
||||
assert layer.output1_scales_scalar is not None, (
|
||||
"Expected output1_scales_gate_scalar to be initialized")
|
||||
"Expected output1_scales_gate_scalar to be initialized"
|
||||
)
|
||||
assert layer.output1_scales_scalar is not None, (
|
||||
"Expected output2_scales_scalar to be initialized")
|
||||
"Expected output2_scales_scalar to be initialized"
|
||||
)
|
||||
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
assert layer.custom_routing_function == Llama4MoE.custom_routing_function, \
|
||||
|
||||
assert layer.custom_routing_function == Llama4MoE.custom_routing_function, (
|
||||
"FusedMoE flashinfer kernels are only supported for Llama4"
|
||||
)
|
||||
return torch.ops.vllm.flashinfer_fused_moe_per_tensor_scale_fp8(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=routing_bias,
|
||||
@@ -140,37 +155,39 @@ def get_moe_scaling_factors(
|
||||
activation_scale: torch.Tensor,
|
||||
gemm2_weights_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
output1_scales_scalar = gemm1_weights_scale * input_scale * (
|
||||
1.0 / activation_scale)
|
||||
output1_scales_scalar = gemm1_weights_scale * input_scale * (1.0 / activation_scale)
|
||||
output1_scales_gate_scalar = gemm1_weights_scale * input_scale
|
||||
output2_scales_scalar = activation_scale * gemm2_weights_scale
|
||||
|
||||
return output1_scales_scalar, output1_scales_gate_scalar, \
|
||||
output2_scales_scalar
|
||||
return output1_scales_scalar, output1_scales_gate_scalar, output2_scales_scalar
|
||||
|
||||
|
||||
def register_moe_scaling_factors(layer: torch.nn.Module) -> None:
|
||||
output1_scales, output1_gate_scales, output2_scales = \
|
||||
get_moe_scaling_factors(
|
||||
layer.w13_input_scale, layer.w13_weight_scale,
|
||||
layer.w2_input_scale, layer.w2_weight_scale
|
||||
)
|
||||
output1_scales, output1_gate_scales, output2_scales = get_moe_scaling_factors(
|
||||
layer.w13_input_scale,
|
||||
layer.w13_weight_scale,
|
||||
layer.w2_input_scale,
|
||||
layer.w2_weight_scale,
|
||||
)
|
||||
layer.register_parameter(
|
||||
'output1_scales_scalar',
|
||||
torch.nn.Parameter(output1_scales, requires_grad=False))
|
||||
"output1_scales_scalar", torch.nn.Parameter(output1_scales, requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
'output1_scales_gate_scalar',
|
||||
torch.nn.Parameter(output1_gate_scales, requires_grad=False))
|
||||
"output1_scales_gate_scalar",
|
||||
torch.nn.Parameter(output1_gate_scales, requires_grad=False),
|
||||
)
|
||||
layer.register_parameter(
|
||||
'output2_scales_scalar',
|
||||
torch.nn.Parameter(output2_scales, requires_grad=False))
|
||||
"output2_scales_scalar", torch.nn.Parameter(output2_scales, requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
'w2_input_scale_inv',
|
||||
torch.nn.Parameter(1.0 / layer.w2_input_scale, requires_grad=False))
|
||||
"w2_input_scale_inv",
|
||||
torch.nn.Parameter(1.0 / layer.w2_input_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
|
||||
def build_flashinfer_fp8_cutlass_moe_prepare_finalize(
|
||||
moe: Optional[FusedMoEConfig], ) -> mk.FusedMoEPrepareAndFinalize:
|
||||
moe: Optional[FusedMoEConfig],
|
||||
) -> mk.FusedMoEPrepareAndFinalize:
|
||||
"""Create a FlashInfer CUTLASS fused-MoE prepare finalize kernel"""
|
||||
use_dp = moe.moe_parallel_config.dp_size > 1 if moe is not None else False
|
||||
return create_flashinfer_prepare_finalize(use_dp)
|
||||
@@ -193,8 +210,7 @@ def select_cutlass_fp8_gemm_impl(
|
||||
tp_size=moe.moe_parallel_config.tp_size,
|
||||
)
|
||||
|
||||
assert out_dtype is not None, (
|
||||
"If moe config is None, out_dtype must be passed")
|
||||
assert out_dtype is not None, "If moe config is None, out_dtype must be passed"
|
||||
return FlashInferExperts(
|
||||
out_dtype=out_dtype,
|
||||
quant_config=quant_config,
|
||||
@@ -217,9 +233,10 @@ def flashinfer_cutlass_moe_fp8(
|
||||
|
||||
fused_experts = mk.FusedMoEModularKernel(
|
||||
build_flashinfer_fp8_cutlass_moe_prepare_finalize(moe=None),
|
||||
select_cutlass_fp8_gemm_impl(moe=None,
|
||||
quant_config=quant_config,
|
||||
out_dtype=hidden_states.dtype))
|
||||
select_cutlass_fp8_gemm_impl(
|
||||
moe=None, quant_config=quant_config, out_dtype=hidden_states.dtype
|
||||
),
|
||||
)
|
||||
|
||||
return fused_experts(
|
||||
hidden_states,
|
||||
@@ -245,4 +262,5 @@ def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
|
||||
allowed_backends = ["throughput", "latency"]
|
||||
raise ValueError(
|
||||
f"Unknown flashinfer moe backend: {flashinfer_moe_backend}"
|
||||
f" expected one of {allowed_backends}")
|
||||
f" expected one of {allowed_backends}"
|
||||
)
|
||||
|
||||
@@ -15,18 +15,26 @@ from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape, group_broadcast)
|
||||
GroupShape,
|
||||
group_broadcast,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
CUTLASS_BLOCK_FP8_SUPPORTED)
|
||||
from vllm.model_executor.parameter import (BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
PerTensorScaleParameter)
|
||||
CUTLASS_BLOCK_FP8_SUPPORTED,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils import direct_register_custom_op
|
||||
from vllm.utils.deep_gemm import (fp8_gemm_nt, is_deep_gemm_e8m0_used,
|
||||
is_deep_gemm_supported,
|
||||
should_use_deepgemm_for_fp8_linear)
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_gemm_nt,
|
||||
is_deep_gemm_e8m0_used,
|
||||
is_deep_gemm_supported,
|
||||
should_use_deepgemm_for_fp8_linear,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -56,7 +64,8 @@ def cutlass_scaled_mm(
|
||||
out_dtype=output_dtype,
|
||||
scale_a=As,
|
||||
# SM90 block FP8 requires row-major scale_b, which we do ahead of time
|
||||
scale_b=Bs if block_size is not None and is_hopper else Bs.T)
|
||||
scale_b=Bs if block_size is not None and is_hopper else Bs.T,
|
||||
)
|
||||
|
||||
|
||||
def rocm_aiter_gemm_w8a8_blockscale_impl(
|
||||
@@ -80,7 +89,6 @@ def rocm_aiter_gemm_w8a8_blockscale_fake(
|
||||
block_size: list[int],
|
||||
output_dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
|
||||
m = A.shape[0]
|
||||
n = B.shape[0]
|
||||
Y = torch.empty(m, n, dtype=output_dtype, device=A.device)
|
||||
@@ -93,9 +101,11 @@ if current_platform.is_rocm():
|
||||
op_func=rocm_aiter_gemm_w8a8_blockscale_impl,
|
||||
fake_impl=rocm_aiter_gemm_w8a8_blockscale_fake,
|
||||
)
|
||||
if (envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
and current_platform.is_fp8_fnuz()):
|
||||
|
||||
if (
|
||||
envs.VLLM_ROCM_USE_AITER
|
||||
and envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
and current_platform.is_fp8_fnuz()
|
||||
):
|
||||
import aiter as rocm_aiter
|
||||
from aiter import get_hip_quant
|
||||
|
||||
@@ -113,8 +123,9 @@ def _w8a8_triton_block_scaled_mm_func(
|
||||
block_size: list[int],
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return w8a8_triton_block_scaled_mm(qx, weight, x_scale, weight_scale,
|
||||
block_size, output_dtype)
|
||||
return w8a8_triton_block_scaled_mm(
|
||||
qx, weight, x_scale, weight_scale, block_size, output_dtype
|
||||
)
|
||||
|
||||
|
||||
def _w8a8_triton_block_scaled_mm_fake(
|
||||
@@ -125,9 +136,9 @@ def _w8a8_triton_block_scaled_mm_fake(
|
||||
block_size: list[int],
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((qx.size(0), weight.size(0)),
|
||||
dtype=output_dtype,
|
||||
device=qx.device)
|
||||
return torch.empty(
|
||||
(qx.size(0), weight.size(0)), dtype=output_dtype, device=qx.device
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
@@ -147,22 +158,24 @@ def _padded_cutlass(
|
||||
) -> torch.Tensor:
|
||||
pad_multiple = 4
|
||||
dim = qx.shape[0]
|
||||
padded = dim if dim % pad_multiple == 0 else dim + pad_multiple - (
|
||||
dim % pad_multiple)
|
||||
padded = (
|
||||
dim if dim % pad_multiple == 0 else dim + pad_multiple - (dim % pad_multiple)
|
||||
)
|
||||
|
||||
padded_shape = [padded, *qx.shape[1:]]
|
||||
padded_qx = torch.zeros(padded_shape, device=qx.device, dtype=qx.dtype)
|
||||
padded_qx[0:qx.shape[0], ...].copy_(qx)
|
||||
padded_qx[0 : qx.shape[0], ...].copy_(qx)
|
||||
|
||||
padded_x_scale_shape = [*x_scale.shape[1:], padded]
|
||||
padded_x_scale = torch.ones(padded_x_scale_shape,
|
||||
device=x_scale.device,
|
||||
dtype=x_scale.dtype).permute(-1, -2)
|
||||
padded_x_scale[0:x_scale.shape[0], ...].copy_(x_scale)
|
||||
padded_x_scale = torch.ones(
|
||||
padded_x_scale_shape, device=x_scale.device, dtype=x_scale.dtype
|
||||
).permute(-1, -2)
|
||||
padded_x_scale[0 : x_scale.shape[0], ...].copy_(x_scale)
|
||||
|
||||
output = cutlass_scaled_mm(padded_qx, weight, padded_x_scale, weight_scale,
|
||||
block_size, output_dtype, True)
|
||||
return output[0:qx.shape[0], ...]
|
||||
output = cutlass_scaled_mm(
|
||||
padded_qx, weight, padded_x_scale, weight_scale, block_size, output_dtype, True
|
||||
)
|
||||
return output[0 : qx.shape[0], ...]
|
||||
|
||||
|
||||
def _padded_cutlass_fake(
|
||||
@@ -173,9 +186,9 @@ def _padded_cutlass_fake(
|
||||
block_size: list[int],
|
||||
output_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty((qx.size(0), weight.size(0)),
|
||||
dtype=output_dtype,
|
||||
device=qx.device)
|
||||
return torch.empty(
|
||||
(qx.size(0), weight.size(0)), dtype=output_dtype, device=qx.device
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
@@ -185,18 +198,30 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def _fp8_gemm_nt_op(q_input: torch.Tensor, input_scale: torch.Tensor,
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor,
|
||||
output: torch.Tensor, use_deep_gemm_e8m0: bool) -> None:
|
||||
fp8_gemm_nt((q_input, input_scale), (weight, weight_scale),
|
||||
output,
|
||||
is_deep_gemm_e8m0_used=use_deep_gemm_e8m0)
|
||||
def _fp8_gemm_nt_op(
|
||||
q_input: torch.Tensor,
|
||||
input_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
use_deep_gemm_e8m0: bool,
|
||||
) -> None:
|
||||
fp8_gemm_nt(
|
||||
(q_input, input_scale),
|
||||
(weight, weight_scale),
|
||||
output,
|
||||
is_deep_gemm_e8m0_used=use_deep_gemm_e8m0,
|
||||
)
|
||||
|
||||
|
||||
def _fp8_gemm_nt_op_fake(q_input: torch.Tensor, input_scale: torch.Tensor,
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
use_deep_gemm_e8m0: bool) -> None:
|
||||
def _fp8_gemm_nt_op_fake(
|
||||
q_input: torch.Tensor,
|
||||
input_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
use_deep_gemm_e8m0: bool,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@@ -233,15 +258,21 @@ class W8A8BlockFp8LinearOp:
|
||||
# We can't use _dispatch_w8a8_blockscale_op to figure out if we want
|
||||
# to use deepgemm because we don't know the shape of weights (and
|
||||
# whether deepgemm supports it) at the init time.
|
||||
self.w8a8_blockscale_op, self.input_quant_op = \
|
||||
self._dispatch_w8a8_blockscale_op(
|
||||
cutlass_block_fp8_supported, use_aiter_and_is_supported)
|
||||
self.deepgemm_input_quant_op = (QuantFP8(
|
||||
False,
|
||||
self.act_quant_group_shape,
|
||||
column_major_scales=True,
|
||||
use_ue8m0=self.use_deep_gemm_e8m0) if self.is_deep_gemm_supported
|
||||
else None)
|
||||
self.w8a8_blockscale_op, self.input_quant_op = (
|
||||
self._dispatch_w8a8_blockscale_op(
|
||||
cutlass_block_fp8_supported, use_aiter_and_is_supported
|
||||
)
|
||||
)
|
||||
self.deepgemm_input_quant_op = (
|
||||
QuantFP8(
|
||||
False,
|
||||
self.act_quant_group_shape,
|
||||
column_major_scales=True,
|
||||
use_ue8m0=self.use_deep_gemm_e8m0,
|
||||
)
|
||||
if self.is_deep_gemm_supported
|
||||
else None
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -257,8 +288,9 @@ class W8A8BlockFp8LinearOp:
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
output_dtype = input.dtype
|
||||
|
||||
if should_use_deepgemm_for_fp8_linear(output_dtype, weight,
|
||||
self.is_deep_gemm_supported):
|
||||
if should_use_deepgemm_for_fp8_linear(
|
||||
output_dtype, weight, self.is_deep_gemm_supported
|
||||
):
|
||||
output = self._run_deepgemm(input_2d, weight, weight_scale)
|
||||
else:
|
||||
output = self.w8a8_blockscale_op(input_2d, weight, weight_scale)
|
||||
@@ -275,12 +307,14 @@ class W8A8BlockFp8LinearOp:
|
||||
) -> torch.Tensor:
|
||||
assert self.deepgemm_input_quant_op is not None
|
||||
q_input, input_scale = self.deepgemm_input_quant_op(input_2d)
|
||||
output = torch.empty((q_input.shape[0], weight.shape[0]),
|
||||
dtype=torch.bfloat16,
|
||||
device=q_input.device)
|
||||
torch.ops.vllm.fp8_gemm_nt_op(q_input, input_scale, weight,
|
||||
weight_scale, output,
|
||||
self.use_deep_gemm_e8m0)
|
||||
output = torch.empty(
|
||||
(q_input.shape[0], weight.shape[0]),
|
||||
dtype=torch.bfloat16,
|
||||
device=q_input.device,
|
||||
)
|
||||
torch.ops.vllm.fp8_gemm_nt_op(
|
||||
q_input, input_scale, weight, weight_scale, output, self.use_deep_gemm_e8m0
|
||||
)
|
||||
return output
|
||||
|
||||
def _run_cutlass(
|
||||
@@ -292,15 +326,24 @@ class W8A8BlockFp8LinearOp:
|
||||
assert self.input_quant_op is not None
|
||||
q_input, input_scale = self.input_quant_op(input_2d)
|
||||
if self.is_hopper:
|
||||
return torch.ops.vllm.padded_cutlass(q_input, weight, input_scale,
|
||||
weight_scale,
|
||||
list(self.weight_group_shape),
|
||||
input_2d.dtype)
|
||||
return torch.ops.vllm.padded_cutlass(
|
||||
q_input,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
list(self.weight_group_shape),
|
||||
input_2d.dtype,
|
||||
)
|
||||
else:
|
||||
return cutlass_scaled_mm(q_input, weight,
|
||||
input_scale, weight_scale,
|
||||
list(self.weight_group_shape),
|
||||
input_2d.dtype, False)
|
||||
return cutlass_scaled_mm(
|
||||
q_input,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
list(self.weight_group_shape),
|
||||
input_2d.dtype,
|
||||
False,
|
||||
)
|
||||
|
||||
def _run_aiter(
|
||||
self,
|
||||
@@ -310,10 +353,16 @@ class W8A8BlockFp8LinearOp:
|
||||
) -> torch.Tensor:
|
||||
assert self.act_quant_group_shape == GroupShape(1, 128)
|
||||
q_input, input_scale = aiter_per1x128_quant(
|
||||
input_2d.contiguous(), quant_dtype=rocm_aiter.dtypes.fp8)
|
||||
input_2d.contiguous(), quant_dtype=rocm_aiter.dtypes.fp8
|
||||
)
|
||||
return torch.ops.vllm.rocm_aiter_gemm_w8a8_blockscale(
|
||||
q_input, weight, input_scale, weight_scale,
|
||||
self.weight_group_shape, input_2d.dtype)
|
||||
q_input,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
self.weight_group_shape,
|
||||
input_2d.dtype,
|
||||
)
|
||||
|
||||
def _run_triton(
|
||||
self,
|
||||
@@ -324,34 +373,52 @@ class W8A8BlockFp8LinearOp:
|
||||
assert self.input_quant_op is not None
|
||||
q_input, input_scale = self.input_quant_op(input_2d)
|
||||
return torch.ops.vllm.w8a8_triton_block_scaled_mm_func(
|
||||
q_input, weight, input_scale, weight_scale,
|
||||
self.weight_group_shape, input_2d.dtype)
|
||||
q_input,
|
||||
weight,
|
||||
input_scale,
|
||||
weight_scale,
|
||||
self.weight_group_shape,
|
||||
input_2d.dtype,
|
||||
)
|
||||
|
||||
def _dispatch_w8a8_blockscale_op(
|
||||
self,
|
||||
use_cutlass: bool,
|
||||
use_aiter_and_is_supported: bool,
|
||||
) -> tuple[Callable[[
|
||||
) -> tuple[
|
||||
Callable[
|
||||
[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
],
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
], torch.Tensor], Optional[QuantFP8]]:
|
||||
],
|
||||
Optional[QuantFP8],
|
||||
]:
|
||||
if use_cutlass:
|
||||
return self._run_cutlass, (QuantFP8(False,
|
||||
self.act_quant_group_shape,
|
||||
column_major_scales=True,
|
||||
use_ue8m0=False))
|
||||
return self._run_cutlass, (
|
||||
QuantFP8(
|
||||
False,
|
||||
self.act_quant_group_shape,
|
||||
column_major_scales=True,
|
||||
use_ue8m0=False,
|
||||
)
|
||||
)
|
||||
if use_aiter_and_is_supported:
|
||||
return self._run_aiter, None
|
||||
return self._run_triton, (QuantFP8(False,
|
||||
self.act_quant_group_shape,
|
||||
column_major_scales=False,
|
||||
use_ue8m0=False))
|
||||
return self._run_triton, (
|
||||
QuantFP8(
|
||||
False,
|
||||
self.act_quant_group_shape,
|
||||
column_major_scales=False,
|
||||
use_ue8m0=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def input_to_float8(
|
||||
x: torch.Tensor,
|
||||
dtype: Optional[torch.dtype] = None
|
||||
x: torch.Tensor, dtype: Optional[torch.dtype] = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values "
|
||||
"with tensor-wise quantization."""
|
||||
@@ -410,8 +477,9 @@ def _per_token_group_quant_fp8(
|
||||
row_g_id = g_id % groups_per_row
|
||||
|
||||
# Ensure offset calculations use int64 to prevent overflow
|
||||
y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (row_g_id.to(tl.int64) *
|
||||
group_size)
|
||||
y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (
|
||||
row_g_id.to(tl.int64) * group_size
|
||||
)
|
||||
y_ptr += y_ptr_offset
|
||||
|
||||
y_q_ptr_offset = g_id.to(tl.int64) * group_size
|
||||
@@ -465,8 +533,9 @@ def _per_token_group_quant_fp8_colmajor(
|
||||
row_g_id = g_id % groups_per_row
|
||||
|
||||
# Ensure offset calculations use int64 to prevent overflow
|
||||
y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (row_g_id.to(tl.int64) *
|
||||
group_size)
|
||||
y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (
|
||||
row_g_id.to(tl.int64) * group_size
|
||||
)
|
||||
y_ptr += y_ptr_offset
|
||||
|
||||
y_q_ptr_offset = g_id.to(tl.int64) * group_size
|
||||
@@ -478,8 +547,7 @@ def _per_token_group_quant_fp8_colmajor(
|
||||
scale_col = g_id % blocks_per_row
|
||||
scale_row = g_id // blocks_per_row
|
||||
# Ensure offset calculation uses int64 for y_s_ptr
|
||||
y_s_ptr_offset = (scale_col.to(tl.int64) * y_s_col_stride) + scale_row.to(
|
||||
tl.int64)
|
||||
y_s_ptr_offset = (scale_col.to(tl.int64) * y_s_col_stride) + scale_row.to(tl.int64)
|
||||
y_s_ptr += y_s_ptr_offset
|
||||
|
||||
cols = tl.arange(0, BLOCK) # group_size <= BLOCK
|
||||
@@ -523,9 +591,10 @@ def per_token_group_quant_fp8(
|
||||
if use_ue8m0 is None:
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
dtype = current_platform.fp8_dtype() if dtype is None else dtype
|
||||
assert (x.shape[-1] % group_size == 0), (
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
f"the last dimension of `x` {x.shape[-1]} must be divisible "
|
||||
f"by `group_size` {group_size}")
|
||||
f"by `group_size` {group_size}"
|
||||
)
|
||||
assert x.stride(-1) == 1, "`x` groups must be contiguous"
|
||||
|
||||
finfo = torch.finfo(dtype)
|
||||
@@ -539,18 +608,18 @@ def per_token_group_quant_fp8(
|
||||
|
||||
# Allocate the scale tensor in either row- or column-major format.
|
||||
if column_major_scales:
|
||||
shape = (x.shape[-1] // group_size, ) + x.shape[:-1]
|
||||
x_s = torch.empty(shape, device=x.device,
|
||||
dtype=torch.float32).permute(-1, -2)
|
||||
shape = (x.shape[-1] // group_size,) + x.shape[:-1]
|
||||
x_s = torch.empty(shape, device=x.device, dtype=torch.float32).permute(-1, -2)
|
||||
else:
|
||||
shape = x.shape[:-1] + (x.shape[-1] // group_size, )
|
||||
shape = x.shape[:-1] + (x.shape[-1] // group_size,)
|
||||
x_s = torch.empty(shape, device=x.device, dtype=torch.float32)
|
||||
|
||||
# prefer CUDA kernel if available
|
||||
# TODO(bnell): this causes some fp8 moe test to fail.
|
||||
if current_platform.is_cuda() and x.is_contiguous():
|
||||
torch.ops._C.per_token_group_fp8_quant(x, x_q, x_s, group_size, eps,
|
||||
fp8_min, fp8_max, use_ue8m0)
|
||||
torch.ops._C.per_token_group_fp8_quant(
|
||||
x, x_q, x_s, group_size, eps, fp8_min, fp8_max, use_ue8m0
|
||||
)
|
||||
return x_q, x_s
|
||||
|
||||
# TRITON FALLBACK
|
||||
@@ -561,7 +630,7 @@ def per_token_group_quant_fp8(
|
||||
num_warps = min(max(BLOCK // 256, 1), 8)
|
||||
num_stages = 1
|
||||
if column_major_scales:
|
||||
_per_token_group_quant_fp8_colmajor[(M, )](
|
||||
_per_token_group_quant_fp8_colmajor[(M,)](
|
||||
x,
|
||||
x_q,
|
||||
x_s,
|
||||
@@ -578,7 +647,7 @@ def per_token_group_quant_fp8(
|
||||
num_stages=num_stages,
|
||||
)
|
||||
else:
|
||||
_per_token_group_quant_fp8[(M, )](
|
||||
_per_token_group_quant_fp8[(M,)](
|
||||
x,
|
||||
x_q,
|
||||
x_s,
|
||||
@@ -656,12 +725,8 @@ def _w8a8_triton_block_scaled_mm(
|
||||
|
||||
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
|
||||
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
|
||||
a = tl.load(a_ptrs,
|
||||
mask=offs_k[None, :] < K - k * BLOCK_SIZE_K,
|
||||
other=0.0)
|
||||
b = tl.load(b_ptrs,
|
||||
mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
|
||||
other=0.0)
|
||||
a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0)
|
||||
|
||||
k_start = k * BLOCK_SIZE_K
|
||||
offs_ks = k_start // group_k
|
||||
@@ -687,8 +752,9 @@ def _w8a8_triton_block_scaled_mm(
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def get_w8a8_block_fp8_configs(N: int, K: int, block_n: int,
|
||||
block_k: int) -> Optional[dict[int, Any]]:
|
||||
def get_w8a8_block_fp8_configs(
|
||||
N: int, K: int, block_n: int, block_k: int
|
||||
) -> Optional[dict[int, Any]]:
|
||||
"""
|
||||
Return optimized configurations for the w8a8 block fp8 kernel.
|
||||
The return value will be a dictionary that maps an irregular grid of
|
||||
@@ -703,7 +769,8 @@ def get_w8a8_block_fp8_configs(N: int, K: int, block_n: int,
|
||||
json_file_name = f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8,block_shape=[{block_n},{block_k}].json" # noqa: E501
|
||||
|
||||
config_file_path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name
|
||||
)
|
||||
if os.path.exists(config_file_path):
|
||||
with open(config_file_path) as f:
|
||||
logger.info(
|
||||
@@ -759,7 +826,7 @@ def w8a8_triton_block_scaled_mm(
|
||||
assert triton.cdiv(N, block_n) == Bs.shape[0]
|
||||
assert triton.cdiv(K, block_k) == Bs.shape[1]
|
||||
|
||||
C_shape = A.shape[:-1] + (N, )
|
||||
C_shape = A.shape[:-1] + (N,)
|
||||
C = A.new_empty(C_shape, dtype=output_dtype)
|
||||
|
||||
configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1])
|
||||
@@ -780,8 +847,9 @@ def w8a8_triton_block_scaled_mm(
|
||||
}
|
||||
|
||||
def grid(META):
|
||||
return (triton.cdiv(M, META["BLOCK_SIZE_M"]) *
|
||||
triton.cdiv(N, META["BLOCK_SIZE_N"]), )
|
||||
return (
|
||||
triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]),
|
||||
)
|
||||
|
||||
_w8a8_triton_block_scaled_mm[grid](
|
||||
A,
|
||||
@@ -811,9 +879,9 @@ def w8a8_triton_block_scaled_mm(
|
||||
|
||||
|
||||
def requant_weight_ue8m0_inplace(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
block_size: Sequence[int] = (128, 128),
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
block_size: Sequence[int] = (128, 128),
|
||||
) -> None:
|
||||
"""Re-quantise *weight* so that its per-block scaling factors are in the
|
||||
UE8M0 (power-of-two) format expected by the new DeepGEMM kernels inplace.
|
||||
@@ -830,8 +898,9 @@ def requant_weight_ue8m0_inplace(
|
||||
return
|
||||
|
||||
if weight.dtype != torch.float8_e4m3fn:
|
||||
raise ValueError("Expected *weight* to be torch.float8_e4m3fn, got "
|
||||
f"{weight.dtype} instead.")
|
||||
raise ValueError(
|
||||
f"Expected *weight* to be torch.float8_e4m3fn, got {weight.dtype} instead."
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import per_block_cast_to_fp8
|
||||
|
||||
@@ -860,8 +929,9 @@ def requant_weight_ue8m0_inplace(
|
||||
s_exp = s_exp[:m_cur, :k_cur]
|
||||
w_dq = w_q.to(torch.float32) * s_exp
|
||||
# Re-quantise using power-of-two scaling (UE8M0).
|
||||
w_requant, s_requant = per_block_cast_to_fp8(w_dq, [block_m, block_k],
|
||||
use_ue8m0=True)
|
||||
w_requant, s_requant = per_block_cast_to_fp8(
|
||||
w_dq, [block_m, block_k], use_ue8m0=True
|
||||
)
|
||||
|
||||
# Write back the results in-place.
|
||||
w_q.copy_(w_requant)
|
||||
@@ -871,28 +941,39 @@ def requant_weight_ue8m0_inplace(
|
||||
def check_aiter_fp8_linear_support() -> bool:
|
||||
"""AITER is only supported on ROCm and only for FP8_FNUZ
|
||||
and at the moment are MI300 series"""
|
||||
return (current_platform.is_rocm() and envs.VLLM_ROCM_USE_AITER
|
||||
and envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
and current_platform.is_fp8_fnuz())
|
||||
return (
|
||||
current_platform.is_rocm()
|
||||
and envs.VLLM_ROCM_USE_AITER
|
||||
and envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
and current_platform.is_fp8_fnuz()
|
||||
)
|
||||
|
||||
|
||||
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Pad the weight tensor. This is an optimization on ROCm platform, which
|
||||
can benefit from tensors located far enough from one another in memory"""
|
||||
if (envs.VLLM_ROCM_FP8_PADDING and current_platform.is_rocm()
|
||||
and weight.stride(-1) == 1
|
||||
and (weight.stride(-2) * weight.element_size()) % 512 == 0):
|
||||
if (
|
||||
envs.VLLM_ROCM_FP8_PADDING
|
||||
and current_platform.is_rocm()
|
||||
and weight.stride(-1) == 1
|
||||
and (weight.stride(-2) * weight.element_size()) % 512 == 0
|
||||
):
|
||||
num_pad = 256 // weight.element_size()
|
||||
import torch.nn.functional as F
|
||||
|
||||
weight = F.pad(weight, (0, num_pad), "constant", 0)[..., :-num_pad]
|
||||
torch.cuda.empty_cache()
|
||||
return weight
|
||||
|
||||
|
||||
def validate_fp8_block_shape(layer: torch.nn.Module, input_size: int,
|
||||
output_size: int, input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
block_size: list[int]) -> None:
|
||||
def validate_fp8_block_shape(
|
||||
layer: torch.nn.Module,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
block_size: list[int],
|
||||
) -> None:
|
||||
"""Validate block quantization shapes for tensor parallelism."""
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
|
||||
@@ -900,15 +981,18 @@ def validate_fp8_block_shape(layer: torch.nn.Module, input_size: int,
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
|
||||
# Required by row parallel
|
||||
if (tp_size > 1 and input_size // input_size_per_partition == tp_size
|
||||
and input_size_per_partition % block_k != 0):
|
||||
if (
|
||||
tp_size > 1
|
||||
and input_size // input_size_per_partition == tp_size
|
||||
and input_size_per_partition % block_k != 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"Weight input_size_per_partition = {input_size_per_partition} "
|
||||
f"is not divisible by weight quantization block_k = {block_k}.")
|
||||
f"is not divisible by weight quantization block_k = {block_k}."
|
||||
)
|
||||
|
||||
# Required by column parallel or enabling merged weights
|
||||
is_tp_split = (tp_size > 1
|
||||
and output_size // sum(output_partition_sizes) == tp_size)
|
||||
is_tp_split = tp_size > 1 and output_size // sum(output_partition_sizes) == tp_size
|
||||
is_merged_gemm = len(output_partition_sizes) > 1
|
||||
if is_tp_split or is_merged_gemm:
|
||||
sizes_to_check = output_partition_sizes
|
||||
@@ -921,33 +1005,44 @@ def validate_fp8_block_shape(layer: torch.nn.Module, input_size: int,
|
||||
raise ValueError(
|
||||
f"Weight output_partition_size = "
|
||||
f"{output_partition_size} is not divisible by "
|
||||
f"weight quantization block_n = {block_n}.")
|
||||
f"weight quantization block_n = {block_n}."
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_weight_parameter(
|
||||
output_size_per_partition: int, input_size_per_partition: int,
|
||||
weight_loader: Optional[Callable]) -> torch.nn.Parameter:
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
weight_loader: Optional[Callable],
|
||||
) -> torch.nn.Parameter:
|
||||
"""Create FP8 weight parameter."""
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
|
||||
return ModelWeightParameter(data=torch.empty(output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=torch.float8_e4m3fn),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader)
|
||||
return ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_scale_parameter(
|
||||
parameter_type: torch.nn.Parameter, output_partition_sizes: list[int],
|
||||
input_size_per_partition: int, block_size: Optional[list[int]],
|
||||
weight_loader: Optional[Callable]) -> torch.nn.Parameter:
|
||||
parameter_type: torch.nn.Parameter,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
block_size: Optional[list[int]],
|
||||
weight_loader: Optional[Callable],
|
||||
) -> torch.nn.Parameter:
|
||||
"""Create scale parameter based on quantization strategy."""
|
||||
if parameter_type == ChannelQuantScaleParameter:
|
||||
scale = parameter_type(data=torch.empty(
|
||||
(sum(output_partition_sizes), 1), dtype=torch.float32),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader)
|
||||
scale = parameter_type(
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
elif parameter_type == BlockQuantScaleParameter:
|
||||
assert block_size is not None
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
@@ -963,9 +1058,10 @@ def create_fp8_scale_parameter(
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
elif parameter_type == PerTensorScaleParameter:
|
||||
scale = parameter_type(data=torch.empty(len(output_partition_sizes),
|
||||
dtype=torch.float32),
|
||||
weight_loader=weight_loader)
|
||||
scale = parameter_type(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown parameter type: {parameter_type}")
|
||||
|
||||
@@ -974,14 +1070,15 @@ def create_fp8_scale_parameter(
|
||||
|
||||
|
||||
def create_fp8_input_scale(
|
||||
output_partition_sizes: list[int],
|
||||
weight_loader: Optional[Callable]) -> torch.nn.Parameter:
|
||||
output_partition_sizes: list[int], weight_loader: Optional[Callable]
|
||||
) -> torch.nn.Parameter:
|
||||
"""Create input scale parameter for static activation quantization."""
|
||||
from vllm.model_executor.parameter import PerTensorScaleParameter
|
||||
|
||||
scale = PerTensorScaleParameter(data=torch.empty(
|
||||
len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader)
|
||||
scale = PerTensorScaleParameter(
|
||||
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
scale[:] = torch.finfo(torch.float32).min
|
||||
return scale
|
||||
|
||||
@@ -990,15 +1087,18 @@ def process_fp8_weight_tensor_strategy(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
logical_widths: list[int],
|
||||
input_scale: Optional[torch.Tensor] = None
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Process weights for tensor-wise quantization strategy."""
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz, requantize_with_max_scale)
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
requantize_with_max_scale,
|
||||
)
|
||||
|
||||
if current_platform.is_fp8_fnuz():
|
||||
weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=weight, weight_scale=weight_scale, input_scale=input_scale)
|
||||
weight=weight, weight_scale=weight_scale, input_scale=input_scale
|
||||
)
|
||||
|
||||
# Requantize with max scale
|
||||
weight_scale, weight = requantize_with_max_scale(
|
||||
@@ -1014,15 +1114,17 @@ def process_fp8_weight_tensor_strategy(
|
||||
def process_fp8_weight_channel_strategy(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
input_scale: Optional[torch.Tensor] = None
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
"""Process weights for channel-wise quantization strategy."""
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz)
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
|
||||
if current_platform.is_fp8_fnuz():
|
||||
weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=weight, weight_scale=weight_scale, input_scale=input_scale)
|
||||
weight=weight, weight_scale=weight_scale, input_scale=input_scale
|
||||
)
|
||||
|
||||
return weight, weight_scale, input_scale
|
||||
|
||||
@@ -1033,37 +1135,48 @@ def process_fp8_weight_block_strategy(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Process weights for block-wise quantization strategy."""
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
normalize_e4m3fn_to_e4m3fnuz)
|
||||
normalize_e4m3fn_to_e4m3fnuz,
|
||||
)
|
||||
|
||||
if current_platform.is_fp8_fnuz():
|
||||
weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight=weight, weight_scale=weight_scale)
|
||||
weight=weight, weight_scale=weight_scale
|
||||
)
|
||||
|
||||
weight = _maybe_pad_fp8_weight(weight)
|
||||
return weight, weight_scale
|
||||
|
||||
|
||||
def maybe_post_process_fp8_weight_block(layer: torch.nn.Module,
|
||||
cutlass_block_fp8_supported: bool):
|
||||
def maybe_post_process_fp8_weight_block(
|
||||
layer: torch.nn.Module, cutlass_block_fp8_supported: bool
|
||||
):
|
||||
assert layer.weight_block_size is not None
|
||||
|
||||
from vllm.utils.deep_gemm import (is_deep_gemm_e8m0_used,
|
||||
should_use_deepgemm_for_fp8_linear)
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_e8m0_used,
|
||||
should_use_deepgemm_for_fp8_linear,
|
||||
)
|
||||
|
||||
# On Blackwell or Hopper, if E8M0 for DeepGemm is used, we need to
|
||||
# requantize the weight and input to the specific scale
|
||||
# at the same time.
|
||||
should_use_deepgemm = should_use_deepgemm_for_fp8_linear(
|
||||
layer.orig_dtype, layer.weight)
|
||||
layer.orig_dtype, layer.weight
|
||||
)
|
||||
if is_deep_gemm_e8m0_used() and should_use_deepgemm:
|
||||
block_sz = tuple(layer.weight_block_size)
|
||||
requant_weight_ue8m0_inplace(layer.weight.data,
|
||||
layer.weight_scale.data, block_sz)
|
||||
requant_weight_ue8m0_inplace(
|
||||
layer.weight.data, layer.weight_scale.data, block_sz
|
||||
)
|
||||
# SM90 Block FP8 CUTLASS requires row-major weight scales
|
||||
elif (current_platform.is_device_capability(90)
|
||||
and cutlass_block_fp8_supported and not should_use_deepgemm):
|
||||
elif (
|
||||
current_platform.is_device_capability(90)
|
||||
and cutlass_block_fp8_supported
|
||||
and not should_use_deepgemm
|
||||
):
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
layer.weight_scale.data.T.contiguous(), requires_grad=False)
|
||||
layer.weight_scale.data.T.contiguous(), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def expert_weight_is_col_major(x: torch.Tensor) -> bool:
|
||||
|
||||
@@ -9,10 +9,11 @@ from typing import TYPE_CHECKING, Optional, Union
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.linear import (LinearBase,
|
||||
UnquantizedLinearMethod)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead, UnquantizedEmbeddingMethod)
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..gptq import GPTQConfig
|
||||
@@ -25,16 +26,13 @@ else:
|
||||
# Match dynamic rules with module name (prefix) and override quantize
|
||||
# config if module (prefix) matches a rule
|
||||
def override_config(config: Union[GPTQConfig, GPTQMarlinConfig], prefix: str):
|
||||
weight_bits = get_dynamic_override(config, prefix, "bits",
|
||||
config.weight_bits)
|
||||
weight_bits = get_dynamic_override(config, prefix, "bits", config.weight_bits)
|
||||
if isinstance(weight_bits, int):
|
||||
config.weight_bits = weight_bits
|
||||
group_size = get_dynamic_override(config, prefix, "group_size",
|
||||
config.group_size)
|
||||
group_size = get_dynamic_override(config, prefix, "group_size", config.group_size)
|
||||
if isinstance(group_size, int):
|
||||
config.group_size = group_size
|
||||
desc_act = get_dynamic_override(config, prefix, "desc_act",
|
||||
config.desc_act)
|
||||
desc_act = get_dynamic_override(config, prefix, "desc_act", config.desc_act)
|
||||
if isinstance(desc_act, bool):
|
||||
config.desc_act = desc_act
|
||||
|
||||
@@ -46,25 +44,27 @@ def override_config(config: Union[GPTQConfig, GPTQMarlinConfig], prefix: str):
|
||||
config.is_sym = is_sym
|
||||
|
||||
if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
|
||||
raise ValueError("Unsupported quantization config: "
|
||||
f"bits={config.weight_bits}, sym={config.is_sym}")
|
||||
raise ValueError(
|
||||
"Unsupported quantization config: "
|
||||
f"bits={config.weight_bits}, sym={config.is_sym}"
|
||||
)
|
||||
|
||||
config.quant_type = config.TYPE_MAP[(config.weight_bits,
|
||||
config.is_sym)]
|
||||
config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
|
||||
elif config.get_name() == "gptq":
|
||||
assert isinstance(config, GPTQConfig)
|
||||
if config.weight_bits not in [2, 3, 4, 8]:
|
||||
raise ValueError(
|
||||
"Currently, only 2/3/4/8-bit weight quantization is "
|
||||
f"supported for GPTQ, but got {config.weight_bits} bits.")
|
||||
f"supported for GPTQ, but got {config.weight_bits} bits."
|
||||
)
|
||||
|
||||
|
||||
def get_dynamic_override(
|
||||
config: Union[GPTQConfig, GPTQMarlinConfig],
|
||||
layer_name: str,
|
||||
key: Optional[str] = None,
|
||||
default_value: Union[int, bool,
|
||||
None] = None) -> Union[dict, int, bool, None]:
|
||||
default_value: Union[int, bool, None] = None,
|
||||
) -> Union[dict, int, bool, None]:
|
||||
for pattern, pattern_dict in config.dynamic.items():
|
||||
# Negative match: matched modules are excluded from quantized init
|
||||
if pattern.startswith("-:"):
|
||||
@@ -83,7 +83,7 @@ def get_dynamic_override(
|
||||
def is_layer_gptq_quantized(
|
||||
prefix: str,
|
||||
quantized_layers: list[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({})
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
) -> bool:
|
||||
# prefix: model.layers.0.self_attn.q_proj
|
||||
# proj_name: q_proj
|
||||
@@ -106,8 +106,9 @@ def is_layer_gptq_quantized(
|
||||
|
||||
is_quantized = None
|
||||
for shard_prefix in shard_prefixes:
|
||||
is_shard_quantized = any(layer in shard_prefix
|
||||
for layer in quantized_layers)
|
||||
is_shard_quantized = any(
|
||||
layer in shard_prefix for layer in quantized_layers
|
||||
)
|
||||
|
||||
if is_quantized is None:
|
||||
is_quantized = is_shard_quantized
|
||||
@@ -115,7 +116,8 @@ def is_layer_gptq_quantized(
|
||||
raise ValueError(
|
||||
f"Detected some but not all shards of {prefix} "
|
||||
"are quantized. All shards of fused layers "
|
||||
"to have the same precision.")
|
||||
"to have the same precision."
|
||||
)
|
||||
else:
|
||||
is_quantized = any(layer in prefix for layer in quantized_layers)
|
||||
|
||||
@@ -130,18 +132,20 @@ def get_linear_quant_method(
|
||||
linear_method_cls: type,
|
||||
):
|
||||
cloned_config = deepcopy(config)
|
||||
parallel_lm_head_quantized = isinstance(
|
||||
layer, ParallelLMHead) and cloned_config.lm_head_quantized
|
||||
parallel_lm_head_quantized = (
|
||||
isinstance(layer, ParallelLMHead) and cloned_config.lm_head_quantized
|
||||
)
|
||||
if isinstance(layer, LinearBase) or parallel_lm_head_quantized:
|
||||
is_layer_quantized = is_layer_gptq_quantized(
|
||||
prefix=prefix,
|
||||
quantized_layers=cloned_config.modules_in_block_to_quantize,
|
||||
fused_mapping=cloned_config.packed_modules_mapping)
|
||||
fused_mapping=cloned_config.packed_modules_mapping,
|
||||
)
|
||||
# False = skip module, None = no override, else = Positive match
|
||||
if get_dynamic_override( # noqa: E712
|
||||
cloned_config, # noqa: E712
|
||||
layer_name=prefix) == False or (
|
||||
not is_layer_quantized): # noqa: E712
|
||||
cloned_config, # noqa: E712
|
||||
layer_name=prefix,
|
||||
) == False or (not is_layer_quantized): # noqa: E712
|
||||
if parallel_lm_head_quantized:
|
||||
return UnquantizedEmbeddingMethod()
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
@@ -30,12 +30,9 @@ def apply_w8a8_block_int8_linear(
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
|
||||
q_input, x_scale = per_token_group_quant_int8(input_2d, block_size[1])
|
||||
output = w8a8_block_int8_matmul(q_input,
|
||||
weight,
|
||||
x_scale,
|
||||
weight_scale,
|
||||
block_size,
|
||||
output_dtype=input.dtype)
|
||||
output = w8a8_block_int8_matmul(
|
||||
q_input, weight, x_scale, weight_scale, block_size, output_dtype=input.dtype
|
||||
)
|
||||
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
@@ -43,8 +40,8 @@ def apply_w8a8_block_int8_linear(
|
||||
|
||||
|
||||
def input_to_int8(
|
||||
x: torch.Tensor,
|
||||
dtype: torch.dtype = torch.int8) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x: torch.Tensor, dtype: torch.dtype = torch.int8
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to int8 values with
|
||||
tensor-wise quantization."""
|
||||
iinfo = torch.iinfo(dtype)
|
||||
@@ -78,8 +75,8 @@ def block_dequant(
|
||||
for i in range(k_tiles):
|
||||
for j in range(n_tiles):
|
||||
x_dq_block[
|
||||
j * block_n:min((j + 1) * block_n, n),
|
||||
i * block_k:min((i + 1) * block_k, k),
|
||||
j * block_n : min((j + 1) * block_n, n),
|
||||
i * block_k : min((i + 1) * block_k, k),
|
||||
] *= x_s[j][i]
|
||||
|
||||
return x_dq_block
|
||||
@@ -91,15 +88,17 @@ if current_platform.is_rocm():
|
||||
# NOTE: This can be removed when hip.libdevice.round() is available.
|
||||
@core.extern
|
||||
def round_f32(arg0, _builder=None):
|
||||
return core.extern_elementwise("",
|
||||
"", [arg0], {
|
||||
(core.dtype("fp32"), ):
|
||||
("llvm.round", core.dtype("fp32")),
|
||||
(core.dtype("fp64"), ):
|
||||
("llvm.round", core.dtype("fp64")),
|
||||
},
|
||||
is_pure=True,
|
||||
_builder=_builder)
|
||||
return core.extern_elementwise(
|
||||
"",
|
||||
"",
|
||||
[arg0],
|
||||
{
|
||||
(core.dtype("fp32"),): ("llvm.round", core.dtype("fp32")),
|
||||
(core.dtype("fp64"),): ("llvm.round", core.dtype("fp64")),
|
||||
},
|
||||
is_pure=True,
|
||||
_builder=_builder,
|
||||
)
|
||||
|
||||
@triton.jit
|
||||
def round_int8(x):
|
||||
@@ -127,8 +126,7 @@ def _per_token_quant_int8(
|
||||
cols = tl.arange(0, BLOCK)
|
||||
mask = cols < N
|
||||
|
||||
x = tl.load(x_ptr + row_id * stride_x + cols, mask=mask,
|
||||
other=0.0).to(tl.float32)
|
||||
x = tl.load(x_ptr + row_id * stride_x + cols, mask=mask, other=0.0).to(tl.float32)
|
||||
absmax = tl.maximum(tl.max(tl.abs(x)), 1e-10)
|
||||
scale_x = absmax / 127
|
||||
x_q = x * (127 / absmax)
|
||||
@@ -142,15 +140,13 @@ def per_token_quant_int8(x):
|
||||
M = x.numel() // x.shape[-1]
|
||||
N = x.shape[-1]
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=torch.int8)
|
||||
scales = torch.empty(x.shape[:-1] + (1, ),
|
||||
device=x.device,
|
||||
dtype=torch.float32)
|
||||
scales = torch.empty(x.shape[:-1] + (1,), device=x.device, dtype=torch.float32)
|
||||
BLOCK = triton.next_power_of_2(N)
|
||||
# heuristics for number of warps
|
||||
num_warps = min(max(BLOCK // 256, 1), 8)
|
||||
|
||||
assert x.is_contiguous()
|
||||
_per_token_quant_int8[(M, )](
|
||||
_per_token_quant_int8[(M,)](
|
||||
x,
|
||||
x_q,
|
||||
scales,
|
||||
@@ -229,8 +225,9 @@ def per_token_group_quant_int8(
|
||||
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
|
||||
scaling factor for quantization.
|
||||
"""
|
||||
assert (x.shape[-1] % group_size == 0
|
||||
), "the last dimension of `x` cannot be divisible by `group_size`"
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` cannot be divisible by `group_size`"
|
||||
)
|
||||
assert x.is_contiguous(), "`x` is not contiguous"
|
||||
|
||||
iinfo = torch.iinfo(dtype)
|
||||
@@ -239,15 +236,15 @@ def per_token_group_quant_int8(
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=dtype)
|
||||
x_s = torch.empty(
|
||||
x.shape[:-1] + (x.shape[-1] // group_size, ),
|
||||
x.shape[:-1] + (x.shape[-1] // group_size,),
|
||||
device=x.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
# prefer CUDA kernel if available
|
||||
if current_platform.is_cuda():
|
||||
torch.ops._C.per_token_group_quant_int8(x, x_q, x_s, group_size, eps,
|
||||
float(int8_min),
|
||||
float(int8_max))
|
||||
torch.ops._C.per_token_group_quant_int8(
|
||||
x, x_q, x_s, group_size, eps, float(int8_min), float(int8_max)
|
||||
)
|
||||
return x_q, x_s
|
||||
|
||||
M = x.numel() // group_size
|
||||
@@ -257,7 +254,7 @@ def per_token_group_quant_int8(
|
||||
# heuristics for number of warps
|
||||
num_warps = min(max(BLOCK // 256, 1), 8)
|
||||
num_stages = 1
|
||||
_per_token_group_quant_int8[(M, )](
|
||||
_per_token_group_quant_int8[(M,)](
|
||||
x,
|
||||
x_q,
|
||||
x_s,
|
||||
@@ -333,20 +330,15 @@ def _w8a8_block_int8_matmul(
|
||||
|
||||
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
|
||||
for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):
|
||||
a = tl.load(a_ptrs,
|
||||
mask=offs_k[None, :] < K - k * BLOCK_SIZE_K,
|
||||
other=0.0)
|
||||
b = tl.load(b_ptrs,
|
||||
mask=offs_k[:, None] < K - k * BLOCK_SIZE_K,
|
||||
other=0.0)
|
||||
a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0)
|
||||
b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0)
|
||||
|
||||
k_start = k * BLOCK_SIZE_K
|
||||
offs_ks = k_start // group_k
|
||||
a_s = tl.load(As_ptrs + offs_ks * stride_As_k)
|
||||
b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k)
|
||||
|
||||
accumulator += tl.dot(a, b).to(tl.float32) * a_s[:,
|
||||
None] * b_s[None, :]
|
||||
accumulator += tl.dot(a, b).to(tl.float32) * a_s[:, None] * b_s[None, :]
|
||||
a_ptrs += BLOCK_SIZE_K * stride_ak
|
||||
b_ptrs += BLOCK_SIZE_K * stride_bk
|
||||
|
||||
@@ -365,8 +357,9 @@ def _w8a8_block_int8_matmul(
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def get_w8a8_block_int8_configs(N: int, K: int, block_n: int,
|
||||
block_k: int) -> Optional[dict[int, Any]]:
|
||||
def get_w8a8_block_int8_configs(
|
||||
N: int, K: int, block_n: int, block_k: int
|
||||
) -> Optional[dict[int, Any]]:
|
||||
"""
|
||||
Return optimized configurations for the w8a8 block fp8 kernel.
|
||||
|
||||
@@ -382,7 +375,8 @@ def get_w8a8_block_int8_configs(N: int, K: int, block_n: int,
|
||||
json_file_name = f"N={N},K={K},device_name={device_name},dtype=int8_w8a8,block_shape=[{block_n}, {block_k}].json" # noqa: E501
|
||||
|
||||
config_file_path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name)
|
||||
os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name
|
||||
)
|
||||
if os.path.exists(config_file_path):
|
||||
with open(config_file_path) as f:
|
||||
logger.info(
|
||||
@@ -395,8 +389,10 @@ def get_w8a8_block_int8_configs(N: int, K: int, block_n: int,
|
||||
# If no optimized configuration is available, we will use the default
|
||||
# configuration
|
||||
logger.warning(
|
||||
("Using default W8A8 Block INT8 kernel config. Performance might "
|
||||
"be sub-optimal! Config file not found at %s"),
|
||||
(
|
||||
"Using default W8A8 Block INT8 kernel config. Performance might "
|
||||
"be sub-optimal! Config file not found at %s"
|
||||
),
|
||||
config_file_path,
|
||||
)
|
||||
return None
|
||||
@@ -441,7 +437,7 @@ def w8a8_block_int8_matmul(
|
||||
assert triton.cdiv(N, block_n) == Bs.shape[0]
|
||||
assert triton.cdiv(K, block_k) == Bs.shape[1]
|
||||
|
||||
C_shape = A.shape[:-1] + (N, )
|
||||
C_shape = A.shape[:-1] + (N,)
|
||||
C = A.new_empty(C_shape, dtype=output_dtype)
|
||||
|
||||
configs = get_w8a8_block_int8_configs(N, K, block_size[0], block_size[1])
|
||||
@@ -462,8 +458,9 @@ def w8a8_block_int8_matmul(
|
||||
}
|
||||
|
||||
def grid(META):
|
||||
return (triton.cdiv(M, META["BLOCK_SIZE_M"]) *
|
||||
triton.cdiv(N, META["BLOCK_SIZE_N"]), )
|
||||
return (
|
||||
triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]),
|
||||
)
|
||||
|
||||
_w8a8_block_int8_matmul[grid](
|
||||
A,
|
||||
|
||||
@@ -20,12 +20,15 @@ def update_tensor_inplace(dst: torch.Tensor, src: torch.Tensor):
|
||||
|
||||
# Newly generated tensors need to replace existing tensors that are
|
||||
# already registered as parameters by vLLM (and won't be freed)
|
||||
def replace_parameter(mod: torch.nn.Module, name: str,
|
||||
new: Union[torch.Tensor, torch.nn.Parameter]) -> None:
|
||||
|
||||
def replace_parameter(
|
||||
mod: torch.nn.Module, name: str, new: Union[torch.Tensor, torch.nn.Parameter]
|
||||
) -> None:
|
||||
old = getattr(mod, name)
|
||||
if type(old) is type(new) and old.dtype == new.dtype and \
|
||||
old.untyped_storage().nbytes() == new.untyped_storage().nbytes():
|
||||
if (
|
||||
type(old) is type(new)
|
||||
and old.dtype == new.dtype
|
||||
and old.untyped_storage().nbytes() == new.untyped_storage().nbytes()
|
||||
):
|
||||
# If we can just update in-place to avoid re-registering
|
||||
# can be faster if the underlying storage is the same
|
||||
update_tensor_inplace(old, new)
|
||||
@@ -36,5 +39,4 @@ def replace_parameter(mod: torch.nn.Module, name: str,
|
||||
# parameters for `torch.compile` compatibility
|
||||
if not isinstance(new, torch.nn.Parameter):
|
||||
new = torch.nn.Parameter(new, requires_grad=False)
|
||||
mod.register_parameter(name,
|
||||
torch.nn.Parameter(new, requires_grad=False))
|
||||
mod.register_parameter(name, torch.nn.Parameter(new, requires_grad=False))
|
||||
|
||||
@@ -39,12 +39,19 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
return [-1, 128]
|
||||
|
||||
|
||||
def check_machete_supports_shape(in_features: int, out_featrues: int) \
|
||||
-> tuple[bool, Optional[str]]:
|
||||
def check_machete_supports_shape(
|
||||
in_features: int, out_featrues: int
|
||||
) -> tuple[bool, Optional[str]]:
|
||||
if in_features % MACHETE_PREPACKED_BLOCK_SHAPE[0] != 0:
|
||||
return False, "Input features size must be divisible by "\
|
||||
f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}"
|
||||
return (
|
||||
False,
|
||||
"Input features size must be divisible by "
|
||||
f"{MACHETE_PREPACKED_BLOCK_SHAPE[0]}",
|
||||
)
|
||||
if out_featrues % MACHETE_PREPACKED_BLOCK_SHAPE[1] != 0:
|
||||
return False, "Output features size must be divisible by "\
|
||||
f"{MACHETE_PREPACKED_BLOCK_SHAPE[1]}"
|
||||
return (
|
||||
False,
|
||||
"Output features size must be divisible by "
|
||||
f"{MACHETE_PREPACKED_BLOCK_SHAPE[1]}",
|
||||
)
|
||||
return True, None
|
||||
|
||||
@@ -40,8 +40,9 @@ def query_marlin_supported_quant_types(
|
||||
):
|
||||
if device_capability is None:
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (-1 if capability_tuple is None else
|
||||
capability_tuple.to_int())
|
||||
device_capability = (
|
||||
-1 if capability_tuple is None else capability_tuple.to_int()
|
||||
)
|
||||
|
||||
if device_capability < 80:
|
||||
return []
|
||||
@@ -50,10 +51,12 @@ def query_marlin_supported_quant_types(
|
||||
# - has_zp is False: return quant_types that has not zero points
|
||||
# - has_zp is None: both
|
||||
if has_zp is None:
|
||||
types0 = query_marlin_supported_quant_types(False, include_fp_type,
|
||||
device_capability)
|
||||
types1 = query_marlin_supported_quant_types(True, include_fp_type,
|
||||
device_capability)
|
||||
types0 = query_marlin_supported_quant_types(
|
||||
False, include_fp_type, device_capability
|
||||
)
|
||||
types1 = query_marlin_supported_quant_types(
|
||||
True, include_fp_type, device_capability
|
||||
)
|
||||
return types0 + types1
|
||||
|
||||
if has_zp:
|
||||
@@ -68,108 +71,126 @@ def query_marlin_supported_quant_types(
|
||||
|
||||
|
||||
def _check_marlin_supported(
|
||||
quant_type: ScalarType,
|
||||
group_size: Optional[int],
|
||||
has_zp: bool,
|
||||
device_capability: Optional[int] = None) -> tuple[bool, Optional[str]]:
|
||||
|
||||
quant_type: ScalarType,
|
||||
group_size: Optional[int],
|
||||
has_zp: bool,
|
||||
device_capability: Optional[int] = None,
|
||||
) -> tuple[bool, Optional[str]]:
|
||||
if device_capability is None:
|
||||
capability_tuple = current_platform.get_device_capability()
|
||||
device_capability = (-1 if capability_tuple is None else
|
||||
capability_tuple.to_int())
|
||||
device_capability = (
|
||||
-1 if capability_tuple is None else capability_tuple.to_int()
|
||||
)
|
||||
|
||||
supported_types = query_marlin_supported_quant_types(
|
||||
has_zp, True, device_capability)
|
||||
has_zp, True, device_capability
|
||||
)
|
||||
|
||||
if quant_type not in supported_types:
|
||||
return (False, f"Marlin does not support weight_bits = {quant_type}. "
|
||||
f"Only types = {supported_types} "
|
||||
f"are supported (for group_size = {group_size}, "
|
||||
f"device_capability = {device_capability}, zp = {has_zp}).")
|
||||
if (group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES):
|
||||
return (False, f"Marlin does not support group_size = {group_size}. "
|
||||
f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} "
|
||||
"are supported.")
|
||||
return (
|
||||
False,
|
||||
f"Marlin does not support weight_bits = {quant_type}. "
|
||||
f"Only types = {supported_types} "
|
||||
f"are supported (for group_size = {group_size}, "
|
||||
f"device_capability = {device_capability}, zp = {has_zp}).",
|
||||
)
|
||||
if group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES:
|
||||
return (
|
||||
False,
|
||||
f"Marlin does not support group_size = {group_size}. "
|
||||
f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} "
|
||||
"are supported.",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def check_marlin_supported(quant_type: ScalarType,
|
||||
group_size: int,
|
||||
has_zp: bool = False,
|
||||
device_capability: Optional[int] = None) -> bool:
|
||||
cond, _ = _check_marlin_supported(quant_type, group_size, has_zp,
|
||||
device_capability)
|
||||
def check_marlin_supported(
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
has_zp: bool = False,
|
||||
device_capability: Optional[int] = None,
|
||||
) -> bool:
|
||||
cond, _ = _check_marlin_supported(quant_type, group_size, has_zp, device_capability)
|
||||
return cond
|
||||
|
||||
|
||||
def verify_marlin_supported(quant_type: ScalarType,
|
||||
group_size: int,
|
||||
has_zp: bool = False) -> None:
|
||||
def verify_marlin_supported(
|
||||
quant_type: ScalarType, group_size: int, has_zp: bool = False
|
||||
) -> None:
|
||||
cond, err_msg = _check_marlin_supported(quant_type, group_size, has_zp)
|
||||
if not cond:
|
||||
assert err_msg is not None
|
||||
raise ValueError(err_msg)
|
||||
|
||||
|
||||
def verify_marlin_supports_shape(output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int, group_size: int) -> None:
|
||||
|
||||
def verify_marlin_supports_shape(
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int,
|
||||
group_size: int,
|
||||
) -> None:
|
||||
# Validate output_size_per_partition
|
||||
if output_size_per_partition % GPTQ_MARLIN_MIN_THREAD_N != 0:
|
||||
raise ValueError(f"Weight output_size_per_partition = "
|
||||
f"{output_size_per_partition} is not divisible by "
|
||||
f" min_thread_n = {GPTQ_MARLIN_MIN_THREAD_N}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq.")
|
||||
raise ValueError(
|
||||
f"Weight output_size_per_partition = "
|
||||
f"{output_size_per_partition} is not divisible by "
|
||||
f" min_thread_n = {GPTQ_MARLIN_MIN_THREAD_N}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
# Validate input_size_per_partition
|
||||
if input_size_per_partition % GPTQ_MARLIN_MIN_THREAD_K != 0:
|
||||
raise ValueError(f"Weight input_size_per_partition = "
|
||||
f"{input_size_per_partition} is not divisible "
|
||||
f"by min_thread_k = {GPTQ_MARLIN_MIN_THREAD_K}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq.")
|
||||
raise ValueError(
|
||||
f"Weight input_size_per_partition = "
|
||||
f"{input_size_per_partition} is not divisible "
|
||||
f"by min_thread_k = {GPTQ_MARLIN_MIN_THREAD_K}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
if (group_size < input_size
|
||||
and input_size_per_partition % group_size != 0):
|
||||
if group_size < input_size and input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
f"Weight input_size_per_partition = {input_size_per_partition}"
|
||||
f" is not divisible by group_size = {group_size}. "
|
||||
"Consider reducing tensor_parallel_size or running "
|
||||
"with --quantization gptq.")
|
||||
"with --quantization gptq."
|
||||
)
|
||||
|
||||
|
||||
def check_marlin_supports_shape(output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int, group_size: int) \
|
||||
-> tuple[bool, Optional[str]]:
|
||||
def check_marlin_supports_shape(
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
input_size: int,
|
||||
group_size: int,
|
||||
) -> tuple[bool, Optional[str]]:
|
||||
try:
|
||||
verify_marlin_supports_shape(output_size_per_partition,
|
||||
input_size_per_partition, input_size,
|
||||
group_size)
|
||||
verify_marlin_supports_shape(
|
||||
output_size_per_partition, input_size_per_partition, input_size, group_size
|
||||
)
|
||||
except ValueError as e:
|
||||
return False, e.__str__()
|
||||
return True, None
|
||||
|
||||
|
||||
def check_marlin_supports_layer(layer: LinearBase, group_size: int) \
|
||||
-> bool:
|
||||
output_size_per_partition = getattr(layer, "output_size_per_partition",
|
||||
None) or layer.output_size
|
||||
input_size_per_partition = getattr(layer, "input_size_per_partition",
|
||||
None) or layer.input_size
|
||||
def check_marlin_supports_layer(layer: LinearBase, group_size: int) -> bool:
|
||||
output_size_per_partition = (
|
||||
getattr(layer, "output_size_per_partition", None) or layer.output_size
|
||||
)
|
||||
input_size_per_partition = (
|
||||
getattr(layer, "input_size_per_partition", None) or layer.input_size
|
||||
)
|
||||
|
||||
return check_marlin_supports_shape(
|
||||
output_size_per_partition=output_size_per_partition,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
input_size=layer.input_size,
|
||||
group_size=group_size)[0]
|
||||
group_size=group_size,
|
||||
)[0]
|
||||
|
||||
|
||||
def check_moe_marlin_supports_layer(layer: LinearBase, group_size: int) \
|
||||
-> bool:
|
||||
def check_moe_marlin_supports_layer(layer: LinearBase, group_size: int) -> bool:
|
||||
hidden_size = layer.hidden_size
|
||||
intermediate_size_per_partition = layer.intermediate_size_per_partition
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
@@ -180,51 +201,58 @@ def check_moe_marlin_supports_layer(layer: LinearBase, group_size: int) \
|
||||
# gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size)
|
||||
# down: (n, k) = (hidden_size, intermediate_size_per_partition)
|
||||
# moe marlin requires n % 128 == 0 and k % 64 == 0
|
||||
supports_shape = hidden_size % 128 == 0 and \
|
||||
intermediate_size_per_partition % max(64, group_size) == 0
|
||||
supports_shape = (
|
||||
hidden_size % 128 == 0
|
||||
and intermediate_size_per_partition % max(64, group_size) == 0
|
||||
)
|
||||
supports_group_size = group_size in [-1, 32, 64, 128]
|
||||
return supports_shape and supports_group_size and \
|
||||
supports_router_weight and supports_activation
|
||||
return (
|
||||
supports_shape
|
||||
and supports_group_size
|
||||
and supports_router_weight
|
||||
and supports_activation
|
||||
)
|
||||
|
||||
|
||||
def marlin_moe_intermediate_size(w1_packed: torch.Tensor,
|
||||
w2_packed: torch.Tensor):
|
||||
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
|
||||
"""
|
||||
Given Marlin packed weight matrices w1_packed, and w2_packed,
|
||||
return the MoE intermediate size N
|
||||
return the MoE intermediate size N
|
||||
"""
|
||||
marlin_tile_size = 16
|
||||
return w2_packed.size(1) * marlin_tile_size
|
||||
|
||||
|
||||
def marlin_make_workspace(output_size_per_partition: int,
|
||||
device: torch.device) -> torch.Tensor:
|
||||
max_workspace_size = (output_size_per_partition //
|
||||
GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
|
||||
def marlin_make_workspace(
|
||||
output_size_per_partition: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
max_workspace_size = (
|
||||
output_size_per_partition // GPTQ_MARLIN_MIN_THREAD_N
|
||||
) * GPTQ_MARLIN_MAX_PARALLEL
|
||||
|
||||
return torch.zeros(max_workspace_size,
|
||||
dtype=torch.int,
|
||||
device=device,
|
||||
requires_grad=False)
|
||||
return torch.zeros(
|
||||
max_workspace_size, dtype=torch.int, device=device, requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def marlin_make_workspace_new(device: torch.device,
|
||||
max_blocks_per_sm: int = 1) -> torch.Tensor:
|
||||
def marlin_make_workspace_new(
|
||||
device: torch.device, max_blocks_per_sm: int = 1
|
||||
) -> torch.Tensor:
|
||||
# In the new marlin kernel, we use the num of threadblocks as workspace
|
||||
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
|
||||
sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
return torch.zeros(sms * max_blocks_per_sm,
|
||||
dtype=torch.int,
|
||||
device=device,
|
||||
requires_grad=False)
|
||||
return torch.zeros(
|
||||
sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
|
||||
return (not act_order) or (act_order and not is_row_parallel)
|
||||
|
||||
|
||||
def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
|
||||
is_row_parallel: bool) -> bool:
|
||||
def marlin_repeat_scales_on_all_ranks(
|
||||
act_order: bool, group_size: int, is_row_parallel: bool
|
||||
) -> bool:
|
||||
# Need to repeat scales on every rank if act_ordering or
|
||||
# channelwise and RowParallelLinear
|
||||
is_channelwise = group_size == -1
|
||||
@@ -232,17 +260,18 @@ def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
|
||||
|
||||
|
||||
def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
|
||||
return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
|
||||
requires_grad=False)
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def marlin_make_empty_zp(device: torch.device) -> torch.Tensor:
|
||||
return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
|
||||
requires_grad=False)
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def marlin_sort_g_idx(
|
||||
g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
|
||||
return g_idx[g_idx_sort_indices], g_idx_sort_indices
|
||||
|
||||
@@ -253,14 +282,13 @@ def get_scale_perms():
|
||||
scale_perm.extend([i + 8 * j for j in range(8)])
|
||||
scale_perm_single: list[int] = []
|
||||
for i in range(4):
|
||||
scale_perm_single.extend(
|
||||
[2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
|
||||
scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
|
||||
return scale_perm, scale_perm_single
|
||||
|
||||
|
||||
def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
|
||||
group_size: int) -> torch.Tensor:
|
||||
|
||||
def marlin_permute_scales(
|
||||
s: torch.Tensor, size_k: int, size_n: int, group_size: int
|
||||
) -> torch.Tensor:
|
||||
scale_perm, scale_perm_single = get_scale_perms()
|
||||
if group_size < size_k and group_size != -1:
|
||||
s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
|
||||
@@ -296,8 +324,9 @@ def marlin_moe_permute_scales(
|
||||
return output
|
||||
|
||||
|
||||
def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int,
|
||||
num_bits: int) -> torch.Tensor:
|
||||
def marlin_zero_points(
|
||||
zp: torch.Tensor, size_k: int, size_n: int, num_bits: int
|
||||
) -> torch.Tensor:
|
||||
# Permute zero-points in a similar way to scales, but do not use the
|
||||
# "single" permutation, since zero-points are applied on every MMA
|
||||
scale_perm, _ = get_scale_perms()
|
||||
@@ -318,8 +347,9 @@ def marlin_zero_points(zp: torch.Tensor, size_k: int, size_n: int,
|
||||
return zp
|
||||
|
||||
|
||||
def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
|
||||
size_n: int, num_bits: int) -> torch.Tensor:
|
||||
def awq_to_marlin_zero_points(
|
||||
q_zp_packed: torch.Tensor, size_k: int, size_n: int, num_bits: int
|
||||
) -> torch.Tensor:
|
||||
# AWQ zero-points are quantized and packed on the column dim.
|
||||
# In addition, the values are permuted based on dequantizer.
|
||||
# Here we undo both of these, and then apply marlin permutation
|
||||
@@ -341,8 +371,9 @@ def awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
|
||||
return marlin_zp
|
||||
|
||||
|
||||
def moe_awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
|
||||
size_n: int, num_bits: int):
|
||||
def moe_awq_to_marlin_zero_points(
|
||||
q_zp_packed: torch.Tensor, size_k: int, size_n: int, num_bits: int
|
||||
):
|
||||
num_experts = q_zp_packed.shape[0]
|
||||
output = torch.empty(
|
||||
(num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]),
|
||||
@@ -350,8 +381,7 @@ def moe_awq_to_marlin_zero_points(q_zp_packed: torch.Tensor, size_k: int,
|
||||
dtype=q_zp_packed.dtype,
|
||||
)
|
||||
for e in range(num_experts):
|
||||
output[e] = awq_to_marlin_zero_points(q_zp_packed[e], size_k, size_n,
|
||||
num_bits)
|
||||
output[e] = awq_to_marlin_zero_points(q_zp_packed[e], size_k, size_n, num_bits)
|
||||
return output
|
||||
|
||||
|
||||
@@ -363,7 +393,8 @@ def maybe_warn_marlin_atomic_add(device, dtype):
|
||||
logger.info_once(
|
||||
"You are running Marlin kernel with bf16 on GPUs before SM90. "
|
||||
"You can consider change to fp16 to achieve better performance "
|
||||
"if possible.")
|
||||
"if possible."
|
||||
)
|
||||
|
||||
|
||||
def maybe_warn_marlin_atomic_add_env():
|
||||
@@ -375,12 +406,13 @@ def maybe_warn_marlin_atomic_add_env():
|
||||
"Marlin kernel can achieve better performance for small size_n "
|
||||
"with experimental use_atomic_add feature. "
|
||||
"You can consider set environment variable "
|
||||
"VLLM_MARLIN_USE_ATOMIC_ADD to 1 if possible.")
|
||||
"VLLM_MARLIN_USE_ATOMIC_ADD to 1 if possible."
|
||||
)
|
||||
|
||||
|
||||
def should_use_atomic_add_reduce(m: int, n: int, k: int, device: torch.device,
|
||||
dtype: torch.dtype) -> bool:
|
||||
|
||||
def should_use_atomic_add_reduce(
|
||||
m: int, n: int, k: int, device: torch.device, dtype: torch.dtype
|
||||
) -> bool:
|
||||
# the performance of atomicAdd is better than global reduce
|
||||
# only when m*n is small and k is large
|
||||
if n >= 2048 or k < 2048 or device.type != "cuda":
|
||||
@@ -402,88 +434,98 @@ def should_use_atomic_add_reduce(m: int, n: int, k: int, device: torch.device,
|
||||
|
||||
|
||||
def apply_gptq_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
g_idx_sort_indices: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
is_k_full: bool,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
g_idx_sort_indices: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
is_k_full: bool,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = input.reshape(-1, input.shape[-1])
|
||||
out_shape = input.shape[:-1] + (output_size_per_partition, )
|
||||
out_shape = input.shape[:-1] + (output_size_per_partition,)
|
||||
|
||||
use_atomic_add = should_use_atomic_add_reduce(m=reshaped_x.size(0),
|
||||
n=output_size_per_partition,
|
||||
k=reshaped_x.size(1),
|
||||
device=input.device,
|
||||
dtype=input.dtype)
|
||||
use_atomic_add = should_use_atomic_add_reduce(
|
||||
m=reshaped_x.size(0),
|
||||
n=output_size_per_partition,
|
||||
k=reshaped_x.size(1),
|
||||
device=input.device,
|
||||
dtype=input.dtype,
|
||||
)
|
||||
|
||||
output = ops.gptq_marlin_gemm(reshaped_x,
|
||||
None,
|
||||
weight,
|
||||
bias,
|
||||
weight_scale,
|
||||
None,
|
||||
weight_zp,
|
||||
g_idx,
|
||||
g_idx_sort_indices,
|
||||
workspace,
|
||||
wtype,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=output_size_per_partition,
|
||||
size_k=input_size_per_partition,
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False)
|
||||
output = ops.gptq_marlin_gemm(
|
||||
reshaped_x,
|
||||
None,
|
||||
weight,
|
||||
bias,
|
||||
weight_scale,
|
||||
None,
|
||||
weight_zp,
|
||||
g_idx,
|
||||
g_idx_sort_indices,
|
||||
workspace,
|
||||
wtype,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=output_size_per_partition,
|
||||
size_k=input_size_per_partition,
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
)
|
||||
|
||||
return output.reshape(out_shape)
|
||||
|
||||
|
||||
def apply_awq_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
g_idx_sort_indices: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
g_idx_sort_indices: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = input.reshape(-1, input.shape[-1])
|
||||
out_shape = input.shape[:-1] + (output_size_per_partition, )
|
||||
out_shape = input.shape[:-1] + (output_size_per_partition,)
|
||||
|
||||
use_atomic_add = should_use_atomic_add_reduce(m=reshaped_x.size(0),
|
||||
n=output_size_per_partition,
|
||||
k=reshaped_x.size(1),
|
||||
device=input.device,
|
||||
dtype=input.dtype)
|
||||
use_atomic_add = should_use_atomic_add_reduce(
|
||||
m=reshaped_x.size(0),
|
||||
n=output_size_per_partition,
|
||||
k=reshaped_x.size(1),
|
||||
device=input.device,
|
||||
dtype=input.dtype,
|
||||
)
|
||||
|
||||
output = ops.gptq_marlin_gemm(reshaped_x,
|
||||
None,
|
||||
weight,
|
||||
bias,
|
||||
weight_scale,
|
||||
None,
|
||||
weight_zp,
|
||||
g_idx,
|
||||
g_idx_sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=output_size_per_partition,
|
||||
size_k=input_size_per_partition,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False)
|
||||
output = ops.gptq_marlin_gemm(
|
||||
reshaped_x,
|
||||
None,
|
||||
weight,
|
||||
bias,
|
||||
weight_scale,
|
||||
None,
|
||||
weight_zp,
|
||||
g_idx,
|
||||
g_idx_sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=output_size_per_partition,
|
||||
size_k=input_size_per_partition,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
)
|
||||
|
||||
return output.reshape(out_shape)
|
||||
|
||||
@@ -8,8 +8,12 @@ import torch
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT, marlin_make_workspace_new, marlin_permute_bias,
|
||||
marlin_permute_scales, should_use_atomic_add_reduce)
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
marlin_permute_scales,
|
||||
should_use_atomic_add_reduce,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -28,7 +32,8 @@ def nvfp4_marlin_process_scales(marlin_scales):
|
||||
"NVFP4 Marlin assumes the scales to be >=0, but has encountered "
|
||||
"negative scales. Accuracy will likely be degraded. This is "
|
||||
"because it changes the scales from FP8-S1E4M3 to a special "
|
||||
"FP8-S0E5M3 format to speedup the dequantization.")
|
||||
"FP8-S0E5M3 format to speedup the dequantization."
|
||||
)
|
||||
|
||||
# convert to half first, we would convert to fp8 later
|
||||
marlin_scales = marlin_scales.to(torch.half)
|
||||
@@ -36,11 +41,13 @@ def nvfp4_marlin_process_scales(marlin_scales):
|
||||
# 8 is the number of scale number using by one thread
|
||||
marlin_scales = marlin_scales.view(marlin_scales.size(0) // 2, 2, -1, 8)
|
||||
marlin_scales = marlin_scales.permute(0, 2, 1, 3).reshape(
|
||||
marlin_scales.size(0) * 2, -1)
|
||||
marlin_scales.size(0) * 2, -1
|
||||
)
|
||||
|
||||
# fit the layout of fp8 dequantization
|
||||
marlin_scales = marlin_scales.view(-1, 4)[:, [0, 2, 1, 3]].view(
|
||||
marlin_scales.size(0), -1)
|
||||
marlin_scales.size(0), -1
|
||||
)
|
||||
|
||||
# We assume that weight_scale (FP8-S1E4M3) is always greater
|
||||
# than or equal to 0. So we can convert
|
||||
@@ -60,11 +67,13 @@ def mxfp4_marlin_process_scales(marlin_scales):
|
||||
# 8 is the number of scale number using by one thread
|
||||
marlin_scales = marlin_scales.view(marlin_scales.size(0) // 2, 2, -1, 8)
|
||||
marlin_scales = marlin_scales.permute(0, 2, 1, 3).reshape(
|
||||
marlin_scales.size(0) * 2, -1)
|
||||
marlin_scales.size(0) * 2, -1
|
||||
)
|
||||
|
||||
# fit the layout of fp8 dequantization
|
||||
marlin_scales = marlin_scales.view(-1, 4)[:, [0, 2, 1, 3]].view(
|
||||
marlin_scales.size(0), -1)
|
||||
marlin_scales.size(0), -1
|
||||
)
|
||||
marlin_scales = marlin_scales.to(torch.float8_e8m0fnu)
|
||||
return marlin_scales
|
||||
|
||||
@@ -78,48 +87,49 @@ def nvfp4_marlin_process_global_scale(global_scale):
|
||||
target_exponent = 8
|
||||
# exponent_bias_fp16 = 2 ** 4 - 2 ** 1 = 14
|
||||
# exponent_bias_bf16 = 2 ** 7 - 2 ** 1 = 126
|
||||
exponent_bias = 2**(target_exponent - 1) - 2**(fp4_exponent - 1)
|
||||
return global_scale * (2.0**(exponent_bias - 7))
|
||||
exponent_bias = 2 ** (target_exponent - 1) - 2 ** (fp4_exponent - 1)
|
||||
return global_scale * (2.0 ** (exponent_bias - 7))
|
||||
|
||||
|
||||
def apply_fp4_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_scale_2: Optional[torch.Tensor],
|
||||
workspace: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_scale_2: Optional[torch.Tensor],
|
||||
workspace: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
) -> torch.Tensor:
|
||||
# For GPUs that lack FP4 hardware support, we can leverage the
|
||||
# Marlin kernel for fast weight-only FP4 quantization
|
||||
|
||||
reshaped_x = input.reshape(-1, input.shape[-1])
|
||||
out_shape = input.shape[:-1] + (size_n, )
|
||||
out_shape = input.shape[:-1] + (size_n,)
|
||||
|
||||
use_atomic_add = should_use_atomic_add_reduce(m=reshaped_x.size(0),
|
||||
n=size_n,
|
||||
k=size_k,
|
||||
device=input.device,
|
||||
dtype=input.dtype)
|
||||
use_atomic_add = should_use_atomic_add_reduce(
|
||||
m=reshaped_x.size(0), n=size_n, k=size_k, device=input.device, dtype=input.dtype
|
||||
)
|
||||
|
||||
output = ops.gptq_marlin_gemm(a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
global_scale=weight_scale_2,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce)
|
||||
output = ops.gptq_marlin_gemm(
|
||||
a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
global_scale=weight_scale_2,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
)
|
||||
|
||||
return output.reshape(out_shape)
|
||||
|
||||
@@ -129,7 +139,8 @@ def prepare_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
"Your GPU does not have native support for FP4 computation but "
|
||||
"FP4 quantization is being used. Weight-only FP4 compression will "
|
||||
"be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads.")
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
|
||||
is_nvfp4 = hasattr(layer, "weight_scale_2")
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
@@ -150,11 +161,13 @@ def prepare_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = layer.weight.view(torch.int32).T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
num_bits=4)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
num_bits=4,
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
|
||||
|
||||
# WEIGHT SCALES
|
||||
@@ -165,27 +178,23 @@ def prepare_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
weight_scale = weight_scale.view(torch.float8_e8m0fnu)
|
||||
|
||||
weight_scale = weight_scale.to(param_dtype)
|
||||
weight_scale = marlin_permute_scales(s=weight_scale,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
group_size=group_size)
|
||||
weight_scale = marlin_permute_scales(
|
||||
s=weight_scale, size_k=part_size_k, size_n=part_size_n, group_size=group_size
|
||||
)
|
||||
|
||||
if is_nvfp4:
|
||||
weight_scale = nvfp4_marlin_process_scales(weight_scale)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale,
|
||||
requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
weight_scale_2 = layer.weight_scale_2.to(param_dtype)
|
||||
weight_scale_2 = nvfp4_marlin_process_global_scale(weight_scale_2)
|
||||
layer.weight_scale_2 = torch.nn.Parameter(weight_scale_2,
|
||||
requires_grad=False)
|
||||
layer.weight_scale_2 = torch.nn.Parameter(weight_scale_2, requires_grad=False)
|
||||
else:
|
||||
weight_scale = mxfp4_marlin_process_scales(weight_scale)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale,
|
||||
requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
if hasattr(layer, "bias") and layer.bias is not None:
|
||||
assert layer.bias.shape == (part_size_n, )
|
||||
assert layer.bias.shape == (part_size_n,)
|
||||
bias = marlin_permute_bias(layer.bias)
|
||||
layer.bias = torch.nn.Parameter(bias, requires_grad=False)
|
||||
|
||||
@@ -197,7 +206,8 @@ def prepare_moe_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
"Your GPU does not have native support for FP4 computation but "
|
||||
"FP4 quantization is being used. Weight-only FP4 compression will "
|
||||
"be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads.")
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
|
||||
is_nvfp4 = hasattr(layer, "w13_weight_scale_2")
|
||||
group_size = 16 if is_nvfp4 else 32
|
||||
@@ -227,11 +237,9 @@ def prepare_moe_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
for i in range(e):
|
||||
qweight = weight[i].view(torch.int32).T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=4
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
|
||||
weight = torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
@@ -247,8 +255,7 @@ def prepare_moe_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
scales = scales.view(torch.float8_e8m0fnu)
|
||||
scales = scales.to(param_dtype)
|
||||
if is_nvfp4:
|
||||
global_scale = getattr(layer,
|
||||
name + "_weight_scale_2").to(param_dtype)
|
||||
global_scale = getattr(layer, name + "_weight_scale_2").to(param_dtype)
|
||||
|
||||
tensor_list = []
|
||||
if "w13" in name:
|
||||
@@ -259,10 +266,9 @@ def prepare_moe_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
for i in range(e):
|
||||
scale = scales[i].T
|
||||
|
||||
marlin_scales = marlin_permute_scales(s=scale,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size)
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scale, size_k=size_k, size_n=size_n, group_size=group_size
|
||||
)
|
||||
if is_nvfp4:
|
||||
marlin_scales = nvfp4_marlin_process_scales(marlin_scales)
|
||||
else:
|
||||
@@ -275,8 +281,7 @@ def prepare_moe_fp4_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
|
||||
if is_nvfp4:
|
||||
global_scale = nvfp4_marlin_process_global_scale(global_scale)
|
||||
global_scale = torch.nn.Parameter(global_scale,
|
||||
requires_grad=False)
|
||||
global_scale = torch.nn.Parameter(global_scale, requires_grad=False)
|
||||
setattr(layer, name + "_weight_scale_2", global_scale)
|
||||
|
||||
# BIAS
|
||||
@@ -306,26 +311,26 @@ def rand_marlin_weight_nvfp4_like(weight, group_size):
|
||||
global_scale = scales.max() / 448
|
||||
scales = (scales / global_scale).to(torch.float8_e4m3fn)
|
||||
|
||||
fp4_weight = torch.randint(0,
|
||||
256, (size_n, size_k // 2),
|
||||
dtype=torch.uint8,
|
||||
device=weight.device)
|
||||
fp4_weight_part_1 = ((fp4_weight & 0b10000000) |
|
||||
((fp4_weight & 0b01110000) >> 2))
|
||||
fp4_weight = torch.randint(
|
||||
0, 256, (size_n, size_k // 2), dtype=torch.uint8, device=weight.device
|
||||
)
|
||||
fp4_weight_part_1 = (fp4_weight & 0b10000000) | ((fp4_weight & 0b01110000) >> 2)
|
||||
fp4_weight_part_1 = fp4_weight_part_1.view(torch.float8_e4m3fn)
|
||||
fp4_weight_part_1 = fp4_weight_part_1.to(weight.dtype) * (2**6)
|
||||
|
||||
fp4_weight2 = fp4_weight << 4
|
||||
fp4_weight_part_2 = ((fp4_weight2 & 0b10000000) |
|
||||
((fp4_weight2 & 0b01110000) >> 2))
|
||||
fp4_weight_part_2 = (fp4_weight2 & 0b10000000) | ((fp4_weight2 & 0b01110000) >> 2)
|
||||
fp4_weight_part_2 = fp4_weight_part_2.view(torch.float8_e4m3fn)
|
||||
fp4_weight_part_2 = fp4_weight_part_2.to(weight.dtype) * (2**6)
|
||||
|
||||
weight_ref = torch.cat(
|
||||
[fp4_weight_part_2.unsqueeze(2),
|
||||
fp4_weight_part_1.unsqueeze(2)], 2).view(size_n, size_k)
|
||||
weight_ref = weight_ref * global_scale.to(weight.dtype) * \
|
||||
scales.repeat_interleave(group_size, 1).to(weight.dtype)
|
||||
[fp4_weight_part_2.unsqueeze(2), fp4_weight_part_1.unsqueeze(2)], 2
|
||||
).view(size_n, size_k)
|
||||
weight_ref = (
|
||||
weight_ref
|
||||
* global_scale.to(weight.dtype)
|
||||
* scales.repeat_interleave(group_size, 1).to(weight.dtype)
|
||||
)
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=fp4_weight.view(torch.int32).T.contiguous(),
|
||||
@@ -335,10 +340,9 @@ def rand_marlin_weight_nvfp4_like(weight, group_size):
|
||||
num_bits=4,
|
||||
)
|
||||
|
||||
marlin_scales = marlin_permute_scales(s=scales.T.to(weight.dtype),
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size)
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales.T.to(weight.dtype), size_k=size_k, size_n=size_n, group_size=group_size
|
||||
)
|
||||
marlin_scales = nvfp4_marlin_process_scales(marlin_scales)
|
||||
|
||||
global_scale = nvfp4_marlin_process_global_scale(global_scale)
|
||||
@@ -351,32 +355,31 @@ def rand_marlin_weight_mxfp4_like(weight, group_size):
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
scales = torch.randint(100,
|
||||
125, (size_n, size_k // group_size),
|
||||
dtype=torch.uint8,
|
||||
device=weight.device)
|
||||
scales = torch.randint(
|
||||
100,
|
||||
125,
|
||||
(size_n, size_k // group_size),
|
||||
dtype=torch.uint8,
|
||||
device=weight.device,
|
||||
)
|
||||
scales = scales.view(torch.float8_e8m0fnu)
|
||||
|
||||
fp4_weight = torch.randint(0,
|
||||
256, (size_n, size_k // 2),
|
||||
dtype=torch.uint8,
|
||||
device=weight.device)
|
||||
fp4_weight_part_1 = ((fp4_weight & 0b10000000) |
|
||||
((fp4_weight & 0b01110000) >> 2))
|
||||
fp4_weight = torch.randint(
|
||||
0, 256, (size_n, size_k // 2), dtype=torch.uint8, device=weight.device
|
||||
)
|
||||
fp4_weight_part_1 = (fp4_weight & 0b10000000) | ((fp4_weight & 0b01110000) >> 2)
|
||||
fp4_weight_part_1 = fp4_weight_part_1.view(torch.float8_e4m3fn)
|
||||
fp4_weight_part_1 = fp4_weight_part_1.to(weight.dtype) * (2**6)
|
||||
|
||||
fp4_weight2 = fp4_weight << 4
|
||||
fp4_weight_part_2 = ((fp4_weight2 & 0b10000000) |
|
||||
((fp4_weight2 & 0b01110000) >> 2))
|
||||
fp4_weight_part_2 = (fp4_weight2 & 0b10000000) | ((fp4_weight2 & 0b01110000) >> 2)
|
||||
fp4_weight_part_2 = fp4_weight_part_2.view(torch.float8_e4m3fn)
|
||||
fp4_weight_part_2 = fp4_weight_part_2.to(weight.dtype) * (2**6)
|
||||
|
||||
weight_ref = torch.cat(
|
||||
[fp4_weight_part_2.unsqueeze(2),
|
||||
fp4_weight_part_1.unsqueeze(2)], 2).view(size_n, size_k)
|
||||
weight_ref = weight_ref * \
|
||||
scales.repeat_interleave(group_size, 1).to(weight.dtype)
|
||||
[fp4_weight_part_2.unsqueeze(2), fp4_weight_part_1.unsqueeze(2)], 2
|
||||
).view(size_n, size_k)
|
||||
weight_ref = weight_ref * scales.repeat_interleave(group_size, 1).to(weight.dtype)
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=fp4_weight.view(torch.int32).T.contiguous(),
|
||||
@@ -386,10 +389,9 @@ def rand_marlin_weight_mxfp4_like(weight, group_size):
|
||||
num_bits=4,
|
||||
)
|
||||
|
||||
marlin_scales = marlin_permute_scales(s=scales.T.to(weight.dtype),
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size)
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales.T.to(weight.dtype), size_k=size_k, size_n=size_n, group_size=group_size
|
||||
)
|
||||
|
||||
marlin_scales = mxfp4_marlin_process_scales(marlin_scales)
|
||||
|
||||
|
||||
@@ -8,8 +8,12 @@ import torch
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT, marlin_make_workspace_new, marlin_permute_bias,
|
||||
marlin_permute_scales, should_use_atomic_add_reduce)
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
marlin_permute_scales,
|
||||
should_use_atomic_add_reduce,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -28,60 +32,63 @@ def fp8_fused_exponent_bias_into_scales(scales):
|
||||
target_exponent = 8
|
||||
# exponent_bias_fp16 = 2 ** 4 - 2 ** 3 = 8
|
||||
# exponent_bias_bf16 = 2 ** 7 - 2 ** 3 = 120
|
||||
exponent_bias = 2**(target_exponent - 1) - 2**(fp8_exponent - 1)
|
||||
exponent_bias = 2 ** (target_exponent - 1) - 2 ** (fp8_exponent - 1)
|
||||
s = torch.ones_like(scales) * 2
|
||||
s = s**exponent_bias
|
||||
return scales * s
|
||||
|
||||
|
||||
def apply_fp8_marlin_linear(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: Optional[torch.Tensor],
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT) -> torch.Tensor:
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
bias: Optional[torch.Tensor],
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
) -> torch.Tensor:
|
||||
# For GPUs that lack FP8 hardware support, we can leverage the
|
||||
# Marlin kernel for fast weight-only FP8 quantization
|
||||
|
||||
reshaped_x = input.reshape(-1, input.shape[-1])
|
||||
out_shape = input.shape[:-1] + (size_n, )
|
||||
out_shape = input.shape[:-1] + (size_n,)
|
||||
|
||||
use_atomic_add = should_use_atomic_add_reduce(m=reshaped_x.size(0),
|
||||
n=size_n,
|
||||
k=size_k,
|
||||
device=input.device,
|
||||
dtype=input.dtype)
|
||||
use_atomic_add = should_use_atomic_add_reduce(
|
||||
m=reshaped_x.size(0), n=size_n, k=size_k, device=input.device, dtype=input.dtype
|
||||
)
|
||||
|
||||
output = ops.gptq_marlin_gemm(a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce)
|
||||
output = ops.gptq_marlin_gemm(
|
||||
a=reshaped_x,
|
||||
c=None,
|
||||
b_q_weight=weight,
|
||||
b_bias=bias,
|
||||
b_scales=weight_scale,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
)
|
||||
|
||||
return output.reshape(out_shape)
|
||||
|
||||
|
||||
def prepare_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
size_k_first: bool = True) -> None:
|
||||
def prepare_fp8_layer_for_marlin(
|
||||
layer: torch.nn.Module, size_k_first: bool = True
|
||||
) -> None:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP8 computation but "
|
||||
"FP8 quantization is being used. Weight-only FP8 compression will "
|
||||
"be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads.")
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
|
||||
part_size_n = layer.output_size_per_partition
|
||||
part_size_k = layer.input_size_per_partition
|
||||
@@ -104,11 +111,13 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
if not size_k_first:
|
||||
qweight = qweight.T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
num_bits=8)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
num_bits=8,
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False)
|
||||
|
||||
# WEIGHT SCALES
|
||||
@@ -151,26 +160,27 @@ def prepare_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
# size_n may not divisible by block_size[0]
|
||||
scales = scales[:, :part_size_n]
|
||||
|
||||
marlin_scales = marlin_permute_scales(s=scales,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n,
|
||||
group_size=group_size)
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales, size_k=part_size_k, size_n=part_size_n, group_size=group_size
|
||||
)
|
||||
marlin_scales = fp8_fused_exponent_bias_into_scales(marlin_scales)
|
||||
layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False)
|
||||
|
||||
if hasattr(layer, "bias") and layer.bias is not None:
|
||||
assert layer.bias.shape == (part_size_n, )
|
||||
assert layer.bias.shape == (part_size_n,)
|
||||
bias = marlin_permute_bias(layer.bias)
|
||||
layer.bias = torch.nn.Parameter(bias, requires_grad=False)
|
||||
|
||||
|
||||
def prepare_moe_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
size_k_first: bool = True) -> None:
|
||||
def prepare_moe_fp8_layer_for_marlin(
|
||||
layer: torch.nn.Module, size_k_first: bool = True
|
||||
) -> None:
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP8 computation but "
|
||||
"FP8 quantization is being used. Weight-only FP8 compression will "
|
||||
"be used leveraging the Marlin kernel. This may degrade "
|
||||
"performance for compute-heavy workloads.")
|
||||
"performance for compute-heavy workloads."
|
||||
)
|
||||
|
||||
e = layer.num_experts
|
||||
k = layer.hidden_size
|
||||
@@ -202,11 +212,9 @@ def prepare_moe_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
if not size_k_first:
|
||||
qweight = qweight.T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=8)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=8
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
|
||||
weight = torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
@@ -265,10 +273,9 @@ def prepare_moe_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
scales = scales[..., :size_n].contiguous()
|
||||
|
||||
for i in range(e):
|
||||
marlin_scales = marlin_permute_scales(s=scales[i],
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size)
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size
|
||||
)
|
||||
tensor_list.append(marlin_scales)
|
||||
|
||||
scales = torch.cat([x.unsqueeze(0) for x in tensor_list], 0)
|
||||
@@ -295,8 +302,9 @@ def prepare_moe_fp8_layer_for_marlin(layer: torch.nn.Module,
|
||||
setattr(layer, name, bias)
|
||||
|
||||
|
||||
def pack_fp8_to_int32(fp8_tensor: torch.Tensor,
|
||||
size_k_first: bool = True) -> torch.Tensor:
|
||||
def pack_fp8_to_int32(
|
||||
fp8_tensor: torch.Tensor, size_k_first: bool = True
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Repack FP8 weights to gptq format (packed int32 elements)
|
||||
"""
|
||||
@@ -335,10 +343,9 @@ def marlin_quant_fp8_torch(weight, group_size):
|
||||
num_bits=8,
|
||||
)
|
||||
|
||||
marlin_scales = marlin_permute_scales(s=scales.T,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
group_size=group_size)
|
||||
marlin_scales = marlin_permute_scales(
|
||||
s=scales.T, size_k=size_k, size_n=size_n, group_size=group_size
|
||||
)
|
||||
|
||||
marlin_scales = fp8_fused_exponent_bias_into_scales(marlin_scales)
|
||||
|
||||
|
||||
@@ -9,24 +9,26 @@ import torch
|
||||
|
||||
from vllm.scalar_type import ScalarType
|
||||
|
||||
from .marlin_utils import (GPTQ_MARLIN_TILE, marlin_permute_scales,
|
||||
marlin_zero_points)
|
||||
from .quant_utils import (get_pack_factor, gptq_quantize_weights,
|
||||
quantize_weights, sort_weights)
|
||||
from .marlin_utils import GPTQ_MARLIN_TILE, marlin_permute_scales, marlin_zero_points
|
||||
from .quant_utils import (
|
||||
get_pack_factor,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
sort_weights,
|
||||
)
|
||||
|
||||
|
||||
class MarlinWorkspace:
|
||||
|
||||
def __init__(self, out_features, min_thread_n, max_parallel):
|
||||
assert (out_features % min_thread_n == 0), (
|
||||
assert out_features % min_thread_n == 0, (
|
||||
"out_features = {} is indivisible by min_thread_n = {}".format(
|
||||
out_features, min_thread_n))
|
||||
out_features, min_thread_n
|
||||
)
|
||||
)
|
||||
|
||||
max_workspace_size = ((out_features // min_thread_n) * max_parallel)
|
||||
max_workspace_size = (out_features // min_thread_n) * max_parallel
|
||||
|
||||
self.scratch = torch.zeros(max_workspace_size,
|
||||
dtype=torch.int,
|
||||
device="cuda")
|
||||
self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda")
|
||||
|
||||
|
||||
def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE):
|
||||
@@ -54,8 +56,7 @@ def marlin_weights(q_w, size_k, size_n, num_bits, perm):
|
||||
|
||||
q_w = q_w.cpu().numpy().astype(np.uint32)
|
||||
|
||||
q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor),
|
||||
dtype=np.uint32)
|
||||
q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32)
|
||||
for i in range(pack_factor):
|
||||
q_packed |= q_w[:, i::pack_factor] << num_bits * i
|
||||
|
||||
@@ -71,10 +72,10 @@ def get_weight_perm(num_bits: int):
|
||||
col = i // 4
|
||||
for block in [0, 1]:
|
||||
for row in [
|
||||
2 * (i % 4),
|
||||
2 * (i % 4) + 1,
|
||||
2 * (i % 4 + 4),
|
||||
2 * (i % 4 + 4) + 1,
|
||||
2 * (i % 4),
|
||||
2 * (i % 4) + 1,
|
||||
2 * (i % 4 + 4),
|
||||
2 * (i % 4 + 4) + 1,
|
||||
]:
|
||||
perm1.append(16 * row + col + 8 * block)
|
||||
for j in range(4):
|
||||
@@ -94,11 +95,13 @@ def get_weight_perm(num_bits: int):
|
||||
return perm
|
||||
|
||||
|
||||
def marlin_quantize(w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: Optional[torch.Tensor] = None):
|
||||
def marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: Optional[torch.Tensor] = None,
|
||||
):
|
||||
size_k, size_n = w.shape
|
||||
num_bits = quant_type.size_bits
|
||||
|
||||
@@ -109,7 +112,8 @@ def marlin_quantize(w: torch.Tensor,
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
w, quant_type, group_size, act_order, test_perm)
|
||||
w, quant_type, group_size, act_order, test_perm
|
||||
)
|
||||
|
||||
# For act_order, sort the "weights" and "g_idx" so that group ids are
|
||||
# increasing
|
||||
@@ -130,8 +134,7 @@ def marlin_quantize(w: torch.Tensor,
|
||||
return res_list
|
||||
|
||||
|
||||
def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType,
|
||||
group_size: int):
|
||||
def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int):
|
||||
size_k, size_n = w.shape
|
||||
|
||||
# Normalize group_size
|
||||
@@ -144,18 +147,13 @@ def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType,
|
||||
num_groups = size_k // group_size
|
||||
|
||||
# Quantize with zp
|
||||
w_ref, q_w, s, zp = quantize_weights(w,
|
||||
quant_type,
|
||||
group_size,
|
||||
zero_points=True)
|
||||
w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True)
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(quant_type.size_bits)
|
||||
marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits,
|
||||
weight_perm)
|
||||
marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits, weight_perm)
|
||||
marlin_s = marlin_permute_scales(s, size_k, size_n, group_size)
|
||||
marlin_zp = marlin_zero_points(zp, num_groups, size_n,
|
||||
quant_type.size_bits)
|
||||
marlin_zp = marlin_zero_points(zp, num_groups, size_n, quant_type.size_bits)
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp]
|
||||
|
||||
@@ -26,8 +26,7 @@ from .quant_utils import gptq_quantize_weights
|
||||
# matrix elements into reordered metadata matrix elements (or,
|
||||
# equivalently, for gathering reordered metadata matrix element back
|
||||
# into metadata matrix elements).
|
||||
def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype,
|
||||
device):
|
||||
def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype, device):
|
||||
dst_rows = torch.arange(0, m, device=device)[:, None].repeat(1, meta_ncols)
|
||||
dst_cols = torch.arange(0, meta_ncols, device=device).repeat(m, 1)
|
||||
|
||||
@@ -35,9 +34,13 @@ def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype,
|
||||
group_x = 64
|
||||
group_y = 32 if meta_dtype.itemsize == 2 else 16
|
||||
|
||||
dst_rows = (dst_rows // group_x * group_x + (dst_rows % 2) * 2 +
|
||||
(dst_rows % 8) // 4 + ((dst_rows % group_y) % 4) // 2 * 32 +
|
||||
((dst_rows % group_x) // 8) * 4)
|
||||
dst_rows = (
|
||||
dst_rows // group_x * group_x
|
||||
+ (dst_rows % 2) * 2
|
||||
+ (dst_rows % 8) // 4
|
||||
+ ((dst_rows % group_y) % 4) // 2 * 32
|
||||
+ ((dst_rows % group_x) // 8) * 4
|
||||
)
|
||||
|
||||
topright = ((dst_rows % 2 == 0) & (dst_cols % 2 == 1)).to(torch.int8)
|
||||
bottomleft = ((dst_rows % 2 == 1) & (dst_cols % 2 == 0)).to(torch.int8)
|
||||
@@ -50,8 +53,7 @@ def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype,
|
||||
interleave = 2
|
||||
cols_maj = dst_cols // interleave
|
||||
cols_min = dst_cols % interleave
|
||||
return (cols_maj * m * interleave + dst_rows * interleave +
|
||||
cols_min).view(-1)
|
||||
return (cols_maj * m * interleave + dst_rows * interleave + cols_min).view(-1)
|
||||
|
||||
|
||||
# This function converts dense matrix into sparse semi-structured
|
||||
@@ -75,17 +77,18 @@ def sparse_semi_structured_from_dense_cutlass(dense):
|
||||
raise RuntimeError(f"Invalid datatype {dense.dtype} of dense matrix")
|
||||
quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4
|
||||
if quadbits_per_meta_elem not in (4, 8):
|
||||
raise RuntimeError(
|
||||
"Invalid number of elements per meta element calculated")
|
||||
raise RuntimeError("Invalid number of elements per meta element calculated")
|
||||
|
||||
if meta_dtype == torch.int32:
|
||||
if m % 16 != 0:
|
||||
raise RuntimeError(
|
||||
f"Number of rows of dense matrix {m} must be divisible by 16")
|
||||
f"Number of rows of dense matrix {m} must be divisible by 16"
|
||||
)
|
||||
else:
|
||||
if m % 32 != 0:
|
||||
raise RuntimeError(
|
||||
f"Number of rows of dense matrix {m} must be divisible by 32")
|
||||
f"Number of rows of dense matrix {m} must be divisible by 32"
|
||||
)
|
||||
if k % (4 * quadbits_per_meta_elem) != 0:
|
||||
raise RuntimeError(
|
||||
f"Number of columns of dense matrix {k} must be divisible by {4 * quadbits_per_meta_elem}" # noqa: E501
|
||||
@@ -146,40 +149,39 @@ def sparse_semi_structured_from_dense_cutlass(dense):
|
||||
idxs1 = bit2 | (bit3.to(torch.int64) << 1)
|
||||
|
||||
if dense.dtype != torch.float:
|
||||
sparse0 = dense_4.gather(
|
||||
-1, idxs0.unsqueeze(-1)) # type: ignore[possibly-undefined]
|
||||
sparse0 = dense_4.gather(-1, idxs0.unsqueeze(-1)) # type: ignore[possibly-undefined]
|
||||
sparse1 = dense_4.gather(-1, idxs1.unsqueeze(-1))
|
||||
sparse = torch.stack((sparse0, sparse1), dim=-1).view(m, k // 2)
|
||||
else:
|
||||
sparse = dense_2.gather(-1,
|
||||
idxs0.unsqueeze(-1) // 2).view(
|
||||
m,
|
||||
k // 2) # type: ignore[possibly-undefined]
|
||||
sparse = dense_2.gather(-1, idxs0.unsqueeze(-1) // 2).view(m, k // 2) # type: ignore[possibly-undefined]
|
||||
|
||||
meta_4 = idxs0 | (idxs1 << 2)
|
||||
meta_n = meta_4.view(
|
||||
(-1, meta_ncols, quadbits_per_meta_elem)).to(meta_dtype)
|
||||
meta_n = meta_4.view((-1, meta_ncols, quadbits_per_meta_elem)).to(meta_dtype)
|
||||
|
||||
if quadbits_per_meta_elem == 4:
|
||||
meta = (meta_n[:, :, 0]
|
||||
| (meta_n[:, :, 1] << 4)
|
||||
| (meta_n[:, :, 2] << 8)
|
||||
| (meta_n[:, :, 3] << 12))
|
||||
meta = (
|
||||
meta_n[:, :, 0]
|
||||
| (meta_n[:, :, 1] << 4)
|
||||
| (meta_n[:, :, 2] << 8)
|
||||
| (meta_n[:, :, 3] << 12)
|
||||
)
|
||||
elif quadbits_per_meta_elem == 8:
|
||||
meta = (meta_n[:, :, 0]
|
||||
| (meta_n[:, :, 1] << 4)
|
||||
| (meta_n[:, :, 2] << 8)
|
||||
| (meta_n[:, :, 3] << 12)
|
||||
| (meta_n[:, :, 4] << 16)
|
||||
| (meta_n[:, :, 5] << 20)
|
||||
| (meta_n[:, :, 6] << 24)
|
||||
| (meta_n[:, :, 7] << 28))
|
||||
meta = (
|
||||
meta_n[:, :, 0]
|
||||
| (meta_n[:, :, 1] << 4)
|
||||
| (meta_n[:, :, 2] << 8)
|
||||
| (meta_n[:, :, 3] << 12)
|
||||
| (meta_n[:, :, 4] << 16)
|
||||
| (meta_n[:, :, 5] << 20)
|
||||
| (meta_n[:, :, 6] << 24)
|
||||
| (meta_n[:, :, 7] << 28)
|
||||
)
|
||||
|
||||
# Reorder meta tensor elements.
|
||||
meta_reordered = meta.new_empty(
|
||||
(m * meta_ncols, )) # type: ignore[possibly-undefined]
|
||||
meta_reordered = meta.new_empty((m * meta_ncols,)) # type: ignore[possibly-undefined]
|
||||
meta_offsets = _calculate_meta_reordering_scatter_offsets(
|
||||
m, meta_ncols, meta_dtype, device)
|
||||
m, meta_ncols, meta_dtype, device
|
||||
)
|
||||
meta_reordered.scatter_(0, meta_offsets, meta.view(-1))
|
||||
|
||||
return (sparse, meta_reordered.view(m, meta_ncols))
|
||||
@@ -222,13 +224,14 @@ def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered):
|
||||
if meta_ncols * ksparse * quadbits_per_meta_elem != 2 * k:
|
||||
raise RuntimeError(
|
||||
f"Number of columns of sparse matrix {k} different from the {meta_ncols * ksparse * quadbits_per_meta_elem // 2}, " # noqa: E501
|
||||
"expected according to the number of columns of meta matrix")
|
||||
"expected according to the number of columns of meta matrix"
|
||||
)
|
||||
|
||||
# Undo meta tensor elements reordering.
|
||||
meta_offsets = _calculate_meta_reordering_scatter_offsets(
|
||||
m, meta_ncols, meta_dtype, device)
|
||||
meta = torch.gather(meta_reordered.view(-1), 0,
|
||||
meta_offsets).view(m, meta_ncols)
|
||||
m, meta_ncols, meta_dtype, device
|
||||
)
|
||||
meta = torch.gather(meta_reordered.view(-1), 0, meta_offsets).view(m, meta_ncols)
|
||||
|
||||
# Unpack sparse tensor back to original dense tensor, using
|
||||
# information provided by meta tensor. Note that torch.float
|
||||
@@ -270,16 +273,17 @@ def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered):
|
||||
meta_2[:, :, 15] = (meta >> 30) & 0b11
|
||||
|
||||
dense_offsets = meta_2.view(-1) + (
|
||||
torch.arange(0, 2 * m * k // ksparse, device=device) * 4).view(
|
||||
-1, 1).repeat(1, 2).view(-1)
|
||||
torch.arange(0, 2 * m * k // ksparse, device=device) * 4
|
||||
).view(-1, 1).repeat(1, 2).view(-1)
|
||||
|
||||
dense = torch.zeros((m * 2 * k, ), dtype=sparse.dtype, device=device)
|
||||
dense = torch.zeros((m * 2 * k,), dtype=sparse.dtype, device=device)
|
||||
if sparse.dtype != torch.float:
|
||||
# dense.scatter_(0, dense_offsets, sparse.view(-1))
|
||||
dense.scatter_(0, dense_offsets, sparse.reshape(-1))
|
||||
else:
|
||||
dense.view(torch.half).scatter_(0, dense_offsets,
|
||||
sparse.view(torch.half).view(-1))
|
||||
dense.view(torch.half).scatter_(
|
||||
0, dense_offsets, sparse.view(torch.half).view(-1)
|
||||
)
|
||||
|
||||
return dense.view(m, 2 * k)
|
||||
|
||||
@@ -287,8 +291,8 @@ def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered):
|
||||
def mask_creator(tensor):
|
||||
"""
|
||||
Class for creating N:M sparsity masks.
|
||||
Masks will be created using the N:M ratio, where for every block of
|
||||
M weights, N will be pruned based on ranked weight value. Each mask
|
||||
Masks will be created using the N:M ratio, where for every block of
|
||||
M weights, N will be pruned based on ranked weight value. Each mask
|
||||
will correspond to the given tensor.
|
||||
|
||||
:param N: The number of weights in a group to keep
|
||||
@@ -301,14 +305,14 @@ def mask_creator(tensor):
|
||||
# for i, tensor in enumerate(tensors):
|
||||
if tensor.numel() % M != 0:
|
||||
raise ValueError(
|
||||
f"Tensor of size {tensor.shape} can't be evenly divided into "
|
||||
f"{M} groups")
|
||||
f"Tensor of size {tensor.shape} can't be evenly divided into {M} groups"
|
||||
)
|
||||
|
||||
num_groups = tensor.numel() // M
|
||||
|
||||
# N:M sparsity for linear layers
|
||||
tensor_temp = tensor.detach().abs().reshape(num_groups, M)
|
||||
index = torch.argsort(tensor_temp, dim=1)[:, :int(M - N)]
|
||||
index = torch.argsort(tensor_temp, dim=1)[:, : int(M - N)]
|
||||
|
||||
w_b = torch.ones(tensor_temp.shape, device=tensor_temp.device)
|
||||
mask = w_b.scatter_(dim=1, index=index, value=0).reshape(tensor.shape)
|
||||
@@ -342,7 +346,7 @@ def check_24(w, num_rows_to_sample=50, _verbose=False):
|
||||
for i in sampled_row_idxs:
|
||||
for j in range(0, num_cols - BLOCK_SIZE, BLOCK_SIZE):
|
||||
total_segments += 1
|
||||
block = w[i, j:j + BLOCK_SIZE]
|
||||
block = w[i, j : j + BLOCK_SIZE]
|
||||
num_nonzero = torch.count_nonzero(block)
|
||||
if num_nonzero > MAX_NON_ZEROS:
|
||||
print("i = {} j = {} block = {}".format(i, j, block))
|
||||
@@ -359,8 +363,7 @@ def compress_quantized_24_weight(q_24, size_k, size_n, wtype: ScalarType):
|
||||
|
||||
# Compress
|
||||
q_24_no_zp = q_24_no_zp.t().contiguous()
|
||||
q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(
|
||||
q_24_no_zp)
|
||||
q_24_no_zp_comp, meta = sparse_semi_structured_from_dense_cutlass(q_24_no_zp)
|
||||
q_24_no_zp_comp = q_24_no_zp_comp.t().contiguous()
|
||||
|
||||
# Restore bias
|
||||
@@ -390,13 +393,12 @@ def get_weight_perm_24(num_bits: int):
|
||||
col_o = col // 2
|
||||
for block in [0, 1]:
|
||||
for row in [
|
||||
2 * (i % 4),
|
||||
2 * (i % 4) + 1,
|
||||
2 * (i % 4 + 4),
|
||||
2 * (i % 4 + 4) + 1,
|
||||
2 * (i % 4),
|
||||
2 * (i % 4) + 1,
|
||||
2 * (i % 4 + 4),
|
||||
2 * (i % 4 + 4) + 1,
|
||||
]:
|
||||
perm1.append(16 * row + col_o * 256 + 8 * (col % 2) +
|
||||
4 * block)
|
||||
perm1.append(16 * row + col_o * 256 + 8 * (col % 2) + 4 * block)
|
||||
for j in range(4):
|
||||
perm_list.extend([p + 1 * j for p in perm1])
|
||||
perm = numpy.array(perm_list)
|
||||
@@ -413,9 +415,9 @@ def get_weight_perm_24(num_bits: int):
|
||||
return perm
|
||||
|
||||
|
||||
def marlin_permute_scales_24(s: torch.Tensor, size_k: int, size_n: int,
|
||||
group_size: int) -> torch.Tensor:
|
||||
|
||||
def marlin_permute_scales_24(
|
||||
s: torch.Tensor, size_k: int, size_n: int, group_size: int
|
||||
) -> torch.Tensor:
|
||||
scale_perm, scale_perm_single = get_scale_perms_24()
|
||||
if group_size < size_k and group_size != -1:
|
||||
s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
|
||||
@@ -443,17 +445,18 @@ def marlin_24_quantize(
|
||||
|
||||
# Quantize
|
||||
w_24_ref, q_w_24, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
w_24, quant_type, group_size, act_order=False)
|
||||
w_24, quant_type, group_size, act_order=False
|
||||
)
|
||||
|
||||
# Compress quantized weight
|
||||
q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n,
|
||||
quant_type)
|
||||
q_w_24_comp, meta = compress_quantized_24_weight(q_w_24, size_k, size_n, quant_type)
|
||||
size_k_comp = size_k // 2
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm_24(quant_type.size_bits)
|
||||
marlin_24_q_w_comp = marlin_weights(q_w_24_comp, size_k_comp, size_n,
|
||||
quant_type.size_bits, weight_perm)
|
||||
marlin_24_q_w_comp = marlin_weights(
|
||||
q_w_24_comp, size_k_comp, size_n, quant_type.size_bits, weight_perm
|
||||
)
|
||||
marlin_24_s = marlin_permute_scales_24(s, size_k, size_n, group_size)
|
||||
|
||||
# Create result
|
||||
|
||||
@@ -14,8 +14,7 @@ OCP_MX_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
""" weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel
|
||||
"""
|
||||
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
from triton_kernels.numerics import InFlexData
|
||||
from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor
|
||||
@@ -25,30 +24,38 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
value_layout_opts: dict[str, Any] = {}
|
||||
scale_layout_opts: dict[str, Any] = {}
|
||||
|
||||
if (current_platform.is_cuda()
|
||||
and current_platform.is_device_capability(90)
|
||||
and not is_torch_equal_or_newer("2.8.1")):
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability(90)
|
||||
and not is_torch_equal_or_newer("2.8.1")
|
||||
):
|
||||
logger.warning_once(
|
||||
"Mxfp4 on hopper is running on torch < 2.8.1, "
|
||||
"this cause swizling to be disabled, which may "
|
||||
"cause performance degradation. Please upgrade to torch nightly")
|
||||
"cause performance degradation. Please upgrade to torch nightly"
|
||||
)
|
||||
value_layout = StridedLayout
|
||||
scale_layout = StridedLayout
|
||||
elif current_platform.is_rocm():
|
||||
from triton_kernels.tensor_details.layout import (GFX950MXScaleLayout,
|
||||
StridedLayout)
|
||||
from triton_kernels.tensor_details.layout import (
|
||||
GFX950MXScaleLayout,
|
||||
StridedLayout,
|
||||
)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
value_layout = StridedLayout
|
||||
scale_layout = GFX950MXScaleLayout if on_gfx950() else StridedLayout
|
||||
else:
|
||||
value_layout, value_layout_opts = \
|
||||
layout.make_default_matmul_mxfp4_w_layout(mx_axis=1)
|
||||
value_layout, value_layout_opts = layout.make_default_matmul_mxfp4_w_layout(
|
||||
mx_axis=1
|
||||
)
|
||||
scale_layout, scale_layout_opts = (
|
||||
layout.make_default_matmul_mxfp4_w_scale_layout(
|
||||
mx_axis=1, num_warps=num_warps))
|
||||
if current_platform.is_cuda() and \
|
||||
current_platform.is_device_capability(100):
|
||||
mx_axis=1, num_warps=num_warps
|
||||
)
|
||||
)
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability(100):
|
||||
constraints = {
|
||||
"is_persistent": True,
|
||||
"epilogue_subtile": 1,
|
||||
@@ -57,66 +64,83 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps):
|
||||
# transpose the tensor so that the quantization axis is on dim1
|
||||
quant_tensor = quant_tensor.transpose(-2, -1)
|
||||
scale = scale.transpose(-2, -1)
|
||||
quant_tensor = convert_layout(wrap_torch_tensor(quant_tensor, dtype=FP4),
|
||||
value_layout, **value_layout_opts)
|
||||
scale = convert_layout(wrap_torch_tensor(scale), scale_layout,
|
||||
**scale_layout_opts)
|
||||
quant_tensor = convert_layout(
|
||||
wrap_torch_tensor(quant_tensor, dtype=FP4), value_layout, **value_layout_opts
|
||||
)
|
||||
scale = convert_layout(wrap_torch_tensor(scale), scale_layout, **scale_layout_opts)
|
||||
return quant_tensor, InFlexData(), scale
|
||||
|
||||
|
||||
def _can_support_mxfp4(use_grouped_topk: bool = False,
|
||||
topk_group: Optional[int] = None,
|
||||
num_expert_group: Optional[int] = None,
|
||||
expert_map: Optional[torch.Tensor] = None,
|
||||
custom_routing_function: Optional[Callable] = None,
|
||||
e_score_correction_bias: Optional[torch.Tensor] = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
scoring_func: str = "softmax",
|
||||
activation: str = "swigluoai",
|
||||
expert_load_view: Optional[torch.Tensor] = None,
|
||||
logical_to_physical_map: Optional[torch.Tensor] = None,
|
||||
logical_replica_count: Optional[torch.Tensor] = None):
|
||||
return not (use_grouped_topk or topk_group or num_expert_group
|
||||
or custom_routing_function or e_score_correction_bias
|
||||
or apply_router_weight_on_input or scoring_func != "softmax"
|
||||
or activation != "swigluoai" or expert_load_view
|
||||
or logical_to_physical_map or logical_replica_count)
|
||||
def _can_support_mxfp4(
|
||||
use_grouped_topk: bool = False,
|
||||
topk_group: Optional[int] = None,
|
||||
num_expert_group: Optional[int] = None,
|
||||
expert_map: Optional[torch.Tensor] = None,
|
||||
custom_routing_function: Optional[Callable] = None,
|
||||
e_score_correction_bias: Optional[torch.Tensor] = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
scoring_func: str = "softmax",
|
||||
activation: str = "swigluoai",
|
||||
expert_load_view: Optional[torch.Tensor] = None,
|
||||
logical_to_physical_map: Optional[torch.Tensor] = None,
|
||||
logical_replica_count: Optional[torch.Tensor] = None,
|
||||
):
|
||||
return not (
|
||||
use_grouped_topk
|
||||
or topk_group
|
||||
or num_expert_group
|
||||
or custom_routing_function
|
||||
or e_score_correction_bias
|
||||
or apply_router_weight_on_input
|
||||
or scoring_func != "softmax"
|
||||
or activation != "swigluoai"
|
||||
or expert_load_view
|
||||
or logical_to_physical_map
|
||||
or logical_replica_count
|
||||
)
|
||||
|
||||
|
||||
def _dequant_mxfp4(x: torch.Tensor, scale: torch.Tensor,
|
||||
float_dtype: torch.dtype) -> torch.Tensor:
|
||||
def _dequant_mxfp4(
|
||||
x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
try:
|
||||
from quark.torch.kernel import mx
|
||||
except ImportError as err:
|
||||
raise ImportError("The package `amd-quark` is required to use "
|
||||
"MX-FP4 models. Please install it with `pip install "
|
||||
"amd-quark`.") from err
|
||||
raise ImportError(
|
||||
"The package `amd-quark` is required to use "
|
||||
"MX-FP4 models. Please install it with `pip install "
|
||||
"amd-quark`."
|
||||
) from err
|
||||
|
||||
return mx.dq_mxfp4(x, scale, float_dtype)
|
||||
|
||||
|
||||
def _dequant_mxfp4_fake(x: torch.Tensor, scale: torch.Tensor,
|
||||
float_dtype: torch.dtype) -> torch.Tensor:
|
||||
return torch.empty((*x.shape[:-1], x.shape[-1] * 2),
|
||||
dtype=float_dtype,
|
||||
device=x.device)
|
||||
def _dequant_mxfp4_fake(
|
||||
x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(*x.shape[:-1], x.shape[-1] * 2), dtype=float_dtype, device=x.device
|
||||
)
|
||||
|
||||
|
||||
def _quant_dequant_mxfp4(x: torch.Tensor,
|
||||
scale_calculation_mode: str = "even") -> torch.Tensor:
|
||||
def _quant_dequant_mxfp4(
|
||||
x: torch.Tensor, scale_calculation_mode: str = "even"
|
||||
) -> torch.Tensor:
|
||||
try:
|
||||
from quark.torch.kernel import mx
|
||||
except ImportError as err:
|
||||
raise ImportError("The package `amd-quark` is required to use "
|
||||
"MX-FP4 models. Please install it with `pip install "
|
||||
"amd-quark`.") from err
|
||||
raise ImportError(
|
||||
"The package `amd-quark` is required to use "
|
||||
"MX-FP4 models. Please install it with `pip install "
|
||||
"amd-quark`."
|
||||
) from err
|
||||
|
||||
return mx.qdq_mxfp4(x, scale_calculation_mode)
|
||||
|
||||
|
||||
def _quant_dequant_mxfp4_fake(x: torch.Tensor,
|
||||
scale_calculation_mode: str = "even"
|
||||
) -> torch.Tensor:
|
||||
def _quant_dequant_mxfp4_fake(
|
||||
x: torch.Tensor, scale_calculation_mode: str = "even"
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
|
||||
@@ -9,12 +9,13 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def mxfp8_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
try:
|
||||
from flashinfer import mxfp8_quantize
|
||||
except ImportError as err:
|
||||
raise ImportError("The package `flashinfer` is required to do "
|
||||
"MX-FP8 quantization. Please install it with" \
|
||||
"`pip install flashinfer`") from err
|
||||
raise ImportError(
|
||||
"The package `flashinfer` is required to do "
|
||||
"MX-FP8 quantization. Please install it with"
|
||||
"`pip install flashinfer`"
|
||||
) from err
|
||||
|
||||
return mxfp8_quantize(x, is_sf_swizzled_layout=False)
|
||||
|
||||
@@ -12,8 +12,9 @@ __all__ = [
|
||||
|
||||
FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
|
||||
|
||||
kE2M1ToFloat = torch.tensor([0., 0.5, 1., 1.5, 2., 3., 4., 6.],
|
||||
dtype=torch.float32)
|
||||
kE2M1ToFloat = torch.tensor(
|
||||
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
def break_fp4_bytes(a, dtype):
|
||||
@@ -45,12 +46,9 @@ def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size):
|
||||
return out[0:m, 0:k]
|
||||
|
||||
|
||||
def dequantize_to_dtype(tensor_fp4,
|
||||
tensor_sf,
|
||||
global_scale,
|
||||
dtype,
|
||||
device,
|
||||
block_size=16):
|
||||
def dequantize_to_dtype(
|
||||
tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16
|
||||
):
|
||||
"""Dequantize the fp4 tensor back to high precision."""
|
||||
# Two fp4 values are packed into one uint8.
|
||||
assert tensor_fp4.dtype == torch.uint8
|
||||
@@ -95,8 +93,7 @@ def ref_nvfp4_quant(x, global_scale, block_size):
|
||||
assert x.ndim == 2
|
||||
m, n = x.shape
|
||||
x = torch.reshape(x, (m, n // block_size, block_size))
|
||||
vec_max = torch.max(torch.abs(x), dim=-1,
|
||||
keepdim=True)[0].to(torch.float32)
|
||||
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
|
||||
scale = torch.clamp(scale, max=448, min=-448)
|
||||
scale = scale.to(torch.float8_e4m3fn).to(torch.float32)
|
||||
@@ -108,10 +105,13 @@ def ref_nvfp4_quant(x, global_scale, block_size):
|
||||
return cast_to_fp4(clipped_x), scale.squeeze(-1)
|
||||
|
||||
|
||||
def run_nvfp4_emulations(x: torch.Tensor, input_global_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale_swizzled: torch.Tensor,
|
||||
weight_global_scale: torch.Tensor):
|
||||
def run_nvfp4_emulations(
|
||||
x: torch.Tensor,
|
||||
input_global_scale: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale_swizzled: torch.Tensor,
|
||||
weight_global_scale: torch.Tensor,
|
||||
):
|
||||
group_size = 16
|
||||
x_m, x_k = x.shape
|
||||
output_dtype = x.dtype
|
||||
@@ -127,9 +127,14 @@ def run_nvfp4_emulations(x: torch.Tensor, input_global_scale: torch.Tensor,
|
||||
|
||||
# dequantize weight
|
||||
w_fp4 = weight.data.view(torch.uint8)
|
||||
w_dq = dequantize_to_dtype(w_fp4, weight_scale_swizzled.data,
|
||||
weight_global_scale, output_dtype, x.device,
|
||||
group_size)
|
||||
w_dq = dequantize_to_dtype(
|
||||
w_fp4,
|
||||
weight_scale_swizzled.data,
|
||||
weight_global_scale,
|
||||
output_dtype,
|
||||
x.device,
|
||||
group_size,
|
||||
)
|
||||
|
||||
# matmul
|
||||
out = torch.matmul(x_dq, w_dq.t())
|
||||
|
||||
@@ -5,11 +5,14 @@ from dataclasses import dataclass
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
is_flashinfer_fp4_cutlass_moe_available)
|
||||
is_flashinfer_fp4_cutlass_moe_available,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
is_fp4_marlin_supported)
|
||||
is_fp4_marlin_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
cutlass_fp4_supported)
|
||||
cutlass_fp4_supported,
|
||||
)
|
||||
|
||||
__all__ = ["detect_nvfp4_moe_support", "NvFp4Support"]
|
||||
|
||||
@@ -29,12 +32,12 @@ def detect_nvfp4_moe_support(class_name: str = "") -> NvFp4Support:
|
||||
"""Detect platform support for NV-FP4 fused-MoE path"""
|
||||
cutlass_supported = cutlass_fp4_supported()
|
||||
|
||||
allow_flashinfer = (cutlass_supported
|
||||
and is_flashinfer_fp4_cutlass_moe_available())
|
||||
allow_flashinfer = cutlass_supported and is_flashinfer_fp4_cutlass_moe_available()
|
||||
|
||||
if allow_flashinfer:
|
||||
_logger.info_once("Using FlashInfer kernels for %s.", class_name
|
||||
or "NVFP4 path")
|
||||
_logger.info_once(
|
||||
"Using FlashInfer kernels for %s.", class_name or "NVFP4 path"
|
||||
)
|
||||
else:
|
||||
if envs.VLLM_USE_FLASHINFER_MOE_FP4:
|
||||
_logger.warning_once(
|
||||
@@ -50,7 +53,8 @@ def detect_nvfp4_moe_support(class_name: str = "") -> NvFp4Support:
|
||||
else:
|
||||
raise ValueError(
|
||||
"Current platform does not support NVFP4 quantization. "
|
||||
"Please use Blackwell GPUs or enable FlashInfer.")
|
||||
"Please use Blackwell GPUs or enable FlashInfer."
|
||||
)
|
||||
|
||||
return NvFp4Support(
|
||||
cutlass_supported=cutlass_supported,
|
||||
|
||||
@@ -11,14 +11,15 @@ if TYPE_CHECKING:
|
||||
# 1. Create a global variable as a placeholder for the module
|
||||
_petit_kernel: Optional["ModuleType"] = None
|
||||
|
||||
_PETIT_INSTALL_MSG = ("Petit is not installed. Please install it with "
|
||||
"`pip install petit-kernel`.")
|
||||
_PETIT_INSTALL_MSG = (
|
||||
"Petit is not installed. Please install it with `pip install petit-kernel`."
|
||||
)
|
||||
|
||||
|
||||
def _import_petit_kernel() -> "ModuleType":
|
||||
"""
|
||||
A helper function to handle the lazy import.
|
||||
The first time this function is called, it will import the petit_kernel
|
||||
The first time this function is called, it will import the petit_kernel
|
||||
library and store it in the global _petit_kernel variable.
|
||||
Subsequent calls will return the already-loaded module directly.
|
||||
"""
|
||||
@@ -28,6 +29,7 @@ def _import_petit_kernel() -> "ModuleType":
|
||||
|
||||
try:
|
||||
import petit_kernel
|
||||
|
||||
_petit_kernel = petit_kernel
|
||||
return _petit_kernel
|
||||
except ImportError:
|
||||
@@ -41,14 +43,16 @@ _require_petit = _import_petit_kernel
|
||||
|
||||
|
||||
def _check_petit_nvfp4_supported(
|
||||
quant_method: str,
|
||||
group_size: Optional[int]) -> tuple[bool, Optional[str]]:
|
||||
quant_method: str, group_size: Optional[int]
|
||||
) -> tuple[bool, Optional[str]]:
|
||||
if quant_method != "NVFP4":
|
||||
return (
|
||||
False,
|
||||
("Petit currently only supports: NVFP4 quantizations in sglang. "
|
||||
"Please check the `hf_quant_config.json` file for your model's "
|
||||
"quant configuration."),
|
||||
(
|
||||
"Petit currently only supports: NVFP4 quantizations in sglang. "
|
||||
"Please check the `hf_quant_config.json` file for your model's "
|
||||
"quant configuration."
|
||||
),
|
||||
)
|
||||
if group_size is not None and group_size != 16:
|
||||
return (
|
||||
@@ -58,10 +62,8 @@ def _check_petit_nvfp4_supported(
|
||||
return (True, None)
|
||||
|
||||
|
||||
def verify_petit_nvfp4_supported(quant_method: str,
|
||||
group_size: Optional[int]) -> None:
|
||||
supported, error_msg = _check_petit_nvfp4_supported(
|
||||
quant_method, group_size)
|
||||
def verify_petit_nvfp4_supported(quant_method: str, group_size: Optional[int]) -> None:
|
||||
supported, error_msg = _check_petit_nvfp4_supported(quant_method, group_size)
|
||||
if not supported:
|
||||
assert error_msg is not None
|
||||
raise ValueError(error_msg)
|
||||
@@ -77,15 +79,15 @@ def prepare_nvfp4_layer_for_petit(layer: torch.nn.Module) -> None:
|
||||
qweight = layer.weight.view(torch.int32).contiguous()
|
||||
|
||||
# 3. Call functions through the imported module variable.
|
||||
petit_qweight = petit_kernel.repack_nvfp4(qweight,
|
||||
size_n=part_size_n,
|
||||
size_k=part_size_k)
|
||||
petit_qweight = petit_kernel.repack_nvfp4(
|
||||
qweight, size_n=part_size_n, size_k=part_size_k
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(petit_qweight, requires_grad=False)
|
||||
|
||||
# Permute scales
|
||||
weight_scale = petit_kernel.process_nvfp4_scales(scales=layer.weight_scale,
|
||||
size_k=part_size_k,
|
||||
size_n=part_size_n)
|
||||
weight_scale = petit_kernel.process_nvfp4_scales(
|
||||
scales=layer.weight_scale, size_k=part_size_k, size_n=part_size_n
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
|
||||
|
||||
@@ -102,7 +104,7 @@ def apply_petit_nvfp4_linear(
|
||||
petit_kernel = _import_petit_kernel()
|
||||
|
||||
reshaped_x = input.reshape(-1, input.shape[-1])
|
||||
out_shape = input.shape[:-1] + (size_n, )
|
||||
out_shape = input.shape[:-1] + (size_n,)
|
||||
|
||||
# TODO: Use auto-tuning to find the performant solution_id
|
||||
# Call the function via the module variable.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This file is used for /tests and /benchmarks"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
@@ -31,8 +32,8 @@ class GroupShape(_GroupShape):
|
||||
"""
|
||||
|
||||
# Aliases for common quantization group shapes
|
||||
PER_TENSOR: ClassVar['GroupShape']
|
||||
PER_TOKEN: ClassVar['GroupShape']
|
||||
PER_TENSOR: ClassVar["GroupShape"]
|
||||
PER_TOKEN: ClassVar["GroupShape"]
|
||||
|
||||
def is_per_tensor(self) -> bool:
|
||||
return self.row == -1 and self.col == -1
|
||||
@@ -56,18 +57,26 @@ class ScaleDesc:
|
||||
static: static scale if True, dynamic if False
|
||||
group_shape: group shape of the scale
|
||||
"""
|
||||
|
||||
dtype: torch.dtype
|
||||
static: bool
|
||||
group_shape: GroupShape
|
||||
|
||||
def __str__(self):
|
||||
group_shape = ('per_tensor'
|
||||
if self.group_shape == GroupShape.PER_TENSOR else
|
||||
('per_token' if self.group_shape == GroupShape.PER_TOKEN
|
||||
else str(self.group_shape)))
|
||||
group_shape = (
|
||||
"per_tensor"
|
||||
if self.group_shape == GroupShape.PER_TENSOR
|
||||
else (
|
||||
"per_token"
|
||||
if self.group_shape == GroupShape.PER_TOKEN
|
||||
else str(self.group_shape)
|
||||
)
|
||||
)
|
||||
|
||||
return (f"{fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"{'static' if self.static else 'dynamic'},{group_shape}")
|
||||
return (
|
||||
f"{fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"{'static' if self.static else 'dynamic'},{group_shape}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -79,6 +88,7 @@ class QuantKey:
|
||||
scale2: second-level scale descriptor
|
||||
symmetric: symmetric if True, asymmetric if False
|
||||
"""
|
||||
|
||||
dtype: torch.dtype
|
||||
scale: ScaleDesc
|
||||
scale2: Optional[ScaleDesc] = None
|
||||
@@ -86,9 +96,11 @@ class QuantKey:
|
||||
|
||||
def __str__(self):
|
||||
scale2_str = f"scale2({self.scale2})," if self.scale2 else ""
|
||||
return (f"QuantKey({fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"scale({self.scale}),{scale2_str}"
|
||||
f"{'a' if not self.symmetric else ''}symmetric)")
|
||||
return (
|
||||
f"QuantKey({fx.graph.dtype_abbrs[self.dtype]},"
|
||||
f"scale({self.scale}),{scale2_str}"
|
||||
f"{'a' if not self.symmetric else ''}symmetric)"
|
||||
)
|
||||
|
||||
|
||||
kStaticTensorScale = ScaleDesc(torch.float32, True, GroupShape.PER_TENSOR)
|
||||
@@ -101,16 +113,16 @@ kDynamicTokenScale = ScaleDesc(torch.float32, False, GroupShape.PER_TOKEN)
|
||||
kFp8DynamicTokenSym = QuantKey(FP8_DTYPE, kDynamicTokenScale, symmetric=True)
|
||||
|
||||
kNvfp4GroupScale = ScaleDesc(FP8_DTYPE, False, GroupShape(1, 16))
|
||||
kNvfp4Quant = QuantKey(FP4_DTYPE,
|
||||
scale=kNvfp4GroupScale,
|
||||
scale2=kStaticTensorScale)
|
||||
kNvfp4Quant = QuantKey(FP4_DTYPE, scale=kNvfp4GroupScale, scale2=kStaticTensorScale)
|
||||
|
||||
|
||||
# Normalize the group_shape to the full extent for any dims that are -1
|
||||
def _normalize_quant_group_shape(x: torch.Tensor, group_shape: GroupShape):
|
||||
# -1 means full extent
|
||||
return (group_shape[0] if group_shape[0] > 0 else x.shape[-2],
|
||||
group_shape[1] if group_shape[1] > 0 else x.shape[-1])
|
||||
return (
|
||||
group_shape[0] if group_shape[0] > 0 else x.shape[-2],
|
||||
group_shape[1] if group_shape[1] > 0 else x.shape[-1],
|
||||
)
|
||||
|
||||
|
||||
# Useful when treating N-dimensional group scaling as extended numpy-style
|
||||
@@ -131,9 +143,11 @@ def group_broadcast(t, shape):
|
||||
for i, s in enumerate(shape):
|
||||
if t.shape[i] != s and t.shape[i] != 1:
|
||||
assert s % t.shape[i] == 0
|
||||
t = t.unsqueeze(i + 1)\
|
||||
.expand(*t.shape[:i+1], s // t.shape[i], *t.shape[i+1:])\
|
||||
t = (
|
||||
t.unsqueeze(i + 1)
|
||||
.expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :])
|
||||
.flatten(i, i + 1)
|
||||
)
|
||||
return t
|
||||
|
||||
|
||||
@@ -151,9 +165,10 @@ def scaled_quantize(
|
||||
quant_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
group_shape = _normalize_quant_group_shape(x, group_shape)
|
||||
assert quant_dtype.is_floating_point, \
|
||||
"currently `scaled_quantize` only supports floating point dtypes " \
|
||||
assert quant_dtype.is_floating_point, (
|
||||
"currently `scaled_quantize` only supports floating point dtypes "
|
||||
"but could be extended to support other dtypes"
|
||||
)
|
||||
|
||||
finfo = torch.finfo(quant_dtype)
|
||||
|
||||
@@ -175,11 +190,13 @@ def scaled_quantize(
|
||||
|
||||
# Apply scale and convert form:
|
||||
# (BLK_M, BLK_N, BLOCK_SIZE_M * BLOCK_SIZE_N) to (M, N)
|
||||
x_scl_sat = (x_blkd_permd * scale.unsqueeze(-1))\
|
||||
.clamp(min=finfo.min, max=finfo.max)\
|
||||
.reshape(blk_m, blk_n, group_shape[0], group_shape[1])\
|
||||
.permute(0, 2, 1, 3)\
|
||||
x_scl_sat = (
|
||||
(x_blkd_permd * scale.unsqueeze(-1))
|
||||
.clamp(min=finfo.min, max=finfo.max)
|
||||
.reshape(blk_m, blk_n, group_shape[0], group_shape[1])
|
||||
.permute(0, 2, 1, 3)
|
||||
.reshape(x.shape)
|
||||
)
|
||||
|
||||
return x_scl_sat.to(quant_dtype).contiguous(), scale.float().reciprocal()
|
||||
|
||||
@@ -200,7 +217,8 @@ def scaled_dequantize(
|
||||
if group_shape is None:
|
||||
raise AssertionError(
|
||||
"if x_s is 1D tensor, group_shape must be provided otherwise "
|
||||
"its ambiguous which dimension to broadcast x_s to")
|
||||
"its ambiguous which dimension to broadcast x_s to"
|
||||
)
|
||||
# unsqueeze the scales for the dimension where we want to broadcast
|
||||
# across the full extent
|
||||
if group_shape[0] == x_q.shape[-2]:
|
||||
@@ -210,7 +228,8 @@ def scaled_dequantize(
|
||||
else:
|
||||
raise AssertionError(
|
||||
"if x_s is a vector we should be broadcasting it to the full "
|
||||
"extent of one of the dimensions")
|
||||
"extent of one of the dimensions"
|
||||
)
|
||||
|
||||
if group_shape is not None:
|
||||
assert x_s.shape[-1] == x_q.shape[-1] // group_shape[1]
|
||||
@@ -219,9 +238,9 @@ def scaled_dequantize(
|
||||
return (x_q.to(torch.float32) * x_s).to(out_dtype)
|
||||
|
||||
|
||||
def pack_quantized_values_into_int32(w_q: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
packed_dim: int = 0):
|
||||
def pack_quantized_values_into_int32(
|
||||
w_q: torch.Tensor, wtype: ScalarType, packed_dim: int = 0
|
||||
):
|
||||
# move dim to pack to the end
|
||||
perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
|
||||
inv_perm = tuple(perm.index(i) for i in range(len(perm)))
|
||||
@@ -241,9 +260,9 @@ def pack_quantized_values_into_int32(w_q: torch.Tensor,
|
||||
return res.permute(inv_perm)
|
||||
|
||||
|
||||
def unpack_quantized_values_into_int32(w_q: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
packed_dim: int = 0):
|
||||
def unpack_quantized_values_into_int32(
|
||||
w_q: torch.Tensor, wtype: ScalarType, packed_dim: int = 0
|
||||
):
|
||||
# move dim to pack to the end
|
||||
perm = (*[i for i in range(len(w_q.shape)) if i != packed_dim], packed_dim)
|
||||
inv_perm = tuple(perm.index(i) for i in range(len(perm)))
|
||||
@@ -265,7 +284,7 @@ def unpack_quantized_values_into_int32(w_q: torch.Tensor,
|
||||
def is_layer_skipped(
|
||||
prefix: str,
|
||||
ignored_layers: list[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({})
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
) -> bool:
|
||||
# prefix: model.layers.0.self_attn.q_proj
|
||||
# proj_name: q_proj
|
||||
@@ -291,12 +310,16 @@ def is_layer_skipped(
|
||||
raise ValueError(
|
||||
f"Detected some but not all shards of {prefix} "
|
||||
"are quantized. All shards of fused layers "
|
||||
"to have the same precision.")
|
||||
"to have the same precision."
|
||||
)
|
||||
elif "experts" in prefix:
|
||||
return any([
|
||||
prefix in layer_name for layer_name in ignored_layers
|
||||
if "experts" in layer_name
|
||||
])
|
||||
return any(
|
||||
[
|
||||
prefix in layer_name
|
||||
for layer_name in ignored_layers
|
||||
if "experts" in layer_name
|
||||
]
|
||||
)
|
||||
else:
|
||||
is_skipped = prefix in ignored_layers
|
||||
|
||||
@@ -309,16 +332,18 @@ def get_pack_factor(num_bits):
|
||||
return 32 // num_bits
|
||||
|
||||
|
||||
def permute_rows(q_w: torch.Tensor,
|
||||
w_ref: torch.Tensor,
|
||||
group_size: int,
|
||||
test_perm: Optional[torch.Tensor] = None):
|
||||
def permute_rows(
|
||||
q_w: torch.Tensor,
|
||||
w_ref: torch.Tensor,
|
||||
group_size: int,
|
||||
test_perm: Optional[torch.Tensor] = None,
|
||||
):
|
||||
assert q_w.shape == w_ref.shape
|
||||
|
||||
orig_device = q_w.device
|
||||
k_size, _ = q_w.shape
|
||||
|
||||
g_idx = torch.zeros((k_size, ), dtype=torch.int32)
|
||||
g_idx = torch.zeros((k_size,), dtype=torch.int32)
|
||||
for i in range(k_size):
|
||||
g_idx[i] = i // group_size
|
||||
|
||||
@@ -337,16 +362,20 @@ def permute_rows(q_w: torch.Tensor,
|
||||
)
|
||||
|
||||
|
||||
def quantize_weights(w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: Optional[int],
|
||||
zero_points: bool = False,
|
||||
ref_zero_points_after_scales: bool = False):
|
||||
assert quant_type.is_integer(), \
|
||||
def quantize_weights(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: Optional[int],
|
||||
zero_points: bool = False,
|
||||
ref_zero_points_after_scales: bool = False,
|
||||
):
|
||||
assert quant_type.is_integer(), (
|
||||
"Floating point quantization may work but has not been tested"
|
||||
assert not zero_points or group_size is not None, \
|
||||
"to have group zero points, group_size must be provided "\
|
||||
)
|
||||
assert not zero_points or group_size is not None, (
|
||||
"to have group zero points, group_size must be provided "
|
||||
"(-1 group_size is channelwise)"
|
||||
)
|
||||
|
||||
orig_device = w.device
|
||||
orig_type = w.dtype
|
||||
@@ -376,14 +405,16 @@ def quantize_weights(w: torch.Tensor,
|
||||
if zero_points:
|
||||
assert not quant_type.is_signed() and quant_type.max() > 0
|
||||
w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max()
|
||||
maybe_w_zp = torch.round(torch.abs(min_val / w_s)) \
|
||||
.clamp(min_q_val, max_q_val).int()
|
||||
maybe_w_zp = (
|
||||
torch.round(torch.abs(min_val / w_s)).clamp(min_q_val, max_q_val).int()
|
||||
)
|
||||
else:
|
||||
# If the bias is such that there are no possible negative/positive
|
||||
# values, set the max value to inf to avoid divide by 0
|
||||
w_s = torch.max(
|
||||
abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)),
|
||||
abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)))
|
||||
abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)),
|
||||
)
|
||||
|
||||
# Quantize
|
||||
w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0)
|
||||
@@ -430,19 +461,22 @@ SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128]
|
||||
SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128]
|
||||
|
||||
|
||||
def gptq_quantize_weights(w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: Optional[torch.Tensor] = None):
|
||||
def gptq_quantize_weights(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: Optional[torch.Tensor] = None,
|
||||
):
|
||||
size_k, _ = w.shape
|
||||
|
||||
assert w.is_floating_point(), "w must be float"
|
||||
assert quant_type in SUPPORTED_GPTQ_QUANT_TYPES, \
|
||||
assert quant_type in SUPPORTED_GPTQ_QUANT_TYPES, (
|
||||
f"Unsupported gptq type = {quant_type}"
|
||||
assert group_size in SUPPORTED_GROUP_SIZES + [
|
||||
size_k
|
||||
], f"Unsupported groupsize = {group_size}"
|
||||
)
|
||||
assert group_size in SUPPORTED_GROUP_SIZES + [size_k], (
|
||||
f"Unsupported groupsize = {group_size}"
|
||||
)
|
||||
|
||||
w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
|
||||
|
||||
@@ -450,13 +484,13 @@ def gptq_quantize_weights(w: torch.Tensor,
|
||||
g_idx = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
if act_order:
|
||||
assert (
|
||||
group_size < size_k
|
||||
), "For act_order, groupsize = {} must be less than size_k = {}".format(
|
||||
group_size, size_k)
|
||||
assert group_size < size_k, (
|
||||
"For act_order, groupsize = {} must be less than size_k = {}".format(
|
||||
group_size, size_k
|
||||
)
|
||||
)
|
||||
|
||||
w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size,
|
||||
test_perm)
|
||||
w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, test_perm)
|
||||
|
||||
return w_ref, w_q, w_s, g_idx, rand_perm
|
||||
|
||||
@@ -464,8 +498,7 @@ def gptq_quantize_weights(w: torch.Tensor,
|
||||
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
|
||||
orig_device = q_w.device
|
||||
|
||||
sort_indices = torch.argsort(g_idx).to(
|
||||
dtype=torch.int32) # Sort based on g_idx
|
||||
sort_indices = torch.argsort(g_idx).to(dtype=torch.int32) # Sort based on g_idx
|
||||
|
||||
g_idx = g_idx[sort_indices].contiguous()
|
||||
q_w = q_w[sort_indices, :].contiguous()
|
||||
@@ -535,10 +568,11 @@ def unpack_cols(
|
||||
):
|
||||
pack_factor = get_pack_factor(num_bits)
|
||||
assert size_n % pack_factor == 0
|
||||
assert packed_q_w.shape == (
|
||||
size_k, size_n // pack_factor
|
||||
), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format(
|
||||
packed_q_w.shape, size_k, size_n, pack_factor)
|
||||
assert packed_q_w.shape == (size_k, size_n // pack_factor), (
|
||||
"packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format(
|
||||
packed_q_w.shape, size_k, size_n, pack_factor
|
||||
)
|
||||
)
|
||||
|
||||
orig_device = packed_q_w.device
|
||||
|
||||
@@ -604,7 +638,8 @@ def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
assert scale.dtype == torch.float8_e4m3fn, (
|
||||
"swizzle_blockscale expects the input tensor to be in "
|
||||
"torch.float8_e4m3fn format.")
|
||||
"torch.float8_e4m3fn format."
|
||||
)
|
||||
|
||||
scale_ndim = scale.ndim
|
||||
if scale_ndim == 2:
|
||||
@@ -619,9 +654,9 @@ def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
|
||||
M_padded = _round_up(M, 128)
|
||||
K_padded = _round_up(K, 4)
|
||||
|
||||
padded = torch.zeros((B, M_padded, K_padded),
|
||||
dtype=scale.dtype,
|
||||
device=scale.device)
|
||||
padded = torch.zeros(
|
||||
(B, M_padded, K_padded), dtype=scale.dtype, device=scale.device
|
||||
)
|
||||
padded[:B, :M, :K] = scale
|
||||
|
||||
# Reshape / permute to the layout required by the kernel.
|
||||
|
||||
@@ -10,8 +10,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm.config import CompilationLevel, get_current_vllm_config
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import direct_register_custom_op
|
||||
from vllm.utils.flashinfer import flashinfer_scaled_fp8_mm, has_flashinfer
|
||||
@@ -24,9 +23,11 @@ TORCH_DEVICE_IDENTITY = None
|
||||
# torch._scaled_mm rowwise feature.
|
||||
# The condition is determined once as the operations
|
||||
# are time-consuming.
|
||||
USE_ROWWISE_TORCH_SCALED_MM = (current_platform.is_rocm() and version.parse(
|
||||
torch.__version__) >= version.parse("2.7")
|
||||
and current_platform.has_device_capability(94))
|
||||
USE_ROWWISE_TORCH_SCALED_MM = (
|
||||
current_platform.is_rocm()
|
||||
and version.parse(torch.__version__) >= version.parse("2.7")
|
||||
and current_platform.has_device_capability(94)
|
||||
)
|
||||
|
||||
|
||||
def sparse_cutlass_supported() -> bool:
|
||||
@@ -74,8 +75,8 @@ CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported()
|
||||
|
||||
|
||||
def per_tensor_dequantize(
|
||||
tensor: torch.Tensor, inv_scale: Union[float,
|
||||
torch.Tensor]) -> torch.Tensor:
|
||||
tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor]
|
||||
) -> torch.Tensor:
|
||||
fake_qweight = tensor.to(torch.float16)
|
||||
dq_weight = fake_qweight * inv_scale
|
||||
return dq_weight
|
||||
@@ -87,12 +88,12 @@ def all_close_1d(x: torch.Tensor) -> bool:
|
||||
|
||||
|
||||
def convert_to_channelwise(
|
||||
weight_scale: torch.Tensor,
|
||||
logical_widths: list[int]) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
weight_scale: torch.Tensor, logical_widths: list[int]
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Create channelwise buffer
|
||||
weight_scale_channel = torch.empty((sum(logical_widths), 1),
|
||||
dtype=torch.float32,
|
||||
device=weight_scale.device)
|
||||
weight_scale_channel = torch.empty(
|
||||
(sum(logical_widths), 1), dtype=torch.float32, device=weight_scale.device
|
||||
)
|
||||
|
||||
# Expand each scale to match the size of each logical matrix.
|
||||
start = 0
|
||||
@@ -105,8 +106,8 @@ def convert_to_channelwise(
|
||||
|
||||
|
||||
def requantize_with_max_scale(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor,
|
||||
logical_widths: list[int]) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor, logical_widths: list[int]
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Max scale to be used for requanitzation.
|
||||
max_w_scale = weight_scale.max()
|
||||
|
||||
@@ -116,8 +117,9 @@ def requantize_with_max_scale(
|
||||
# from disk in this case. Skip requantization in this case (since)
|
||||
# we already are quantized with the single scale.
|
||||
# * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8
|
||||
unfused_module_in_checkpoint = (weight_scale[-1]
|
||||
> torch.finfo(torch.float8_e4m3fn).min)
|
||||
unfused_module_in_checkpoint = (
|
||||
weight_scale[-1] > torch.finfo(torch.float8_e4m3fn).min
|
||||
)
|
||||
|
||||
# If unfused checkpoint, need requanize with the single scale.
|
||||
if unfused_module_in_checkpoint:
|
||||
@@ -127,10 +129,8 @@ def requantize_with_max_scale(
|
||||
if logical_width == 0:
|
||||
continue
|
||||
end = start + logical_width
|
||||
weight_dq = per_tensor_dequantize(weight[start:end, :],
|
||||
weight_scale[idx])
|
||||
weight[start:end, :], _ = ops.scaled_fp8_quant(
|
||||
weight_dq, max_w_scale)
|
||||
weight_dq = per_tensor_dequantize(weight[start:end, :], weight_scale[idx])
|
||||
weight[start:end, :], _ = ops.scaled_fp8_quant(weight_dq, max_w_scale)
|
||||
start = end
|
||||
|
||||
return max_w_scale, weight
|
||||
@@ -143,75 +143,102 @@ def maybe_create_device_identity():
|
||||
TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32)
|
||||
|
||||
|
||||
def cutlass_w8a8_scaled_mm(*, qinput: torch.Tensor, weight: torch.Tensor,
|
||||
out_dtype: torch.dtype, scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor, bias: torch.Tensor,
|
||||
output_shape: list, **kwargs) -> torch.Tensor:
|
||||
|
||||
def cutlass_w8a8_scaled_mm(
|
||||
*,
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
# Fused GEMM_DQ
|
||||
output = ops.cutlass_scaled_mm(qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
bias=bias)
|
||||
output = ops.cutlass_scaled_mm(
|
||||
qinput, weight, out_dtype=out_dtype, scale_a=scale_a, scale_b=scale_b, bias=bias
|
||||
)
|
||||
return output.view(*output_shape)
|
||||
|
||||
|
||||
def flashinfer_w8a8_scaled_mm(*, qinput: torch.Tensor, weight: torch.Tensor,
|
||||
out_dtype: torch.dtype, scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor, bias: torch.Tensor,
|
||||
output_shape: list, **kwargs) -> torch.Tensor:
|
||||
|
||||
return flashinfer_scaled_fp8_mm(qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
bias=bias)
|
||||
def flashinfer_w8a8_scaled_mm(
|
||||
*,
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
return flashinfer_scaled_fp8_mm(
|
||||
qinput, weight, out_dtype=out_dtype, scale_a=scale_a, scale_b=scale_b, bias=bias
|
||||
)
|
||||
|
||||
|
||||
def rocm_per_tensor_w8a8_scaled_mm_impl(qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor) -> torch.Tensor:
|
||||
def rocm_per_tensor_w8a8_scaled_mm_impl(
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
if envs.VLLM_ROCM_USE_SKINNY_GEMM and on_mi3xx() and \
|
||||
qinput.shape[0] == 1 and \
|
||||
qinput.shape[1] % 16 == 0 and \
|
||||
((bias is None) or (bias.dtype == out_dtype)) :
|
||||
output = ops.wvSplitKQ(weight.t(), qinput, out_dtype, scale_a, scale_b,
|
||||
current_platform.get_cu_count(), bias)
|
||||
|
||||
if (
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and on_mi3xx()
|
||||
and qinput.shape[0] == 1
|
||||
and qinput.shape[1] % 16 == 0
|
||||
and ((bias is None) or (bias.dtype == out_dtype))
|
||||
):
|
||||
output = ops.wvSplitKQ(
|
||||
weight.t(),
|
||||
qinput,
|
||||
out_dtype,
|
||||
scale_a,
|
||||
scale_b,
|
||||
current_platform.get_cu_count(),
|
||||
bias,
|
||||
)
|
||||
else:
|
||||
output = torch._scaled_mm(qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
bias=bias)
|
||||
output = torch._scaled_mm(
|
||||
qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
bias=bias,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def rocm_per_tensor_w8a8_scaled_mm_fake(qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor) -> torch.Tensor:
|
||||
return qinput.new_empty((*qinput.shape[:-1], weight.shape[1]),
|
||||
dtype=out_dtype)
|
||||
def rocm_per_tensor_w8a8_scaled_mm_fake(
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return qinput.new_empty((*qinput.shape[:-1], weight.shape[1]), dtype=out_dtype)
|
||||
|
||||
|
||||
def rocm_per_tensor_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor, bias: torch.Tensor,
|
||||
output_shape: list) -> torch.Tensor:
|
||||
def rocm_per_tensor_w8a8_scaled_mm(
|
||||
*,
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
output = torch.ops.vllm.rocm_per_tensor_w8a8_scaled_mm_impl(
|
||||
qinput, weight, out_dtype, scale_a, scale_b, bias)
|
||||
qinput, weight, out_dtype, scale_a, scale_b, bias
|
||||
)
|
||||
return torch.narrow(output, 0, 0, qinput.shape[0]).view(*output_shape)
|
||||
|
||||
|
||||
@@ -222,18 +249,19 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def torch_per_tensor_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor, bias: torch.Tensor,
|
||||
output_shape: list) -> torch.Tensor:
|
||||
output = torch._scaled_mm(qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
bias=bias)
|
||||
def torch_per_tensor_w8a8_scaled_mm(
|
||||
*,
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
output = torch._scaled_mm(
|
||||
qinput, weight, out_dtype=out_dtype, scale_a=scale_a, scale_b=scale_b, bias=bias
|
||||
)
|
||||
# A fix for discrepancy in scaled_mm which returns tuple
|
||||
# for torch < 2.5 and a single value in torch >= 2.5
|
||||
if type(output) is tuple and len(output) == 2:
|
||||
@@ -242,13 +270,17 @@ def torch_per_tensor_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
return torch.narrow(output, 0, 0, qinput.shape[0]).view(*output_shape)
|
||||
|
||||
|
||||
def torch_per_token_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor, bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
**kwargs) -> torch.Tensor:
|
||||
def torch_per_token_w8a8_scaled_mm(
|
||||
*,
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
# Note: Callers of this function should check USE_ROWWISE_TORCH_SCALED_MM
|
||||
# when using it.
|
||||
# For now it has only been validated on ROCm platform.
|
||||
@@ -260,25 +292,31 @@ def torch_per_token_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
# rowwise scaled GEMM before using it
|
||||
|
||||
# Fused GEMM_DQ Rowwise GEMM
|
||||
output = torch._scaled_mm(qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b.t(),
|
||||
bias=bias)
|
||||
output = torch._scaled_mm(
|
||||
qinput,
|
||||
weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b.t(),
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
output = torch.narrow(output, 0, 0, qinput.shape[0])
|
||||
output = output.view(*output_shape)
|
||||
return output
|
||||
|
||||
|
||||
def torch_channelwise_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor, bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
**kwargs) -> torch.Tensor:
|
||||
def torch_channelwise_w8a8_scaled_mm(
|
||||
*,
|
||||
qinput: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
bias: torch.Tensor,
|
||||
output_shape: list,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
# Use unfused DQ due to limitations with scaled_mm
|
||||
|
||||
# Symmetric quantized GEMM by definition computes the following:
|
||||
@@ -296,11 +334,13 @@ def torch_channelwise_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
# GEMM
|
||||
# This computes C = (X * W).
|
||||
# Output in fp32 to allow subsequent ops to happen in-place
|
||||
output = torch._scaled_mm(qinput,
|
||||
weight,
|
||||
scale_a=TORCH_DEVICE_IDENTITY,
|
||||
scale_b=TORCH_DEVICE_IDENTITY,
|
||||
out_dtype=torch.float32)
|
||||
output = torch._scaled_mm(
|
||||
qinput,
|
||||
weight,
|
||||
scale_a=TORCH_DEVICE_IDENTITY,
|
||||
scale_b=TORCH_DEVICE_IDENTITY,
|
||||
out_dtype=torch.float32,
|
||||
)
|
||||
# A fix for discrepancy in scaled_mm which returns tuple
|
||||
# for torch < 2.5 and a single value in torch >= 2.5
|
||||
if type(output) is tuple and len(output) == 2:
|
||||
@@ -318,9 +358,8 @@ def torch_channelwise_w8a8_scaled_mm(*, qinput: torch.Tensor,
|
||||
|
||||
|
||||
def dispatch_w8a8_scaled_mm(
|
||||
preferred_backend: str, per_tensor_weights: bool,
|
||||
per_tensor_activations: bool) -> Callable[..., torch.Tensor]:
|
||||
|
||||
preferred_backend: str, per_tensor_weights: bool, per_tensor_activations: bool
|
||||
) -> Callable[..., torch.Tensor]:
|
||||
if per_tensor_weights and per_tensor_activations:
|
||||
if preferred_backend == "rocm":
|
||||
return rocm_per_tensor_w8a8_scaled_mm
|
||||
@@ -335,8 +374,11 @@ def dispatch_w8a8_scaled_mm(
|
||||
return cutlass_w8a8_scaled_mm
|
||||
|
||||
# If torch.scaled_mm supports per-channel (weights) per-token (inputs)
|
||||
if not per_tensor_weights and not per_tensor_activations \
|
||||
and USE_ROWWISE_TORCH_SCALED_MM:
|
||||
if (
|
||||
not per_tensor_weights
|
||||
and not per_tensor_activations
|
||||
and USE_ROWWISE_TORCH_SCALED_MM
|
||||
):
|
||||
return torch_per_token_w8a8_scaled_mm
|
||||
# Normally, torch.scaled_mm supports per tensor weights + activations only
|
||||
# so fallback to naive if per channel or per token
|
||||
@@ -353,15 +395,16 @@ class Fp8LinearOp:
|
||||
in the __init__ method, as reading config is not allowed inside forward.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
act_quant_static: bool,
|
||||
act_quant_group_shape: GroupShape = GroupShape.PER_TENSOR,
|
||||
pad_output: Optional[bool] = None):
|
||||
def __init__(
|
||||
self,
|
||||
act_quant_static: bool,
|
||||
act_quant_group_shape: GroupShape = GroupShape.PER_TENSOR,
|
||||
pad_output: Optional[bool] = None,
|
||||
):
|
||||
if current_platform.is_rocm():
|
||||
self.preferred_backend = "rocm"
|
||||
elif current_platform.is_cuda() and cutlass_fp8_supported():
|
||||
if has_flashinfer() and current_platform.has_device_capability(
|
||||
100):
|
||||
if has_flashinfer() and current_platform.has_device_capability(100):
|
||||
self.preferred_backend = "flashinfer"
|
||||
else:
|
||||
self.preferred_backend = "cutlass"
|
||||
@@ -375,15 +418,19 @@ class Fp8LinearOp:
|
||||
# as it breaks with dynamic shapes.
|
||||
if pad_output is None:
|
||||
config = get_current_vllm_config().compilation_config
|
||||
pad_output = config.level < CompilationLevel.PIECEWISE and \
|
||||
self.preferred_backend == "torch"
|
||||
pad_output = (
|
||||
config.level < CompilationLevel.PIECEWISE
|
||||
and self.preferred_backend == "torch"
|
||||
)
|
||||
|
||||
self.output_padding = 17 if pad_output else None
|
||||
self.act_quant_static = act_quant_static
|
||||
self.act_quant_group_shape = act_quant_group_shape
|
||||
self.quant_fp8 = QuantFP8(static=act_quant_static,
|
||||
group_shape=act_quant_group_shape,
|
||||
num_token_padding=self.output_padding)
|
||||
self.quant_fp8 = QuantFP8(
|
||||
static=act_quant_static,
|
||||
group_shape=act_quant_group_shape,
|
||||
num_token_padding=self.output_padding,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -417,27 +464,29 @@ class Fp8LinearOp:
|
||||
else:
|
||||
qinput, x_scale = input_2d, input_scale
|
||||
|
||||
per_tensor_weights = (weight_scale.numel() == 1)
|
||||
per_tensor_activations = (x_scale.numel() == 1)
|
||||
per_tensor_weights = weight_scale.numel() == 1
|
||||
per_tensor_activations = x_scale.numel() == 1
|
||||
|
||||
# TODO(luka) do this dispatch during init (after ScaledMM refactor)
|
||||
w8a8_scaled_mm_func = dispatch_w8a8_scaled_mm(self.preferred_backend,
|
||||
per_tensor_weights,
|
||||
per_tensor_activations)
|
||||
w8a8_scaled_mm_func = dispatch_w8a8_scaled_mm(
|
||||
self.preferred_backend, per_tensor_weights, per_tensor_activations
|
||||
)
|
||||
|
||||
return w8a8_scaled_mm_func(qinput=qinput,
|
||||
weight=weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=x_scale,
|
||||
scale_b=weight_scale,
|
||||
bias=bias,
|
||||
output_shape=output_shape)
|
||||
return w8a8_scaled_mm_func(
|
||||
qinput=qinput,
|
||||
weight=weight,
|
||||
out_dtype=out_dtype,
|
||||
scale_a=x_scale,
|
||||
scale_b=weight_scale,
|
||||
bias=bias,
|
||||
output_shape=output_shape,
|
||||
)
|
||||
|
||||
|
||||
def normalize_e4m3fn_to_e4m3fnuz(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
input_scale: Optional[torch.Tensor] = None
|
||||
input_scale: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
|
||||
assert weight.dtype == torch.float8_e4m3fn
|
||||
# The bits pattern 10000000(-128) represents zero in e4m3fn
|
||||
|
||||
Reference in New Issue
Block a user