[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -74,6 +74,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import (
|
||||
from vllm.model_executor.kernels.linear.mxfp4.flashinfer import (
|
||||
FlashInferMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp4.humming import (
|
||||
HummingMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp4.marlin import (
|
||||
MarlinMxFp4LinearKernel,
|
||||
)
|
||||
@@ -91,6 +94,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.humming import (
|
||||
HummingMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
|
||||
MarlinMxfp8LinearKernel,
|
||||
)
|
||||
@@ -120,6 +126,9 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
FlashInferTrtllmNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.humming import (
|
||||
HummingNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4.marlin import (
|
||||
MarlinNvFp4LinearKernel,
|
||||
)
|
||||
@@ -154,6 +163,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import (
|
||||
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
|
||||
FlashInferFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.humming import (
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
HummingInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -225,6 +238,14 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"flashinfer_b12x": {
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
},
|
||||
"humming": {
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
HummingInt8ScaledMMLinearKernel,
|
||||
HummingLinearKernel,
|
||||
HummingMxfp8LinearKernel,
|
||||
HummingMxFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
},
|
||||
"marlin": {
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
MarlinLinearKernel,
|
||||
@@ -292,6 +313,7 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
|
||||
PlatformEnum.CUDA: [
|
||||
CutlassInt8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
HummingInt8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
|
||||
}
|
||||
@@ -304,6 +326,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
@@ -335,6 +358,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
CutlassFp8BlockScaledMMKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
AiterFp8BlockScaledMMKernel,
|
||||
@@ -351,6 +375,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
|
||||
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
@@ -371,10 +396,10 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
MacheteLinearKernel,
|
||||
AllSparkLinearKernel,
|
||||
MarlinLinearKernel,
|
||||
HummingLinearKernel,
|
||||
ConchLinearKernel,
|
||||
ExllamaLinearKernel,
|
||||
TritonW4A16LinearKernel,
|
||||
HummingLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
RDNA3W4A16LinearKernel,
|
||||
@@ -400,6 +425,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
MarlinMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
HummingMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
|
||||
@@ -426,6 +452,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
FlashInferCudnnNvFp4LinearKernel,
|
||||
FbgemmNvFp4LinearKernel,
|
||||
EmulationNvFp4LinearKernel,
|
||||
HummingNvFp4LinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
EmulationNvFp4LinearKernel,
|
||||
@@ -436,6 +463,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
|
||||
PlatformEnum.CUDA: [
|
||||
FlashInferMxFp4LinearKernel,
|
||||
MarlinMxFp4LinearKernel,
|
||||
HummingMxFp4LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp4LinearKernel,
|
||||
|
||||
@@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import (
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.exllama import (
|
||||
ExllamaLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.humming import (
|
||||
HummingLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.machete import (
|
||||
MacheteLinearKernel,
|
||||
)
|
||||
@@ -52,6 +55,7 @@ __all__ = [
|
||||
"CutlassW4A8LinearKernel",
|
||||
"Dynamic4bitLinearKernel",
|
||||
"ExllamaLinearKernel",
|
||||
"HummingLinearKernel",
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"RDNA3W4A16LinearKernel",
|
||||
|
||||
@@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
return False, "Humming is not installed"
|
||||
if c.has_g_idx:
|
||||
return False, "Humming does not support act-order (g_idx)"
|
||||
if c.zero_points:
|
||||
return False, "Humming linear kernel only supports symmetric weights"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
"group_size": 0 if group_size == -1 else group_size,
|
||||
}
|
||||
|
||||
if self.config.zero_points:
|
||||
assert self.w_zp_name is not None
|
||||
name_map["zero_point"] = self.w_zp_name
|
||||
quant_config["has_zero_point"] = True
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
|
||||
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
"""Humming GEMM Kernel for MXFP4."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
|
||||
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
|
||||
|
||||
quant_config = {
|
||||
"quant_method": "humming",
|
||||
"dtype": "float4e2m1",
|
||||
"scale_dtype": "float8e8m0",
|
||||
"group_size": 32,
|
||||
"weight_scale_type": "group",
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Humming GEMM Kernel for MXFP8."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
|
||||
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
|
||||
|
||||
quant_config = {
|
||||
"quant_method": "humming",
|
||||
"dtype": "float8e4m3",
|
||||
"scale_dtype": "float8e8m0",
|
||||
"group_size": 32,
|
||||
"weight_scale_type": "group",
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
@@ -0,0 +1,73 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_layer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
"""Humming GEMM Kernel for NVFP4."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Route through humming's compressed-tensors nvfp4 loader (same path as
|
||||
# the MoE oracle); the native group_tensor schema mishandles a scalar
|
||||
# global scale.
|
||||
quant_config = {
|
||||
"quant_method": "compressed-tensors",
|
||||
"format": "nvfp4-pack-quantized",
|
||||
"type": "float",
|
||||
"num_bits": 4,
|
||||
"strategy": "group",
|
||||
"group_size": 16,
|
||||
}
|
||||
# CT pack-quantized reads `weight_packed`; the scheme renamed it to `weight`.
|
||||
if not hasattr(layer, "weight_packed"):
|
||||
layer.weight_packed = layer.weight
|
||||
del layer.weight
|
||||
# The CT linear scheme inverts the global scale (1/scale) for
|
||||
# marlin/cutlass; humming wants the original.
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
1.0 / layer.weight_global_scale, requires_grad=False
|
||||
)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Int8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
"""Humming GEMM Kernel for FP8."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(
|
||||
cls, config: FP8ScaledMMLinearLayerConfig
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.utils.humming import dtypes
|
||||
|
||||
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
|
||||
scale_torch_dtype = self.config.weight_quant_key.scale.dtype
|
||||
scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype)
|
||||
|
||||
quant_config = {
|
||||
"quant_method": "humming",
|
||||
"dtype": "float8e4m3",
|
||||
"scale_dtype": scale_dtype,
|
||||
}
|
||||
|
||||
assert self.config.weight_quant_key.scale2 is None
|
||||
scale_group_shape = self.config.weight_quant_key.scale.group_shape
|
||||
if scale_group_shape.is_per_tensor():
|
||||
quant_config["weight_scale_type"] = "tensor"
|
||||
if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
|
||||
del name_map["weight_scale"]
|
||||
name_map["global_scale"] = "weight_scale"
|
||||
elif scale_group_shape.is_per_channel():
|
||||
quant_config["weight_scale_type"] = "channel"
|
||||
elif scale_group_shape.is_per_group():
|
||||
quant_config["weight_scale_type"] = "group"
|
||||
quant_config["group_size"] = scale_group_shape.col
|
||||
else:
|
||||
assert scale_group_shape.row > 0 and scale_group_shape.col > 0
|
||||
quant_config["weight_scale_type"] = "block"
|
||||
quant_config["weight_scale_group_size_n"] = scale_group_shape.row
|
||||
quant_config["weight_scale_group_size"] = scale_group_shape.col
|
||||
|
||||
if hasattr(layer, "weight_scale_inv"):
|
||||
name_map["weight_scale"] = "weight_scale_inv"
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
"""Humming GEMM Kernel for INT8."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(
|
||||
cls, config: Int8ScaledMMLinearLayerConfig
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight_name, weight_scale_name, *_ = self.layer_param_names
|
||||
name_map = {"weight": weight_name, "weight_scale": weight_scale_name}
|
||||
quant_config = {"quant_method": "humming", "dtype": "int8"}
|
||||
weight = getattr(layer, weight_name)
|
||||
weight.data = weight.data + 128
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
Reference in New Issue
Block a user