Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -82,6 +82,7 @@ def register_quantization_config(quantization: str):
|
||||
>>>
|
||||
>>> get_quantization_config("my_quant")
|
||||
<class 'MyQuantConfig'>
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
def _wrapper(quant_config_cls):
|
||||
|
||||
@@ -393,6 +393,7 @@ class AutoAWQMarlinLinearMethod(LinearMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: The AWQ Marlin quantization config.
|
||||
|
||||
"""
|
||||
|
||||
_kernel_backends_being_used: set[str] = set()
|
||||
@@ -701,7 +702,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
"""Build the FusedMoEKernel for this layer."""
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
self.moe_kernel = make_wna16_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
@@ -880,6 +880,7 @@ class AutoAWQLinearMethod(BaseAWQLinearMethod):
|
||||
|
||||
Args:
|
||||
quant_config: The AWQ quantization config.
|
||||
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
@@ -305,6 +305,7 @@ class AutoGPTQLinearMethod(LinearMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: The AutoGPTQ quantization config.
|
||||
|
||||
"""
|
||||
|
||||
_kernel_backends_being_used: set[str] = set()
|
||||
@@ -685,7 +686,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
"""Build the FusedMoEKernel for this layer."""
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
self.moe_kernel = make_wna16_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
|
||||
@@ -46,21 +46,24 @@ class QuantizeMethodBase(ABC):
|
||||
):
|
||||
"""Create weights for a layer.
|
||||
|
||||
The weights will be set as attributes of the layer."""
|
||||
The weights will be set as attributes of the layer.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
|
||||
"""Apply the weights in layer to the input tensor.
|
||||
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
|
||||
"""Gather embeddings in the layer based on indices in the input tensor.
|
||||
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# Not required functions
|
||||
@@ -73,7 +76,8 @@ class QuantizeMethodBase(ABC):
|
||||
Quantization methods that need special weight handling (e.g. repacked
|
||||
weights) override this.
|
||||
|
||||
Expects create_weights to have been called before on the layer."""
|
||||
Expects create_weights to have been called before on the layer.
|
||||
"""
|
||||
layer.weight = embed_tokens.weight
|
||||
return layer
|
||||
|
||||
@@ -86,8 +90,7 @@ class QuantizeMethodBase(ABC):
|
||||
|
||||
|
||||
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
|
||||
"""
|
||||
Not all quant methods have embedding implemented, so we need to check that
|
||||
"""Not all quant methods have embedding implemented, so we need to check that
|
||||
it exists for our given method. We check this by making sure the function
|
||||
has been changed from the base implementation.
|
||||
"""
|
||||
@@ -158,8 +161,7 @@ class QuantizationConfig(ABC):
|
||||
user_quant: str | None,
|
||||
hf_config: Any = None,
|
||||
) -> QuantizationMethods | None:
|
||||
"""
|
||||
Detects if this quantization method can support a given checkpoint
|
||||
"""Detects if this quantization method can support a given checkpoint
|
||||
format by overriding the user specified quantization method --
|
||||
this method should only be overwritten by subclasses in exceptional
|
||||
circumstances.
|
||||
@@ -169,6 +171,7 @@ class QuantizationConfig(ABC):
|
||||
user_quant: The user-specified quantization method string.
|
||||
hf_config: The HuggingFace model config object (e.g. for
|
||||
model_type checks). May be None if not available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -203,6 +206,7 @@ class QuantizationConfig(ABC):
|
||||
Returns:
|
||||
The quantize method. None if the given layer doesn't support quant
|
||||
method.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -251,13 +255,13 @@ class QuantizationConfig(ABC):
|
||||
def apply_vllm_mapper( # noqa: B027
|
||||
self, hf_to_vllm_mapper: "WeightsMapper"
|
||||
):
|
||||
"""
|
||||
Interface for models to update module names referenced in
|
||||
"""Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the vllm model structure
|
||||
|
||||
Args:
|
||||
hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
|
||||
"""
|
||||
# TODO (@kylesayrs): add implementations for all subclasses
|
||||
pass
|
||||
@@ -268,14 +272,14 @@ class QuantizationConfig(ABC):
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
revision: str | None = None,
|
||||
):
|
||||
"""
|
||||
Interface to update values after config initialization.
|
||||
"""Interface to update values after config initialization.
|
||||
|
||||
Args:
|
||||
model_name: The name of the model
|
||||
hf_config: The Hugging Face config of the model
|
||||
revision: The revision of the model
|
||||
Returns:
|
||||
|
||||
"""
|
||||
# TODO: revision is never passed currently in vllm.py,
|
||||
# but is used in subclasses, should we remove this parameter?
|
||||
|
||||
@@ -121,8 +121,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
return "compressed-tensors"
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
"""
|
||||
Transform layer paths in config targets to match vLLM's naming.
|
||||
"""Transform layer paths in config targets to match vLLM's naming.
|
||||
|
||||
The WeightsMapper is designed for weight paths, but some backends
|
||||
(e.g. transformers) use broad prefix mappings like "" -> "model."
|
||||
@@ -214,8 +213,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
return None
|
||||
|
||||
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
|
||||
"""
|
||||
Helper function to update target_scheme_map
|
||||
"""Helper function to update target_scheme_map
|
||||
since linear layers get fused into RoutedExperts
|
||||
targeting 'Linear' needs to also match
|
||||
RoutedExperts modules.
|
||||
@@ -269,8 +267,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def _parse_sparsity_config(
|
||||
cls, config: dict[str, Any]
|
||||
) -> tuple[dict[str, SparsityCompressionConfig], list[str]]:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
config: The `quantization_config` dictionary from config.json
|
||||
|
||||
Returns:
|
||||
@@ -278,6 +275,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
1. A dictionary mapping target layer names to their corresponding
|
||||
sparsity_config
|
||||
2. A list of layer names to ignore for sparsity
|
||||
|
||||
"""
|
||||
if not (sparsity_config := config.get(SPARSITY_CONFIG_NAME)):
|
||||
return dict(), []
|
||||
@@ -301,13 +299,13 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def _quantization_scheme_map_from_config(
|
||||
cls, config: dict[str, Any]
|
||||
) -> QUANTIZATION_SCHEME_MAP_TYPE:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
config: The `quantization_config` dictionary from config.json
|
||||
|
||||
Returns:
|
||||
A dictionary mapping target layer names to their corresponding
|
||||
quantization_args for weights and input activations
|
||||
|
||||
"""
|
||||
target_scheme_map: dict[str, Any] = dict()
|
||||
quant_format = cast(str, config.get("format"))
|
||||
@@ -667,7 +665,8 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
) -> bool:
|
||||
"""Weight N-bit INT (pack-quantized for sub-byte, int-quantized for 8-bit)
|
||||
with static per-tensor INT8 input/output activation quant, applied as a float
|
||||
fake-quant around a weight-only matmul."""
|
||||
fake-quant around a weight-only matmul.
|
||||
"""
|
||||
is_int_pack_format = format in (
|
||||
CompressionFormat.pack_quantized.value,
|
||||
CompressionFormat.int_quantized.value,
|
||||
@@ -703,7 +702,8 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
format: str | None,
|
||||
) -> bool:
|
||||
"""Weight N-bit INT with symmetric dynamic INT activation quant
|
||||
via Humming kernel."""
|
||||
via Humming kernel.
|
||||
"""
|
||||
if input_quant is None:
|
||||
return False
|
||||
is_pack_format = format == CompressionFormat.pack_quantized.value
|
||||
@@ -895,8 +895,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def get_scheme(
|
||||
self, layer: torch.nn.Module, layer_name: str | None = None
|
||||
) -> "CompressedTensorsScheme | None":
|
||||
"""
|
||||
compressed-tensors supports non uniform in the following way:
|
||||
"""compressed-tensors supports non uniform in the following way:
|
||||
|
||||
targets of config_groups: There can be N config_groups which each
|
||||
have a quantization scheme. Each config_group has a list of targets
|
||||
@@ -907,7 +906,6 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
use the quantization scheme corresponding to the matched target
|
||||
to select the CompressedTensorsScheme used for inference.
|
||||
"""
|
||||
|
||||
# Use the new get_quant_args method to extract QuantizationArgs
|
||||
scheme_dict = self.get_scheme_dict(layer, layer_name)
|
||||
|
||||
@@ -944,8 +942,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
def get_scheme_dict(
|
||||
self, layer: torch.nn.Module, layer_name: str | None = None
|
||||
) -> dict[str, QuantizationArgs | str | None] | None:
|
||||
"""
|
||||
Extract the QuantizationArgs for a given layer.
|
||||
"""Extract the QuantizationArgs for a given layer.
|
||||
|
||||
Returns:
|
||||
dict with {
|
||||
@@ -953,6 +950,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
"input_activations": QuantizationArgs | None,
|
||||
"format": str | None
|
||||
} | None
|
||||
|
||||
"""
|
||||
# TODO (@kylesayrs): support ignore module names with ct matching utils
|
||||
if should_ignore_layer(
|
||||
@@ -1004,8 +1002,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
"""
|
||||
Use the CompressedTensorsScheme associated with each layer to create
|
||||
"""Use the CompressedTensorsScheme associated with each layer to create
|
||||
the necessary parameters for the layer. See LinearMethodBase for param
|
||||
details
|
||||
"""
|
||||
@@ -1026,8 +1023,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Use the output of create_weights and the CompressedTensorsScheme
|
||||
"""Use the output of create_weights and the CompressedTensorsScheme
|
||||
associated with the layer to apply the forward pass with the
|
||||
layer input. See LinearMethodBase for param details
|
||||
|
||||
@@ -1039,8 +1035,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
|
||||
|
||||
|
||||
class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Supports loading kv-cache scaling factors from compressed-tensors
|
||||
"""Supports loading kv-cache scaling factors from compressed-tensors
|
||||
checkpoints.
|
||||
"""
|
||||
|
||||
@@ -1050,12 +1045,12 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
|
||||
@staticmethod
|
||||
def validate_kv_cache_scheme(kv_cache_scheme: dict[str, Any] | None):
|
||||
"""
|
||||
Validator for the kv cache scheme. Useful for controlling the
|
||||
"""Validator for the kv cache scheme. Useful for controlling the
|
||||
kv cache quantization schemes, that are being supported in vLLM
|
||||
|
||||
Args:
|
||||
kv_cache_scheme: the compressed-tensors kv cache scheme
|
||||
|
||||
"""
|
||||
if kv_cache_scheme is None:
|
||||
return
|
||||
@@ -1091,8 +1086,7 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
)
|
||||
|
||||
def create_weights(self, layer: torch.nn.Module):
|
||||
"""
|
||||
Initialize placeholder scales and zero points to enable loading of
|
||||
"""Initialize placeholder scales and zero points to enable loading of
|
||||
quantized params from compressed-tensors checkpoints.
|
||||
"""
|
||||
strategy = None # for backward compatibility
|
||||
@@ -1199,8 +1193,7 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Override the default vLLM placeholder scales with the llm-compressor loaded
|
||||
"""Override the default vLLM placeholder scales with the llm-compressor loaded
|
||||
scales. Zero points are not used as only symmetric quantization is supported.
|
||||
"""
|
||||
layer._k_scale = layer.k_scale
|
||||
|
||||
+2
-1
@@ -37,7 +37,8 @@ def _dequant_gather_kernel(
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Gather embedding rows by token id, unpack int32-packed INT weights, and
|
||||
dequantize to ``out`` dtype in one pass (no int8 intermediate)."""
|
||||
dequantize to ``out`` dtype in one pass (no int8 intermediate).
|
||||
"""
|
||||
row = tl.program_id(0)
|
||||
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
col_mask = col < hidden
|
||||
|
||||
+1
-3
@@ -184,9 +184,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
set_weight_attrs(w2_input_scale, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
|
||||
# NOTE(rob): wN_weight_packed -> wN_weight is because ModularKernelMethod
|
||||
# requires this naming convention. However, the name change breaks
|
||||
# reloading because the state dict no longer matches disk. Once we
|
||||
|
||||
+1
-2
@@ -36,8 +36,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
"""
|
||||
CPU-only MoE method using dynamic 4-bit matmul kernels on Arm Platform
|
||||
"""CPU-only MoE method using dynamic 4-bit matmul kernels on Arm Platform
|
||||
- Weights: int4 (stored as int8 values in [-8,7], packed to uint8 nibbles)
|
||||
- Scales: Fp32 for Channelwise , bf16 for groupwise quantization
|
||||
- Bias: Same data type as original weights
|
||||
|
||||
+1
-2
@@ -143,8 +143,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
num_groups_w2: int | None = None,
|
||||
num_groups_w13: int | None = None,
|
||||
) -> tuple[int, int, int]:
|
||||
"""
|
||||
Get the shape of the weight based on the weight name, number of experts
|
||||
"""Get the shape of the weight based on the weight name, number of experts
|
||||
hidden size, intermediate size per partition, number of groups for w2,
|
||||
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
|
||||
for weight scales/zero_points.
|
||||
|
||||
+6
-13
@@ -9,33 +9,26 @@ __all__ = ["CompressedTensorsScheme"]
|
||||
|
||||
|
||||
class CompressedTensorsScheme(ABC):
|
||||
"""
|
||||
Abstract class used to describe the weight creation and forward pass
|
||||
"""Abstract class used to describe the weight creation and forward pass
|
||||
of different quantization schemes supported by CompressedTensors.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
"""
|
||||
Get minimum device capability.
|
||||
"""
|
||||
"""Get minimum device capability."""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(self, *args, **kwargs):
|
||||
"""
|
||||
Weight creation for the particular scheme. Inputs to this function
|
||||
|
||||
"""
|
||||
"""Weight creation for the particular scheme. Inputs to this function."""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def apply_weights(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
):
|
||||
"""
|
||||
Run the forward pass for the particular scheme. This is where
|
||||
"""Run the forward pass for the particular scheme. This is where
|
||||
scheme-specific dequant/quant steps/kernels should be applied.
|
||||
|
||||
Args:
|
||||
@@ -43,13 +36,13 @@ class CompressedTensorsScheme(ABC):
|
||||
other parameters relevant to the particular scheme.
|
||||
x: input to the layer
|
||||
bias: bias parameter
|
||||
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
"""
|
||||
Called after weight loading is complete for any cleanup that
|
||||
"""Called after weight loading is complete for any cleanup that
|
||||
needs to occur.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
+1
-2
@@ -21,8 +21,7 @@ __all__ = ["CompressedTensorsW4A4Mxfp4"]
|
||||
|
||||
|
||||
class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
|
||||
"""
|
||||
Compressed tensors scheme for MXFP4.
|
||||
"""Compressed tensors scheme for MXFP4.
|
||||
|
||||
Supports models quantized with the compressed-tensors mxfp4-pack-quantized
|
||||
format.
|
||||
|
||||
+1
-2
@@ -22,8 +22,7 @@ __all__ = ["CompressedTensorsW8A8Mxfp8"]
|
||||
|
||||
|
||||
class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme):
|
||||
"""
|
||||
Compressed tensors scheme for MXFP8 quantization (W8A8).
|
||||
"""Compressed tensors scheme for MXFP8 quantization (W8A8).
|
||||
|
||||
Loads pre-quantized MXFP8 weights from compressed-tensors checkpoints.
|
||||
Activations are dynamically quantized to MXFP8 at runtime.
|
||||
|
||||
+2
-1
@@ -211,7 +211,8 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
|
||||
def _pack_int_quantized_weight(self, layer: torch.nn.Module) -> None:
|
||||
"""Normalize an int-quantized (plain int8) weight to the canonical
|
||||
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect."""
|
||||
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect.
|
||||
"""
|
||||
weight = layer.weight
|
||||
out_features, in_features = weight.shape
|
||||
packed = pack_to_int32(weight.data.contiguous(), self.num_bits)
|
||||
|
||||
@@ -29,8 +29,7 @@ from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class CompressedTensorsLinearTransformMethod(LinearMethodBase):
|
||||
"""
|
||||
Wraps `CompressedTensorsLinearMethod` or `UnquantizedLinearMethod` and adds
|
||||
"""Wraps `CompressedTensorsLinearMethod` or `UnquantizedLinearMethod` and adds
|
||||
input and output transforms to either side of the original apply method
|
||||
"""
|
||||
|
||||
@@ -240,8 +239,7 @@ def get_schemes_args(
|
||||
def get_layer_partition_names(
|
||||
layer_name: str, packed_modules_mapping: dict[str, list[str]]
|
||||
) -> list[str]:
|
||||
"""
|
||||
Get all partition names associated with this layer.
|
||||
"""Get all partition names associated with this layer.
|
||||
Names are returned in order of their partition indices.
|
||||
|
||||
```python
|
||||
@@ -251,7 +249,9 @@ def get_layer_partition_names(
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
]
|
||||
assert get_layer_partition_names("mlp.down_proj", mapping) == ["down_proj"]"""
|
||||
assert get_layer_partition_names("mlp.down_proj", mapping) == ["down_proj"]
|
||||
```
|
||||
"""
|
||||
for fused_suffix, part_suffixes in packed_modules_mapping.items():
|
||||
if layer_name.endswith(fused_suffix):
|
||||
return [
|
||||
|
||||
@@ -23,8 +23,7 @@ from vllm.model_executor.parameter import SharedWeightParameter
|
||||
|
||||
|
||||
class HadamardTransform(torch.nn.Module):
|
||||
"""
|
||||
Class which handles weight loading, postprocessing, and application of
|
||||
"""Class which handles weight loading, postprocessing, and application of
|
||||
transforms. Meant to be used with `CompressedTensorsLinearTransformMethod`
|
||||
and attention transforms method (not implemented yet)
|
||||
"""
|
||||
|
||||
+1
-2
@@ -76,8 +76,7 @@ class QutlassNvFP4LinearMethod(CompressedTensorsLinearTransformMethod):
|
||||
|
||||
@staticmethod
|
||||
def _get_flashinfer_gemm_backend(kernel: NvFp4LinearKernel) -> str:
|
||||
"""
|
||||
Given a kernel, find the string that is needed to be passed into
|
||||
"""Given a kernel, find the string that is needed to be passed into
|
||||
`flashinfer_scaled_fp4_mm`, using
|
||||
vllm.model_executor.kernels.linear._LINEAR_BACKEND_KERNEL_MAP as source of truth
|
||||
"""
|
||||
|
||||
@@ -76,8 +76,7 @@ def find_matched_target(
|
||||
targets: Iterable[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
) -> str | None:
|
||||
"""
|
||||
Helper function to look up which "target" in the compressed-tensors
|
||||
"""Helper function to look up which "target" in the compressed-tensors
|
||||
config that a layer corresponds to.
|
||||
|
||||
Recall that a compressed-tensors configs has a concept of
|
||||
@@ -98,8 +97,8 @@ def find_matched_target(
|
||||
module: torch.nn.Module
|
||||
targets: list of targets to match the layer against
|
||||
fused_mapping: map from fused layer names to its components
|
||||
"""
|
||||
|
||||
"""
|
||||
if layer_name is None:
|
||||
layer_name = ""
|
||||
|
||||
@@ -115,8 +114,7 @@ def find_matched_target(
|
||||
def _find_first_match(
|
||||
value: str, targets: Iterable[str], check_contains: bool = False
|
||||
) -> str | None:
|
||||
"""
|
||||
Returns first element of target that matches value either
|
||||
"""Returns first element of target that matches value either
|
||||
exactly or as a regex after 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
|
||||
@@ -124,8 +122,8 @@ def _find_first_match(
|
||||
value: string to compare the list of targets against
|
||||
targets: list of targets to match the layer against
|
||||
check_contains: whether or not to do a substring match
|
||||
"""
|
||||
|
||||
"""
|
||||
for target in targets:
|
||||
if is_equal_or_regex_match(value, target, check_contains=check_contains):
|
||||
return target
|
||||
@@ -137,8 +135,7 @@ def _match_fused_layer(
|
||||
target_layers: Iterable[str],
|
||||
fused_mapping: Mapping[str, list[str]],
|
||||
) -> str | None:
|
||||
"""
|
||||
Match a fused layer name to its corresponding individual layer in
|
||||
"""Match a fused layer name to its corresponding individual layer in
|
||||
target_layers. Returns first value in fused_mapping which matches targets
|
||||
|
||||
Implements an "all" matching strategy where a fused layer matches iff
|
||||
@@ -154,6 +151,7 @@ def _match_fused_layer(
|
||||
target_layers = ["model.layers.0.self_attn.q_proj",
|
||||
"model.layers.0.self_attn.k_proj",
|
||||
"model.layers.0.self_attn.v_proj"]
|
||||
|
||||
"""
|
||||
# find layer_name in mapping
|
||||
fused = next((key for key in fused_mapping if layer_name.endswith(key)), None)
|
||||
|
||||
@@ -251,6 +251,7 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: The quantization config.
|
||||
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
@@ -490,6 +491,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: The quantization config.
|
||||
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
@@ -881,9 +883,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
|
||||
class Fp8KVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Supports loading kv-cache scaling factors from FP8 checkpoints.
|
||||
"""
|
||||
"""Supports loading kv-cache scaling factors from FP8 checkpoints."""
|
||||
|
||||
def __init__(self, quant_config: Fp8Config):
|
||||
super().__init__(quant_config)
|
||||
|
||||
@@ -98,6 +98,7 @@ class FPQuantLinearMethod(LinearMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: The FPQuant quantization config.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: FPQuantConfig):
|
||||
|
||||
@@ -783,8 +783,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply Humming-quantized MoE computation using the standard kernel flow.
|
||||
"""Apply Humming-quantized MoE computation using the standard kernel flow.
|
||||
|
||||
This method uses FusedMoEKernel.apply() which orchestrates:
|
||||
1. Preparation (quantization if needed - skipped for Humming via
|
||||
|
||||
@@ -86,7 +86,8 @@ class INCConfigParser:
|
||||
|
||||
def is_explicitly_configured(name: str) -> bool:
|
||||
"""Return True if *name* has an explicit entry in extra_config,
|
||||
either via exact key match or via a regex pattern key."""
|
||||
either via exact key match or via a regex pattern key.
|
||||
"""
|
||||
if not self._config.extra_config:
|
||||
return False
|
||||
if name in self._config.extra_config:
|
||||
|
||||
@@ -48,7 +48,8 @@ class INCScheme(ABC):
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "FusedMoEMethodBase | None":
|
||||
"""Optional. Override if this scheme supports MoE.
|
||||
Default raises NotImplementedError."""
|
||||
Default raises NotImplementedError.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support MoE layers. "
|
||||
f"Layer config: {layer_config}"
|
||||
|
||||
@@ -28,8 +28,7 @@ _FP8_MIN_SCALING_FACTOR = 1.0 / (_FP8_MAX * 512.0)
|
||||
# --8<-- [start:quant_fp8]
|
||||
@CustomOp.register("quant_fp8")
|
||||
class QuantFP8(CustomOp):
|
||||
"""
|
||||
Quantize input tensor to FP8 (per-tensor, per-token, per-channel, or per-group).
|
||||
"""Quantize input tensor to FP8 (per-tensor, per-token, per-channel, or per-group).
|
||||
This CustomOp supports both static and dynamic quantization.
|
||||
"""
|
||||
|
||||
@@ -45,18 +44,18 @@ class QuantFP8(CustomOp):
|
||||
use_ue8m0: bool | None = None, # for Torch compile
|
||||
compile_native: bool = True,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
static: static or dynamic quantization
|
||||
group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
|
||||
PER_CHANNEL, or arbitrary block size)
|
||||
num_token_padding: Pad the token dimension of output to this
|
||||
size
|
||||
tma_aligned_scales: For group quantization, output scales in
|
||||
TMA-aligned layout
|
||||
column_major_scales: For group quantization, output scales in
|
||||
column major format
|
||||
compile_native: Manually compile forward_native if compile mode > None
|
||||
"""Args:
|
||||
static: static or dynamic quantization
|
||||
group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
|
||||
PER_CHANNEL, or arbitrary block size)
|
||||
num_token_padding: Pad the token dimension of output to this
|
||||
size
|
||||
tma_aligned_scales: For group quantization, output scales in
|
||||
TMA-aligned layout
|
||||
column_major_scales: For group quantization, output scales in
|
||||
column major format
|
||||
compile_native: Manually compile forward_native if compile mode > None
|
||||
|
||||
"""
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.static = static
|
||||
|
||||
@@ -41,8 +41,7 @@ class KVCacheScaleParameter(torch.nn.Parameter):
|
||||
|
||||
|
||||
class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
"""
|
||||
Quant method that adds `_k_scale` and `_v_scale` attributes to the
|
||||
"""Quant method that adds `_k_scale` and `_v_scale` attributes to the
|
||||
Attention layer to support loading those scaling factors from checkpoints.
|
||||
The k/v_scale will be used to:
|
||||
- quantize k/v_cache entries before saving them to the cache
|
||||
@@ -50,6 +49,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: the appropriate QuantizationConfig
|
||||
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
@@ -58,8 +58,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(self, layer: torch.nn.Module):
|
||||
"""
|
||||
Create "weight" (aka q_scale, k_scale and v_scale)
|
||||
"""Create "weight" (aka q_scale, k_scale and v_scale)
|
||||
for an attention layer.
|
||||
"""
|
||||
# Initialize the Q and KV cache scales to -1.0, an invalid value.
|
||||
|
||||
@@ -153,9 +153,7 @@ def algos_owned_by(config_name: str) -> tuple[str, ...]:
|
||||
|
||||
|
||||
class ModelOptKVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Supports loading kv-cache scaling factors from FP8 or NVFP4 checkpoints.
|
||||
"""
|
||||
"""Supports loading kv-cache scaling factors from FP8 or NVFP4 checkpoints."""
|
||||
|
||||
def __init__(self, quant_config: "ModelOptQuantConfigBase"):
|
||||
super().__init__(quant_config)
|
||||
@@ -178,8 +176,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
|
||||
self.exclude_modules: list[str] = exclude_modules
|
||||
|
||||
def is_layer_excluded(self, prefix: str) -> bool:
|
||||
"""
|
||||
Check if a layer should be excluded from quantization.
|
||||
"""Check if a layer should be excluded from quantization.
|
||||
|
||||
Handles both exact matching (for fused layers) and ModelOpt wildcard matching.
|
||||
|
||||
@@ -472,8 +469,10 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
"""MoE method for ModelOpt FP8.
|
||||
Supports loading FP8 checkpoints with static weight scale and
|
||||
activation scale.
|
||||
|
||||
Args:
|
||||
quant_config: The ModelOpt quantization config.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -818,10 +817,11 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
|
||||
|
||||
class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
"""
|
||||
MoE Method for FP4 Quantization.
|
||||
"""MoE Method for FP4 Quantization.
|
||||
|
||||
Args:
|
||||
quant_config: NVFP4 Quant Config
|
||||
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
@@ -851,9 +851,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
def uses_weight_scale_2_pattern(self) -> bool:
|
||||
"""
|
||||
FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales.
|
||||
"""
|
||||
"""FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales."""
|
||||
return True
|
||||
|
||||
def create_weights(
|
||||
@@ -973,9 +971,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_input_scale", w2_input_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
|
||||
if is_weights_pre_processed():
|
||||
if self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
raise RuntimeError(
|
||||
@@ -1992,7 +1988,8 @@ class KNvfp4Static(QuantKeyScheme):
|
||||
|
||||
class KNvfp4Dynamic(QuantKeyScheme):
|
||||
"""NVFP4 activation scheme (W4A4). Has a static global input scale on disk;
|
||||
the per-group scale is computed at runtime inside the kernel."""
|
||||
the per-group scale is computed at runtime inside the kernel.
|
||||
"""
|
||||
|
||||
key = kNvfp4Dynamic
|
||||
|
||||
@@ -2029,7 +2026,8 @@ class KNvfp4Dynamic(QuantKeyScheme):
|
||||
|
||||
class KFp8StaticTensor(QuantKeyScheme):
|
||||
"""Plain per-tensor static FP8 — bivalent: serves BOTH the weight slot and
|
||||
the activation slot (W8A8). One key in both QuantSpec slots."""
|
||||
the activation slot (W8A8). One key in both QuantSpec slots.
|
||||
"""
|
||||
|
||||
key = kFp8StaticTensorSym
|
||||
|
||||
@@ -2099,7 +2097,8 @@ class KFp8StaticTensor(QuantKeyScheme):
|
||||
|
||||
class KFp8StaticChannel(QuantKeyScheme):
|
||||
"""Per-channel static FP8 weight (the 'PcPt' weight). Weight-role only —
|
||||
there is no static per-channel *activation* today."""
|
||||
there is no static per-channel *activation* today.
|
||||
"""
|
||||
|
||||
key = kFp8StaticTokenSym
|
||||
|
||||
@@ -2140,7 +2139,8 @@ class KFp8StaticChannel(QuantKeyScheme):
|
||||
class KFp8Block128(QuantKeyScheme):
|
||||
"""128x128 block-static FP8 weight ('PbWo'). Weight-role only. ModelOpt
|
||||
exports the scale 4-D [out_blk,1,in_blk,1]; process squeezes to 2-D.
|
||||
No transpose (block kernel keeps [out,in])."""
|
||||
No transpose (block kernel keeps [out,in]).
|
||||
"""
|
||||
|
||||
key = kFp8Static128BlockSym
|
||||
|
||||
@@ -2196,7 +2196,8 @@ class KFp8Block128(QuantKeyScheme):
|
||||
|
||||
class KMxfp8Static(QuantKeyScheme):
|
||||
"""MXFP8 weight: fp8-e4m3 values + per-32-block e8m0 (uint8) scale.
|
||||
Weight-role only. process is validate-only plus an idempotency guard."""
|
||||
Weight-role only. process is validate-only plus an idempotency guard.
|
||||
"""
|
||||
|
||||
key = kMxfp8Static
|
||||
|
||||
@@ -2271,7 +2272,8 @@ class KDynamicNoParam(QuantKeyScheme):
|
||||
"""Dynamic activation with no stored scale (W8A8): quantized at runtime in
|
||||
the kernel. NOT the same as activation=None (weight-only) — init_fp8 needs a
|
||||
non-None activation key. Activation-role only. Serves the fp8 per-token, fp8
|
||||
per-block, and mxfp8 dynamic activation keys."""
|
||||
per-block, and mxfp8 dynamic activation keys.
|
||||
"""
|
||||
|
||||
def create_weights(self, layer, role, ctx, shapes, wl) -> None:
|
||||
if role is not ACT:
|
||||
@@ -2297,7 +2299,7 @@ SCHEME_FOR: dict[QuantKey | None, QuantKeyScheme] = {
|
||||
|
||||
|
||||
def maybe_fuse_global_scales(layer) -> None:
|
||||
"""alpha = input_global_scale * weight_global_scale, presence-gated.
|
||||
"""Alpha = input_global_scale * weight_global_scale, presence-gated.
|
||||
|
||||
W4A4 has both -> computed; W4A16 has no input_global_scale -> skipped.
|
||||
"""
|
||||
|
||||
@@ -229,6 +229,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
|
||||
Args:
|
||||
quant_config: The MOE WNA16 (W8A16/W4A16) quantization config.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: MoeWNA16Config, moe: "FusedMoEConfig") -> None:
|
||||
|
||||
@@ -79,7 +79,8 @@ class Mxfp4Config(QuantizationConfig):
|
||||
|
||||
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
|
||||
"""MoE method for RoutedExperts. Subclasses override to pick a
|
||||
checkpoint-specific kernel family."""
|
||||
checkpoint-specific kernel family.
|
||||
"""
|
||||
return Mxfp4MoEMethod(moe)
|
||||
|
||||
def get_quant_method(
|
||||
|
||||
@@ -139,7 +139,8 @@ def _find_matching_targets(
|
||||
|
||||
class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""Model-level config for online quantization (quantize fp16/bf16 weights
|
||||
during model loading, without requiring a pre-quantized checkpoint)."""
|
||||
during model loading, without requiring a pre-quantized checkpoint).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -227,6 +228,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
Returns:
|
||||
The matching method class, or None when ``spec`` has no weight
|
||||
quantization.
|
||||
|
||||
"""
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
@@ -260,6 +262,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
and method class. Returns None when online quantization does not
|
||||
apply to the layer.
|
||||
|
||||
"""
|
||||
quant_spec: QuantSpec | None
|
||||
if self.args.targets is not None:
|
||||
@@ -313,6 +316,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
and dispatch table. Returns None when no pattern applies or the
|
||||
layer is ignored.
|
||||
|
||||
"""
|
||||
assert self.args.targets is not None
|
||||
ignored = should_ignore_layer(
|
||||
|
||||
@@ -114,7 +114,8 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
|
||||
|
||||
class OnlineLinearBase(LinearMethodBase):
|
||||
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
weights onto meta device and materializes them just-in-time.
|
||||
"""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
@@ -158,7 +159,8 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -259,7 +261,8 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -430,7 +433,8 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
"""Shared base for online FP8 MoE methods. Loads fp16/bf16 checkpoint
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
weights onto meta device and materializes them just-in-time.
|
||||
"""
|
||||
|
||||
# Declared here for mypy; actual values are set in __init__.
|
||||
fp8_backend: "Fp8MoeBackend"
|
||||
@@ -551,7 +555,8 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
|
||||
class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
"""Online tensorwise FP8 MoE quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -608,7 +613,8 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
|
||||
class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
"""Online blockwise FP8 MoE quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -126,7 +126,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
revision: str | None = None,
|
||||
):
|
||||
"""Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
|
||||
|
||||
if hf_config is None:
|
||||
return
|
||||
|
||||
@@ -162,13 +161,13 @@ class QuarkConfig(QuantizationConfig):
|
||||
def apply_vllm_mapper( # noqa: B027
|
||||
self, hf_to_vllm_mapper: "WeightsMapper"
|
||||
):
|
||||
"""
|
||||
Interface for models to update module names referenced in
|
||||
"""Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the vllm model structure
|
||||
|
||||
Args:
|
||||
hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
|
||||
"""
|
||||
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
|
||||
|
||||
@@ -657,8 +656,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
input_quant: dict[str, Any] | None,
|
||||
allow_static_fp8: bool = False,
|
||||
) -> QuantKeyMatch:
|
||||
"""
|
||||
This check returns True only if it is an OCP-MX weight quantization.
|
||||
"""This check returns True only if it is an OCP-MX weight quantization.
|
||||
The activation can be FP16/BF16, OCP MXFP4, MXFP8, FP8.
|
||||
The rationale for checking only the weight type is that
|
||||
the model loading concept and process primarily concerns the weights themselves.
|
||||
@@ -984,8 +982,7 @@ class QuarkLinearMethod(LinearMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
"""
|
||||
Use the CompressedTensorsScheme associated with each layer to create
|
||||
"""Use the CompressedTensorsScheme associated with each layer to create
|
||||
the necessary parameters for the layer. See LinearMethodBase for param
|
||||
details
|
||||
"""
|
||||
@@ -1006,8 +1003,7 @@ class QuarkLinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Use the output of create_weights and the CompressedTensorsScheme
|
||||
"""Use the output of create_weights and the CompressedTensorsScheme
|
||||
associated with the layer to apply the forward pass with the
|
||||
layer input. See LinearMethodBase for param details
|
||||
|
||||
@@ -1020,9 +1016,7 @@ class QuarkLinearMethod(LinearMethodBase):
|
||||
|
||||
|
||||
class QuarkKVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Supports loading kv-cache scaling factors from quark checkpoints.
|
||||
"""
|
||||
"""Supports loading kv-cache scaling factors from quark checkpoints."""
|
||||
|
||||
def __init__(self, quant_config: QuarkConfig):
|
||||
self.validate_kv_cache_config(quant_config.kv_cache_config)
|
||||
@@ -1030,12 +1024,12 @@ class QuarkKVCacheMethod(BaseKVCacheMethod):
|
||||
|
||||
@staticmethod
|
||||
def validate_kv_cache_config(kv_cache_config: dict[str, Any] | None):
|
||||
"""
|
||||
Validator for the kv cache configuration. Useful for controlling the
|
||||
"""Validator for the kv cache configuration. Useful for controlling the
|
||||
kv cache quantization schemes, that are being supported in vLLM
|
||||
|
||||
Args:
|
||||
kv_cache_config: the quark kv cache scheme
|
||||
|
||||
"""
|
||||
if kv_cache_config is None:
|
||||
return
|
||||
|
||||
@@ -1748,10 +1748,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
set_weight_attrs(w2_input_scale_2, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
|
||||
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
|
||||
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
|
||||
if self.moe.is_act_and_mul and not torch.allclose(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
|
||||
@@ -31,8 +31,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkNVFP4(QuarkScheme):
|
||||
"""
|
||||
Quark NVFP4 quantization scheme.
|
||||
"""Quark NVFP4 quantization scheme.
|
||||
|
||||
Supports loading NVFP4 checkpoints with the following structure:
|
||||
- weight: uint8, shape [out_features, in_features // 2] (packed FP4)
|
||||
|
||||
@@ -11,8 +11,7 @@ __all__ = ["QuarkScheme"]
|
||||
|
||||
|
||||
class QuarkScheme(ABC):
|
||||
"""
|
||||
Abstract class used to describe the weight creation and forward pass
|
||||
"""Abstract class used to describe the weight creation and forward pass
|
||||
of different quantization schemes supported by Quark.
|
||||
"""
|
||||
|
||||
@@ -36,25 +35,19 @@ class QuarkScheme(ABC):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
"""
|
||||
Get minimum device capability.
|
||||
"""
|
||||
"""Get minimum device capability."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(self, *args, **kwargs):
|
||||
"""
|
||||
Weight creation for the particular scheme. Inputs to this function
|
||||
|
||||
"""
|
||||
"""Weight creation for the particular scheme. Inputs to this function."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply_weights(
|
||||
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
):
|
||||
"""
|
||||
Run the forward pass for the particular scheme. This is where
|
||||
"""Run the forward pass for the particular scheme. This is where
|
||||
scheme-specific dequant/quant steps/kernels should be applied.
|
||||
|
||||
Args:
|
||||
@@ -62,13 +55,13 @@ class QuarkScheme(ABC):
|
||||
other parameters relevant to the particular scheme.
|
||||
x: input to the layer
|
||||
bias: bias parameter
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
"""
|
||||
Called after weight loading is complete for any cleanup that
|
||||
"""Called after weight loading is complete for any cleanup that
|
||||
needs to occur.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -33,8 +33,7 @@ OCP_MX_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
class QuarkW4A8_MXFP4_FP8(QuarkScheme):
|
||||
"""
|
||||
- Weights: MXFP4 with E8M0 scales per block of 32
|
||||
"""- Weights: MXFP4 with E8M0 scales per block of 32
|
||||
- Activations: FP8 E4M3 (static per-tensor quantization)
|
||||
|
||||
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
|
||||
|
||||
@@ -31,8 +31,7 @@ def triton_scale_swizzle(
|
||||
BLOCK_ROWS: tl.constexpr,
|
||||
BLOCK_COLS: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Rearranges tensor data from row-major to block-scaled swizzle format.
|
||||
"""Rearranges tensor data from row-major to block-scaled swizzle format.
|
||||
|
||||
Args:
|
||||
scale_ptr: Pointer to the input scale tensor
|
||||
@@ -43,6 +42,7 @@ def triton_scale_swizzle(
|
||||
output_block_stride: Stride between blocks in the output tensor
|
||||
BLOCK_ROWS: Number of rows in a tile (compile-time constant)
|
||||
BLOCK_COLS: Number of columns in a tile (compile-time constant)
|
||||
|
||||
"""
|
||||
pid_row = tl.program_id(0)
|
||||
pid_col = tl.program_id(1)
|
||||
@@ -86,8 +86,7 @@ def triton_scale_swizzle(
|
||||
|
||||
@torch.library.custom_op("vllm::triton_mx_block_rearrange", mutates_args=())
|
||||
def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Rearranges an E8M0 tensor scale from row-major format to
|
||||
"""Rearranges an E8M0 tensor scale from row-major format to
|
||||
block-scaled swizzle format.
|
||||
|
||||
This format is suitable for Tmem as described in NVIDIA documentation:
|
||||
@@ -98,6 +97,7 @@ def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Returns:
|
||||
Rearranged tensor in block-scaled swizzle format
|
||||
|
||||
"""
|
||||
assert scale_tensor.element_size() == 1, (
|
||||
"Expected element size to be 1 byte (8 bits)"
|
||||
@@ -154,8 +154,7 @@ def _triton_mx_block_rearrange_fake(scale_tensor: torch.Tensor) -> torch.Tensor:
|
||||
def to_blocked(
|
||||
input_matrix: torch.Tensor, backend: Literal["torch", "triton"] = "triton"
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Rearrange a large matrix by breaking it into blocks and applying
|
||||
"""Rearrange a large matrix by breaking it into blocks and applying
|
||||
the rearrangement pattern.
|
||||
|
||||
See:
|
||||
@@ -167,6 +166,7 @@ def to_blocked(
|
||||
|
||||
Returns:
|
||||
Rearranged flattened tensor of size (32*cdiv(H,128) * 16*cdiv(W,4))
|
||||
|
||||
"""
|
||||
if backend == "triton":
|
||||
return triton_mx_block_rearrange(input_matrix).flatten()
|
||||
|
||||
@@ -74,8 +74,7 @@ def torchao_version_at_least(torchao_version: str) -> bool:
|
||||
|
||||
|
||||
def should_skip(prefix: str, skip_modules: list[str]) -> bool:
|
||||
"""
|
||||
Robust skipping logic:
|
||||
"""Robust skipping logic:
|
||||
should_skip("model.model.layers.1.q_proj",
|
||||
["model.model.layers.1.q_proj"]) # True
|
||||
should_skip("model.model.layers.10.o_proj", ["o_proj"]) -> True
|
||||
@@ -163,14 +162,14 @@ class TorchAOConfig(QuantizationConfig):
|
||||
|
||||
@staticmethod
|
||||
def get_config_filenames() -> list[str]:
|
||||
"""torchao doesn't require additional config files, we use
|
||||
"""Torchao doesn't require additional config files, we use
|
||||
`config.json` from huggingface: `model_config.hf_config`
|
||||
"""
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "TorchAOConfig":
|
||||
"""Create the quant config from an hf model config"""
|
||||
"""Create the quant config from an hf model config."""
|
||||
try:
|
||||
from torchao.core.config import config_from_dict
|
||||
except ImportError as err:
|
||||
@@ -287,12 +286,13 @@ class TorchAOConfig(QuantizationConfig):
|
||||
def torchao_quantize_param_data(
|
||||
param: torch.Tensor, torchao_config: Any
|
||||
) -> torch.nn.Parameter:
|
||||
"""Quantize a Tensor with torchao quantization specified by torchao_config
|
||||
"""Quantize a Tensor with torchao quantization specified by torchao_config.
|
||||
|
||||
Args:
|
||||
param: weight parameter of the linear module
|
||||
torchao_config: type of quantization and their arguments we want to
|
||||
use to quantize the Tensor
|
||||
|
||||
"""
|
||||
from torchao.core.config import AOBaseConfig
|
||||
from torchao.quantization import quantize_
|
||||
@@ -322,6 +322,7 @@ class TorchAOLinearMethod(LinearMethodBase):
|
||||
Args:
|
||||
quant_config: The torchao quantization config, a string that encodes
|
||||
the type of quantization and all relevant arguments.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: TorchAOConfig):
|
||||
|
||||
@@ -45,6 +45,7 @@ def solve_lloyd_max(
|
||||
Returns:
|
||||
centroids: Sorted tensor of 2^bits optimal centroids.
|
||||
boundaries: Sorted tensor of 2^bits - 1 decision boundaries.
|
||||
|
||||
"""
|
||||
n_levels = 2**bits
|
||||
sigma2 = 1.0 / d
|
||||
|
||||
@@ -83,6 +83,7 @@ class TurboQuantConfig:
|
||||
norm_correction: Re-normalize centroid vectors to unit norm before
|
||||
inverse rotation during dequant. Fixes quantization-induced norm
|
||||
distortion, improving PPL by ~0.8% at 4-bit.
|
||||
|
||||
"""
|
||||
|
||||
head_dim: int = 128
|
||||
|
||||
@@ -63,6 +63,7 @@ def is_shared_expert_quant_fse_compatible(
|
||||
|
||||
Returns:
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
|
||||
"""
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
@@ -303,13 +304,11 @@ def is_equal_or_regex_match(
|
||||
check_contains: bool = False,
|
||||
use_fnmatch: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Checks whether a value is exactly equal or a regex match for target
|
||||
"""Checks whether a value is exactly equal or a regex match for target
|
||||
if target starts with 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
If use_fnmatch is set, supports shell-style patterns in target.
|
||||
"""
|
||||
|
||||
if target.startswith("re:"):
|
||||
pattern = target[3:]
|
||||
if re.match(pattern, value):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
|
||||
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path"""
|
||||
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path."""
|
||||
|
||||
import functools
|
||||
|
||||
@@ -37,8 +37,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
|
||||
gemm2_weights: torch.Tensor,
|
||||
gemm2_scales: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Prepare MxInt4 weights for TRT-LLM kernel.
|
||||
"""Prepare MxInt4 weights for TRT-LLM kernel.
|
||||
|
||||
Input:
|
||||
gemm1_weights: [num_experts, 2*intermediate_size, hidden_size//8] int32
|
||||
@@ -51,6 +50,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
|
||||
Returns:
|
||||
Dict with keys 'gemm1_weights', 'gemm1_scales', 'gemm2_weights',
|
||||
'gemm2_scales' containing shuffled/packed tensors ready for kernel
|
||||
|
||||
"""
|
||||
from flashinfer import block_scale_interleave
|
||||
from flashinfer.fused_moe import (
|
||||
@@ -192,8 +192,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
routing_method_type: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
"""Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
|
||||
Args:
|
||||
x: Input hidden states. dtype: bfloat16
|
||||
@@ -209,11 +208,14 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
ep_rank: Expert parallelism rank (default: 0)
|
||||
num_expert_group: Number of expert groups (default: None -> 0)
|
||||
topk_group: Top-k within groups (default: None -> 0)
|
||||
routing_replay_out: Optional buffer receiving the routing decisions
|
||||
so they can be replayed later
|
||||
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
||||
routing_method_type: FlashInfer RoutingMethodType enum value
|
||||
|
||||
Returns:
|
||||
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
||||
|
||||
"""
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
@@ -59,7 +59,7 @@ def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
|
||||
def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
|
||||
gemm1_weights: torch.Tensor, gemm2_weights: torch.Tensor, is_gated_activation: bool
|
||||
):
|
||||
"""Shuffle weights for FI TRT-LLM Format"""
|
||||
"""Shuffle weights for FI TRT-LLM Format."""
|
||||
from flashinfer import reorder_rows_for_gated_act_gemm, shuffle_matrix_a
|
||||
|
||||
epilogue_tile_m = 128
|
||||
@@ -210,7 +210,6 @@ def align_fp4_moe_weights_for_fi(
|
||||
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
|
||||
gate/up and down projection weights along the intermediate dim.
|
||||
"""
|
||||
|
||||
# Current local intermediate size (per partition) is the K dimension of
|
||||
# the down projection.
|
||||
num_experts, hidden_size, intermediate = w2.shape
|
||||
@@ -316,7 +315,6 @@ def align_moe_weights_for_fi(
|
||||
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
|
||||
gate/up and down projection weights along the intermediate dim.
|
||||
"""
|
||||
|
||||
# Current local intermediate size (per partition) is the K dimension of
|
||||
# the down projection.
|
||||
num_experts, hidden_size, intermediate = w2.shape
|
||||
@@ -453,8 +451,7 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
w2_input_scale: torch.Tensor | None,
|
||||
is_trtllm: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Convert Fp8 MoE weights to flashinfer kernel format
|
||||
"""Convert Fp8 MoE weights to flashinfer kernel format.
|
||||
|
||||
Note that for trtllm we update the model state dict
|
||||
with the scale format needed for these kernels.
|
||||
@@ -462,7 +459,6 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
Note that for per-tensor, we update the layer's
|
||||
intermediate size if the weights needed padding.
|
||||
"""
|
||||
|
||||
assert hasattr(layer.moe_config, "is_act_and_mul")
|
||||
block_quant = (
|
||||
hasattr(layer, "weight_block_size") and layer.weight_block_size is not None
|
||||
|
||||
@@ -69,7 +69,8 @@ def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""This function quantizes input values to float8 values "
|
||||
"with tensor-wise quantization."""
|
||||
"with tensor-wise quantization.
|
||||
"""
|
||||
dtype = current_platform.fp8_dtype() if dtype is None else dtype
|
||||
finfo = torch.finfo(dtype)
|
||||
min_val, max_val = x.aminmax()
|
||||
@@ -347,12 +348,10 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
BLOCK_N: tl.constexpr,
|
||||
):
|
||||
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
|
||||
"""
|
||||
Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
||||
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
|
||||
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
|
||||
the outputs tensors at the right positions.
|
||||
"""
|
||||
|
||||
pid_m = tl.program_id(0)
|
||||
pid_n = tl.program_id(1)
|
||||
N_2 = N // 2
|
||||
@@ -421,8 +420,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
):
|
||||
"""
|
||||
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
|
||||
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
|
||||
"""
|
||||
GROUP_SIZE = group_size
|
||||
@@ -564,18 +562,23 @@ def per_token_group_quant_fp8(
|
||||
"""Function to perform per-token-group quantization on an input tensor `x`.
|
||||
It converts the tensor values into signed float8 values and returns the
|
||||
quantized tensor along with the scaling factor used for quantization.
|
||||
|
||||
Args:
|
||||
x: The input tensor with ndim >= 2.
|
||||
group_size: The group size used for quantization.
|
||||
eps: The minimum to avoid dividing zero.
|
||||
dtype: The dtype of output tensor. Note that only `torch.float8_e4m3fn`
|
||||
is supported for now.
|
||||
is supported for now.
|
||||
column_major_scales: Outputs scales in column major.
|
||||
tma_aligned_scales: Outputs scales in TMA-aligned layout.
|
||||
out_q: Optional output tensor. If not provided, function will create.
|
||||
use_ue8m0: If True, round each scale down to a power of two (UE8M0
|
||||
format). None defers to the platform default.
|
||||
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
|
||||
scaling factor.
|
||||
|
||||
"""
|
||||
if use_ue8m0 is None:
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
@@ -695,6 +698,7 @@ def per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x_s_packed: Int32 tensor with logical shape
|
||||
[mn, ceil(num_groups_per_row / 4)], laid out with
|
||||
TMA-aligned stride along the packed-K dimension
|
||||
|
||||
"""
|
||||
if use_ue8m0 is None:
|
||||
use_ue8m0 = is_deep_gemm_e8m0_used()
|
||||
@@ -789,7 +793,6 @@ def _w8a8_triton_block_scaled_mm(
|
||||
product) on input tensors `A` and `B` with block-wise quantization, and
|
||||
store the result in output tensor `C`.
|
||||
"""
|
||||
|
||||
pid = tl.program_id(axis=0)
|
||||
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
|
||||
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
|
||||
@@ -842,14 +845,12 @@ def _w8a8_triton_block_scaled_mm(
|
||||
def get_w8a8_block_fp8_configs(
|
||||
N: int, K: int, block_n: int, block_k: int
|
||||
) -> dict[int, Any] | None:
|
||||
"""
|
||||
Return optimized configurations for the w8a8 block fp8 kernel.
|
||||
"""Return optimized configurations for the w8a8 block fp8 kernel.
|
||||
The return value will be a dictionary that maps an irregular grid of
|
||||
batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the
|
||||
kernel on a given batch size bs, the closest batch size in the grid should
|
||||
be picked and the associated configuration chosen to invoke the kernel.
|
||||
"""
|
||||
|
||||
# First look up if an optimized configuration is available in the configs
|
||||
# directory
|
||||
device_name = get_device_name_as_file_name()
|
||||
@@ -889,18 +890,20 @@ def w8a8_triton_block_scaled_mm(
|
||||
quantization.
|
||||
It takes two input tensors `A` and `B` with scales `As` and `Bs`.
|
||||
The output is returned in the specified `output_dtype`.
|
||||
|
||||
Args:
|
||||
A: The input tensor, e.g., activation.
|
||||
B: The input tensor, e.g., weight.
|
||||
As: The per-token-group quantization scale for `A`.
|
||||
Bs: The per-block quantization scale for `B`.
|
||||
block_size: The block size for per-block quantization. It should
|
||||
be 2-dim, e.g., [128, 128].
|
||||
output_dytpe: The dtype of the returned tensor.
|
||||
be 2-dim, e.g., [128, 128].
|
||||
output_dtype: The dtype of the returned tensor.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: The result of matmul.
|
||||
"""
|
||||
|
||||
"""
|
||||
_on_gfx1250 = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1250
|
||||
@@ -1022,6 +1025,7 @@ def requant_weight_ue8m0_inplace(
|
||||
with shape `(..., M // block_size[0], K // block_size[1])`.
|
||||
block_size: 2-element iterable `[block_m, block_k]` describing the
|
||||
block quantisation granularity.
|
||||
|
||||
"""
|
||||
if weight.numel() == 0:
|
||||
return
|
||||
@@ -1191,7 +1195,8 @@ def prepare_fp8_moe_layer_for_deepgemm(
|
||||
|
||||
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Pad the weight tensor. This is an optimization on ROCm platform, which
|
||||
can benefit from tensors located far enough from one another in memory"""
|
||||
can benefit from tensors located far enough from one another in memory
|
||||
"""
|
||||
if (
|
||||
envs.VLLM_ROCM_FP8_PADDING
|
||||
and current_platform.is_rocm()
|
||||
@@ -1486,7 +1491,6 @@ def process_fp8_input_tensor_strategy_moe(
|
||||
enable_eplb: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Process moe input scales for tensor-wise quantization strategy."""
|
||||
|
||||
if not all_close_1d(w13_input_scale) or not all_close_1d(w2_input_scale):
|
||||
logger.info_once(
|
||||
"Found input_scales that are not equal for "
|
||||
|
||||
@@ -88,8 +88,7 @@ if has_humming():
|
||||
|
||||
|
||||
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
|
||||
"""
|
||||
Map humming group sizes to QuantKey GroupShape.
|
||||
"""Map humming group sizes to QuantKey GroupShape.
|
||||
|
||||
group_size: elements per group along K (col); 0 means full dimension.
|
||||
group_size_n: elements per group along N (row); 0 means 1 (per-row).
|
||||
@@ -298,7 +297,8 @@ def _humming_input_schema_to_quant_key(
|
||||
schema: "HummingInputSchema",
|
||||
) -> QuantKey | None:
|
||||
"""Convert a HummingInputSchema to a QuantKey. Returns None if
|
||||
the schema represents unquantized (bf16/fp16) inputs."""
|
||||
the schema represents unquantized (bf16/fp16) inputs.
|
||||
"""
|
||||
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
|
||||
return None
|
||||
|
||||
@@ -427,7 +427,8 @@ def convert_linear_layer_to_humming_standard(
|
||||
):
|
||||
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
|
||||
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
|
||||
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
|
||||
"""
|
||||
for name, checkpoint_name in name_map.items():
|
||||
tensor = getattr(layer, checkpoint_name)
|
||||
delattr(layer, checkpoint_name)
|
||||
@@ -691,11 +692,9 @@ def select_humming_moe_experts(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> type[mk.FusedMoEExperts] | None:
|
||||
"""
|
||||
Select the primary Humming MoE Experts class
|
||||
"""Select the primary Humming MoE Experts class
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
if not has_humming():
|
||||
return None
|
||||
|
||||
@@ -809,14 +808,14 @@ def _replace_layer_parameters(
|
||||
tensors: dict[str, torch.Tensor],
|
||||
preserve_bias: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Replace layer parameters for a sublayer with new tensors.
|
||||
"""Replace layer parameters for a sublayer with new tensors.
|
||||
|
||||
Args:
|
||||
layer: The RoutedExperts layer
|
||||
sublayer_name: Name of the sublayer (e.g., "w13", "w2")
|
||||
tensors: Dict of parameter name to tensor
|
||||
preserve_bias: If True, don't delete bias parameters
|
||||
|
||||
"""
|
||||
# Delete old parameters
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
@@ -843,11 +842,11 @@ def _convert_sublayer_to_humming(
|
||||
num_experts: int,
|
||||
param_dtype: torch.dtype,
|
||||
) -> tuple[Any, Any]:
|
||||
"""
|
||||
Convert a sublayer's weights from checkpoint format to Humming format.
|
||||
"""Convert a sublayer's weights from checkpoint format to Humming format.
|
||||
|
||||
Returns:
|
||||
Tuple of (converted_weight_schema, converted_input_schema)
|
||||
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
@@ -931,8 +930,7 @@ def _process_single_sublayer(
|
||||
param_dtype: torch.dtype,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> tuple[Any, Any, "LayerConfig"]:
|
||||
"""
|
||||
Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
|
||||
|
||||
This combines the common logic from convert_to_humming_moe_kernel_format
|
||||
for processing a single sublayer.
|
||||
@@ -951,6 +949,7 @@ def _process_single_sublayer(
|
||||
|
||||
Returns:
|
||||
Tuple of the final weight schema, input schema, and Humming layer config.
|
||||
|
||||
"""
|
||||
from vllm.utils.humming import HummingWeightSchema
|
||||
|
||||
@@ -1005,8 +1004,7 @@ def convert_to_humming_moe_kernel_format(
|
||||
input_schema: Any | None = None,
|
||||
force_weight_schema: Any | None = None,
|
||||
) -> dict[str, "LayerConfig"]:
|
||||
"""
|
||||
Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
"""Convert MoE weights from checkpoint format to Humming kernel format.
|
||||
|
||||
This function processes weights for each sublayer (w13, w2) by:
|
||||
1. Converting from checkpoint format to humming format if needed
|
||||
@@ -1032,8 +1030,8 @@ def convert_to_humming_moe_kernel_format(
|
||||
- Modifies layer parameters in place
|
||||
- Sets layer.weight_schemas and layer.input_schemas
|
||||
- Sets layer.humming_configs for quant config construction
|
||||
"""
|
||||
|
||||
"""
|
||||
# Build schemas from quant_config if not provided
|
||||
has_bias = layer.moe_config.has_bias
|
||||
num_experts = layer.moe_config.num_local_experts
|
||||
|
||||
@@ -178,6 +178,7 @@ def per_token_group_quant_int8(
|
||||
Returns:
|
||||
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
|
||||
scaling factor for quantization.
|
||||
|
||||
"""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` cannot be divisible by `group_size`"
|
||||
|
||||
@@ -17,8 +17,7 @@ def query_machete_supported_quant_types(zero_points: bool) -> list[ScalarType]:
|
||||
|
||||
|
||||
def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
"""
|
||||
Queries the supported group sizes for Machete based on the activation type.
|
||||
"""Queries the supported group sizes for Machete based on the activation type.
|
||||
|
||||
Args:
|
||||
act_type: The activation data type (torch.float16, torch.bfloat16).
|
||||
@@ -27,6 +26,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
|
||||
A list of supported group sizes. The group size must
|
||||
be divisible by `TileShapeK = 128 * 8 // num_bits(act_type)`.
|
||||
-1 indicates per-channel quantization.
|
||||
|
||||
"""
|
||||
if act_type in [torch.float16, torch.bfloat16]:
|
||||
return [-1, 64, 128]
|
||||
|
||||
@@ -245,7 +245,8 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
|
||||
|
||||
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
|
||||
"""Recover the (size_n, size_k) a Marlin weight was repacked with
|
||||
(including any tile padding) from its packed shape."""
|
||||
(including any tile padding) from its packed shape.
|
||||
"""
|
||||
pack_factor = 32 // num_bits
|
||||
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
|
||||
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
|
||||
@@ -256,7 +257,8 @@ def marlin_pad_qweight(
|
||||
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
|
||||
gptq_marlin_repack."""
|
||||
gptq_marlin_repack.
|
||||
"""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return qweight
|
||||
pack_factor = size_k // qweight.size(0)
|
||||
@@ -274,7 +276,8 @@ def marlin_pad_scales(
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad weight scales (num_groups, size_n); call before
|
||||
marlin_permute_scales and pass the padded extents to it."""
|
||||
marlin_permute_scales and pass the padded extents to it.
|
||||
"""
|
||||
if (padded_n, padded_k) == (size_n, size_k):
|
||||
return scales
|
||||
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
|
||||
@@ -397,8 +400,7 @@ def check_moe_marlin_supports_layer(
|
||||
|
||||
|
||||
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
|
||||
"""
|
||||
Given Marlin packed weight matrices w1_packed, and w2_packed,
|
||||
"""Given Marlin packed weight matrices w1_packed, and w2_packed,
|
||||
return the MoE intermediate size N
|
||||
"""
|
||||
marlin_tile_size = 16
|
||||
|
||||
@@ -41,8 +41,8 @@ def _nvfp4_compute_scale_factor(
|
||||
) -> float:
|
||||
"""Compute the power-of-2 scale_factor needed so that all non-zero
|
||||
values in marlin_scales * 2^7 are >= 2 after rescaling.
|
||||
Returns a Python float (power of 2, >= 1.0)."""
|
||||
|
||||
Returns a Python float (power of 2, >= 1.0).
|
||||
"""
|
||||
# Since half has a smaller dynamic range compared to bfloat16,
|
||||
# no rescaling is applied here if active dtype is half.
|
||||
if a_dtype is not None and a_dtype == torch.half:
|
||||
@@ -76,9 +76,12 @@ def nvfp4_marlin_process_scales(
|
||||
given value is used directly. The caller is responsible for
|
||||
dividing ``global_scale`` by the returned ``scale_factor`` to
|
||||
preserve numerical correctness.
|
||||
a_dtype: dtype of the activations the scales will be used with. None
|
||||
keeps the dtype of ``marlin_scales``.
|
||||
|
||||
Returns:
|
||||
A tuple of (processed_scales, scale_factor).
|
||||
|
||||
"""
|
||||
if not (marlin_scales >= 0).all():
|
||||
logger.warning_once(
|
||||
@@ -373,7 +376,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w13(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
|
||||
tensor to padded_N rows."""
|
||||
tensor to padded_N rows.
|
||||
"""
|
||||
if padded_N == N:
|
||||
return x
|
||||
x = x.view(E, num_shards, N, x.size(-1))
|
||||
@@ -382,7 +386,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
|
||||
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
|
||||
tensor."""
|
||||
tensor.
|
||||
"""
|
||||
if padded_N == N:
|
||||
return x
|
||||
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
|
||||
|
||||
@@ -217,7 +217,8 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
|
||||
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
|
||||
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
|
||||
"""
|
||||
if padded_n == n:
|
||||
return x
|
||||
e = x.size(0)
|
||||
@@ -241,14 +242,12 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Shuffle weights and scales into marlin format.
|
||||
"""Shuffle weights and scales into marlin format.
|
||||
|
||||
Note that this function has the side effect of adding a `workspace`
|
||||
attribute to the layer. This `workspace` does not need to be
|
||||
registered as a Parameter as it is not used during weight reloading.
|
||||
"""
|
||||
|
||||
logger.warning_once(
|
||||
"Your GPU does not have native support for FP8 computation but "
|
||||
"FP8 quantization is being used. Weight-only FP8 compression will "
|
||||
@@ -372,9 +371,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
def pack_fp8_to_int32(
|
||||
fp8_tensor: torch.Tensor, size_k_first: bool = True
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Repack FP8 weights to gptq format (packed int32 elements)
|
||||
"""
|
||||
"""Repack FP8 weights to gptq format (packed int32 elements)."""
|
||||
assert fp8_tensor.dtype == torch.float8_e4m3fn
|
||||
assert fp8_tensor.ndim == 2
|
||||
|
||||
@@ -524,6 +521,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
|
||||
Returns:
|
||||
(w13, w2, w13_scale, w2_scale) in Marlin format.
|
||||
|
||||
"""
|
||||
group_size = 32
|
||||
e = w13.shape[0]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility functions used for tests and benchmarks"""
|
||||
"""Utility functions used for tests and benchmarks."""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
@@ -28,7 +28,7 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
|
||||
|
||||
|
||||
def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
|
||||
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
|
||||
"""Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel."""
|
||||
assert has_triton_kernels()
|
||||
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
|
||||
from triton_kernels.numerics import InFlexData
|
||||
@@ -374,8 +374,7 @@ def downcast_to_mxfp(
|
||||
BLOCK_OUT_DIM: int = 128,
|
||||
BLOCK_QUANT_DIM: int = 32,
|
||||
):
|
||||
"""
|
||||
Convert the src weights to MXFP4. The src weight is quantized along the
|
||||
"""Convert the src weights to MXFP4. The src weight is quantized along the
|
||||
axis dimension into packed e2m1 values (torch.uint8, two values per
|
||||
byte), so the size of that dimension in the output is half of the
|
||||
logical (unpacked) size.
|
||||
|
||||
@@ -657,8 +657,7 @@ def ref_nvfp4_quant(x, global_scale, block_size):
|
||||
def ref_nvfp4_quant_dequant(
|
||||
x: torch.Tensor, global_scale: torch.Tensor, block_size: int
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
NVFP4 quantize-dequantize operation.
|
||||
"""NVFP4 quantize-dequantize operation.
|
||||
|
||||
`global_scale` is expected to have a single element.
|
||||
"""
|
||||
|
||||
@@ -11,18 +11,15 @@ from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Pad and block-interleave the FP4 block-scales so that they match the data
|
||||
"""Pad and block-interleave the FP4 block-scales so that they match the data
|
||||
layout expected by the CUTLASS / FlashInfer kernels.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
scale: torch.Tensor
|
||||
Args:
|
||||
scale: The FP4 block-scale tensor to swizzle.
|
||||
|
||||
Returns:
|
||||
The swizzled tensor with the same logical shape as `scale`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
torch.Tensor
|
||||
The swizzled tensor with the same logical shape as *scale*.
|
||||
"""
|
||||
assert scale.dtype == torch.float8_e4m3fn, (
|
||||
"swizzle_blockscale expects the input tensor to be in "
|
||||
@@ -65,8 +62,7 @@ def pad_nvfp4_weight_for_cutlass(
|
||||
weight: torch.Tensor,
|
||||
alignment: int = 32,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""
|
||||
Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
|
||||
"""Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
|
||||
the alignment constraints required by CUTLASS / FlashInfer FP4 kernels.
|
||||
|
||||
CUTLASS FP4 kernel requires both K and N matrix dimensions to be divisible
|
||||
@@ -99,8 +95,7 @@ def pad_nvfp4_weight_for_cutlass(
|
||||
|
||||
|
||||
def nvfp4_weight_padding_bytes(layer: torch.nn.Module) -> int:
|
||||
"""
|
||||
Compute K-dim padding (packed bytes) of an NVFP4 weight,
|
||||
"""Compute K-dim padding (packed bytes) of an NVFP4 weight,
|
||||
Aligned with `pad_nvfp4_weight_for_cutlass` above.
|
||||
"""
|
||||
return layer.weight.shape[1] - layer.input_size_per_partition // 2
|
||||
@@ -110,8 +105,7 @@ def pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4: torch.Tensor,
|
||||
weights_padding_bytes: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Pad packed FP4 activations to match the K-dimension padding applied to weights.
|
||||
"""Pad packed FP4 activations to match the K-dimension padding applied to weights.
|
||||
The padding is in bytes (tensor dimension), not FP4 elements.
|
||||
"""
|
||||
if weights_padding_bytes > 0:
|
||||
@@ -123,9 +117,7 @@ def slice_nvfp4_output(
|
||||
out: torch.Tensor,
|
||||
output_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Slice the output tensor to remove padding in N dimension if weight was padded.
|
||||
"""
|
||||
"""Slice the output tensor to remove padding in N dimension if weight was padded."""
|
||||
if out.shape[-1] != output_size:
|
||||
return out[..., :output_size].contiguous()
|
||||
return out
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This file is used for /tests and /benchmarks"""
|
||||
"""This file is used for /tests and /benchmarks."""
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
@@ -109,8 +109,7 @@ class _GroupShape(NamedTuple):
|
||||
|
||||
|
||||
class GroupShape(_GroupShape):
|
||||
"""
|
||||
This class describes the quantization group shape.
|
||||
"""This class describes the quantization group shape.
|
||||
It includes static members for common shapes (per-tensor, per-token).
|
||||
"""
|
||||
|
||||
@@ -139,8 +138,7 @@ GroupShape.PER_CHANNEL = GroupShape(-1, 1)
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScaleDesc:
|
||||
"""
|
||||
Class for describing a single quantization scaling factor.
|
||||
"""Class for describing a single quantization scaling factor.
|
||||
dtype: data type of the scale
|
||||
static: static scale if True, dynamic if False
|
||||
group_shape: group shape of the scale
|
||||
@@ -166,8 +164,7 @@ class ScaleDesc:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuantKey:
|
||||
"""
|
||||
Class for identifying the type of quantization.
|
||||
"""Class for identifying the type of quantization.
|
||||
dtype: quantized data type
|
||||
scale: scale descriptor
|
||||
scale2: second-level scale descriptor
|
||||
@@ -378,8 +375,7 @@ def prep_scale_for_group_broadcast(
|
||||
x: torch.Tensor,
|
||||
group_shape: GroupShape | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Prepare the input quantization scale for group broadcasting.
|
||||
"""Prepare the input quantization scale for group broadcasting.
|
||||
|
||||
Args:
|
||||
scale: The scale tensor (scalar or 1D).
|
||||
@@ -388,6 +384,7 @@ def prep_scale_for_group_broadcast(
|
||||
|
||||
Returns:
|
||||
scale reshaped for correct broadcasting.
|
||||
|
||||
"""
|
||||
if scale.numel() == 1:
|
||||
# For per-tensor quant, keep the scale as a scalar (not reshaped to (1, 1)).
|
||||
@@ -432,13 +429,13 @@ def scaled_quantize(
|
||||
quant_dtype: torch.dtype,
|
||||
compute_dtype: torch.dtype | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Args:
|
||||
x: Input tensor to quantize
|
||||
group_shape: Shape of quantization groups
|
||||
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
|
||||
compute_dtype: Optional dtype for intermediate computations.
|
||||
If None, uses input dtype. Use torch.float32 for higher precision.
|
||||
"""Args:
|
||||
x: Input tensor to quantize
|
||||
group_shape: Shape of quantization groups
|
||||
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
|
||||
compute_dtype: Optional dtype for intermediate computations.
|
||||
If None, uses input dtype. Use torch.float32 for higher precision.
|
||||
|
||||
"""
|
||||
group_shape = _normalize_quant_group_shape(x, group_shape)
|
||||
assert quant_dtype.is_floating_point, (
|
||||
@@ -506,7 +503,7 @@ def get_attribute_fallback(obj, attributes: list[str]):
|
||||
def get_and_maybe_dequant_weights(
|
||||
layer: "LinearBase", out_dtype: torch.dtype = torch.float32
|
||||
):
|
||||
"""Return layer's unquantized weights in [out, in] layout"""
|
||||
"""Return layer's unquantized weights in [out, in] layout."""
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
@@ -924,8 +921,7 @@ def awq_pack(
|
||||
def convert_bf16_scales_to_fp8(
|
||||
quant_fp8: Callable, scales: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
|
||||
"""Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
|
||||
expected by W4A8 GEMM kernels.
|
||||
"""
|
||||
assert scales.is_contiguous(), (
|
||||
@@ -949,9 +945,7 @@ def convert_bf16_scales_to_fp8(
|
||||
|
||||
|
||||
def convert_packed_uint4b8_to_signed_int4_inplace(t: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Convert int4b8 (packed to int32) to signed int4
|
||||
"""
|
||||
"""Convert int4b8 (packed to int32) to signed int4."""
|
||||
assert t.is_cuda, "tensor must be on gpu"
|
||||
assert t.dtype == torch.int32, f"expected int32 packed weights but got {t.dtype}"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user