Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -82,6 +82,7 @@ def register_quantization_config(quantization: str):
>>>
>>> get_quantization_config("my_quant")
<class 'MyQuantConfig'>
""" # noqa: E501
def _wrapper(quant_config_cls):
@@ -393,6 +393,7 @@ class AutoAWQMarlinLinearMethod(LinearMethodBase):
Args:
quant_config: The AWQ Marlin quantization config.
"""
_kernel_backends_being_used: set[str] = set()
@@ -701,7 +702,6 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
def _setup_kernel(self, layer: RoutedExperts) -> None:
"""Build the FusedMoEKernel for this layer."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
@@ -880,6 +880,7 @@ class AutoAWQLinearMethod(BaseAWQLinearMethod):
Args:
quant_config: The AWQ quantization config.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -305,6 +305,7 @@ class AutoGPTQLinearMethod(LinearMethodBase):
Args:
quant_config: The AutoGPTQ quantization config.
"""
_kernel_backends_being_used: set[str] = set()
@@ -685,7 +686,6 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
def _setup_kernel(self, layer: RoutedExperts) -> None:
"""Build the FusedMoEKernel for this layer."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
@@ -46,21 +46,24 @@ class QuantizeMethodBase(ABC):
):
"""Create weights for a layer.
The weights will be set as attributes of the layer."""
The weights will be set as attributes of the layer.
"""
raise NotImplementedError
@abstractmethod
def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
"""Apply the weights in layer to the input tensor.
Expects create_weights to have been called before on the layer."""
Expects create_weights to have been called before on the layer.
"""
raise NotImplementedError
# Not required functions
def embedding(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor:
"""Gather embeddings in the layer based on indices in the input tensor.
Expects create_weights to have been called before on the layer."""
Expects create_weights to have been called before on the layer.
"""
raise NotImplementedError
# Not required functions
@@ -73,7 +76,8 @@ class QuantizeMethodBase(ABC):
Quantization methods that need special weight handling (e.g. repacked
weights) override this.
Expects create_weights to have been called before on the layer."""
Expects create_weights to have been called before on the layer.
"""
layer.weight = embed_tokens.weight
return layer
@@ -86,8 +90,7 @@ class QuantizeMethodBase(ABC):
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
"""
Not all quant methods have embedding implemented, so we need to check that
"""Not all quant methods have embedding implemented, so we need to check that
it exists for our given method. We check this by making sure the function
has been changed from the base implementation.
"""
@@ -158,8 +161,7 @@ class QuantizationConfig(ABC):
user_quant: str | None,
hf_config: Any = None,
) -> QuantizationMethods | None:
"""
Detects if this quantization method can support a given checkpoint
"""Detects if this quantization method can support a given checkpoint
format by overriding the user specified quantization method --
this method should only be overwritten by subclasses in exceptional
circumstances.
@@ -169,6 +171,7 @@ class QuantizationConfig(ABC):
user_quant: The user-specified quantization method string.
hf_config: The HuggingFace model config object (e.g. for
model_type checks). May be None if not available.
"""
return None
@@ -203,6 +206,7 @@ class QuantizationConfig(ABC):
Returns:
The quantize method. None if the given layer doesn't support quant
method.
"""
raise NotImplementedError
@@ -251,13 +255,13 @@ class QuantizationConfig(ABC):
def apply_vllm_mapper( # noqa: B027
self, hf_to_vllm_mapper: "WeightsMapper"
):
"""
Interface for models to update module names referenced in
"""Interface for models to update module names referenced in
quantization configs in order to reflect the vllm model structure
Args:
hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
"""
# TODO (@kylesayrs): add implementations for all subclasses
pass
@@ -268,14 +272,14 @@ class QuantizationConfig(ABC):
hf_config: PretrainedConfig | None = None,
revision: str | None = None,
):
"""
Interface to update values after config initialization.
"""Interface to update values after config initialization.
Args:
model_name: The name of the model
hf_config: The Hugging Face config of the model
revision: The revision of the model
Returns:
"""
# TODO: revision is never passed currently in vllm.py,
# but is used in subclasses, should we remove this parameter?
@@ -121,8 +121,7 @@ class CompressedTensorsConfig(QuantizationConfig):
return "compressed-tensors"
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
"""
Transform layer paths in config targets to match vLLM's naming.
"""Transform layer paths in config targets to match vLLM's naming.
The WeightsMapper is designed for weight paths, but some backends
(e.g. transformers) use broad prefix mappings like "" -> "model."
@@ -214,8 +213,7 @@ class CompressedTensorsConfig(QuantizationConfig):
return None
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
"""
Helper function to update target_scheme_map
"""Helper function to update target_scheme_map
since linear layers get fused into RoutedExperts
targeting 'Linear' needs to also match
RoutedExperts modules.
@@ -269,8 +267,7 @@ class CompressedTensorsConfig(QuantizationConfig):
def _parse_sparsity_config(
cls, config: dict[str, Any]
) -> tuple[dict[str, SparsityCompressionConfig], list[str]]:
"""
Args:
"""Args:
config: The `quantization_config` dictionary from config.json
Returns:
@@ -278,6 +275,7 @@ class CompressedTensorsConfig(QuantizationConfig):
1. A dictionary mapping target layer names to their corresponding
sparsity_config
2. A list of layer names to ignore for sparsity
"""
if not (sparsity_config := config.get(SPARSITY_CONFIG_NAME)):
return dict(), []
@@ -301,13 +299,13 @@ class CompressedTensorsConfig(QuantizationConfig):
def _quantization_scheme_map_from_config(
cls, config: dict[str, Any]
) -> QUANTIZATION_SCHEME_MAP_TYPE:
"""
Args:
"""Args:
config: The `quantization_config` dictionary from config.json
Returns:
A dictionary mapping target layer names to their corresponding
quantization_args for weights and input activations
"""
target_scheme_map: dict[str, Any] = dict()
quant_format = cast(str, config.get("format"))
@@ -667,7 +665,8 @@ class CompressedTensorsConfig(QuantizationConfig):
) -> bool:
"""Weight N-bit INT (pack-quantized for sub-byte, int-quantized for 8-bit)
with static per-tensor INT8 input/output activation quant, applied as a float
fake-quant around a weight-only matmul."""
fake-quant around a weight-only matmul.
"""
is_int_pack_format = format in (
CompressionFormat.pack_quantized.value,
CompressionFormat.int_quantized.value,
@@ -703,7 +702,8 @@ class CompressedTensorsConfig(QuantizationConfig):
format: str | None,
) -> bool:
"""Weight N-bit INT with symmetric dynamic INT activation quant
via Humming kernel."""
via Humming kernel.
"""
if input_quant is None:
return False
is_pack_format = format == CompressionFormat.pack_quantized.value
@@ -895,8 +895,7 @@ class CompressedTensorsConfig(QuantizationConfig):
def get_scheme(
self, layer: torch.nn.Module, layer_name: str | None = None
) -> "CompressedTensorsScheme | None":
"""
compressed-tensors supports non uniform in the following way:
"""compressed-tensors supports non uniform in the following way:
targets of config_groups: There can be N config_groups which each
have a quantization scheme. Each config_group has a list of targets
@@ -907,7 +906,6 @@ class CompressedTensorsConfig(QuantizationConfig):
use the quantization scheme corresponding to the matched target
to select the CompressedTensorsScheme used for inference.
"""
# Use the new get_quant_args method to extract QuantizationArgs
scheme_dict = self.get_scheme_dict(layer, layer_name)
@@ -944,8 +942,7 @@ class CompressedTensorsConfig(QuantizationConfig):
def get_scheme_dict(
self, layer: torch.nn.Module, layer_name: str | None = None
) -> dict[str, QuantizationArgs | str | None] | None:
"""
Extract the QuantizationArgs for a given layer.
"""Extract the QuantizationArgs for a given layer.
Returns:
dict with {
@@ -953,6 +950,7 @@ class CompressedTensorsConfig(QuantizationConfig):
"input_activations": QuantizationArgs | None,
"format": str | None
} | None
"""
# TODO (@kylesayrs): support ignore module names with ct matching utils
if should_ignore_layer(
@@ -1004,8 +1002,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
"""
Use the CompressedTensorsScheme associated with each layer to create
"""Use the CompressedTensorsScheme associated with each layer to create
the necessary parameters for the layer. See LinearMethodBase for param
details
"""
@@ -1026,8 +1023,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: torch.Tensor | None = None,
):
"""
Use the output of create_weights and the CompressedTensorsScheme
"""Use the output of create_weights and the CompressedTensorsScheme
associated with the layer to apply the forward pass with the
layer input. See LinearMethodBase for param details
@@ -1039,8 +1035,7 @@ class CompressedTensorsLinearMethod(LinearMethodBase):
class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
"""
Supports loading kv-cache scaling factors from compressed-tensors
"""Supports loading kv-cache scaling factors from compressed-tensors
checkpoints.
"""
@@ -1050,12 +1045,12 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
@staticmethod
def validate_kv_cache_scheme(kv_cache_scheme: dict[str, Any] | None):
"""
Validator for the kv cache scheme. Useful for controlling the
"""Validator for the kv cache scheme. Useful for controlling the
kv cache quantization schemes, that are being supported in vLLM
Args:
kv_cache_scheme: the compressed-tensors kv cache scheme
"""
if kv_cache_scheme is None:
return
@@ -1091,8 +1086,7 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
)
def create_weights(self, layer: torch.nn.Module):
"""
Initialize placeholder scales and zero points to enable loading of
"""Initialize placeholder scales and zero points to enable loading of
quantized params from compressed-tensors checkpoints.
"""
strategy = None # for backward compatibility
@@ -1199,8 +1193,7 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""
Override the default vLLM placeholder scales with the llm-compressor loaded
"""Override the default vLLM placeholder scales with the llm-compressor loaded
scales. Zero points are not used as only symmetric quantization is supported.
"""
layer._k_scale = layer.k_scale
@@ -37,7 +37,8 @@ def _dequant_gather_kernel(
BLOCK: tl.constexpr,
):
"""Gather embedding rows by token id, unpack int32-packed INT weights, and
dequantize to ``out`` dtype in one pass (no int8 intermediate)."""
dequantize to ``out`` dtype in one pass (no int8 intermediate).
"""
row = tl.program_id(0)
col = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
col_mask = col < hidden
@@ -184,9 +184,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w2_input_scale, extra_weight_attrs)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
"""
Convert NVFP4 MoE weights into kernel format and setup the kernel.
"""
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
# NOTE(rob): wN_weight_packed -> wN_weight is because ModularKernelMethod
# requires this naming convention. However, the name change breaks
# reloading because the state dict no longer matches disk. Once we
@@ -36,8 +36,7 @@ logger = init_logger(__name__)
class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
"""
CPU-only MoE method using dynamic 4-bit matmul kernels on Arm Platform
"""CPU-only MoE method using dynamic 4-bit matmul kernels on Arm Platform
- Weights: int4 (stored as int8 values in [-8,7], packed to uint8 nibbles)
- Scales: Fp32 for Channelwise , bf16 for groupwise quantization
- Bias: Same data type as original weights
@@ -143,8 +143,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_groups_w2: int | None = None,
num_groups_w13: int | None = None,
) -> tuple[int, int, int]:
"""
Get the shape of the weight based on the weight name, number of experts
"""Get the shape of the weight based on the weight name, number of experts
hidden size, intermediate size per partition, number of groups for w2,
and number of groups for w13. Pass in num_groups_w2 and num_groups_w13
for weight scales/zero_points.
@@ -9,33 +9,26 @@ __all__ = ["CompressedTensorsScheme"]
class CompressedTensorsScheme(ABC):
"""
Abstract class used to describe the weight creation and forward pass
"""Abstract class used to describe the weight creation and forward pass
of different quantization schemes supported by CompressedTensors.
"""
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
"""
Get minimum device capability.
"""
"""Get minimum device capability."""
raise NotImplementedError()
@abstractmethod
def create_weights(self, *args, **kwargs):
"""
Weight creation for the particular scheme. Inputs to this function
"""
"""Weight creation for the particular scheme. Inputs to this function."""
raise NotImplementedError()
@abstractmethod
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
):
"""
Run the forward pass for the particular scheme. This is where
"""Run the forward pass for the particular scheme. This is where
scheme-specific dequant/quant steps/kernels should be applied.
Args:
@@ -43,13 +36,13 @@ class CompressedTensorsScheme(ABC):
other parameters relevant to the particular scheme.
x: input to the layer
bias: bias parameter
"""
raise NotImplementedError()
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module):
"""
Called after weight loading is complete for any cleanup that
"""Called after weight loading is complete for any cleanup that
needs to occur.
"""
raise NotImplementedError()
@@ -21,8 +21,7 @@ __all__ = ["CompressedTensorsW4A4Mxfp4"]
class CompressedTensorsW4A4Mxfp4(CompressedTensorsScheme):
"""
Compressed tensors scheme for MXFP4.
"""Compressed tensors scheme for MXFP4.
Supports models quantized with the compressed-tensors mxfp4-pack-quantized
format.
@@ -22,8 +22,7 @@ __all__ = ["CompressedTensorsW8A8Mxfp8"]
class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme):
"""
Compressed tensors scheme for MXFP8 quantization (W8A8).
"""Compressed tensors scheme for MXFP8 quantization (W8A8).
Loads pre-quantized MXFP8 weights from compressed-tensors checkpoints.
Activations are dynamically quantized to MXFP8 at runtime.
@@ -211,7 +211,8 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
def _pack_int_quantized_weight(self, layer: torch.nn.Module) -> None:
"""Normalize an int-quantized (plain int8) weight to the canonical
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect."""
``weight_packed`` int32 + ``weight_shape`` layout the MP kernels expect.
"""
weight = layer.weight
out_features, in_features = weight.shape
packed = pack_to_int32(weight.data.contiguous(), self.num_bits)
@@ -29,8 +29,7 @@ from vllm.platforms import current_platform
class CompressedTensorsLinearTransformMethod(LinearMethodBase):
"""
Wraps `CompressedTensorsLinearMethod` or `UnquantizedLinearMethod` and adds
"""Wraps `CompressedTensorsLinearMethod` or `UnquantizedLinearMethod` and adds
input and output transforms to either side of the original apply method
"""
@@ -240,8 +239,7 @@ def get_schemes_args(
def get_layer_partition_names(
layer_name: str, packed_modules_mapping: dict[str, list[str]]
) -> list[str]:
"""
Get all partition names associated with this layer.
"""Get all partition names associated with this layer.
Names are returned in order of their partition indices.
```python
@@ -251,7 +249,9 @@ def get_layer_partition_names(
"gate_proj",
"up_proj",
]
assert get_layer_partition_names("mlp.down_proj", mapping) == ["down_proj"]"""
assert get_layer_partition_names("mlp.down_proj", mapping) == ["down_proj"]
```
"""
for fused_suffix, part_suffixes in packed_modules_mapping.items():
if layer_name.endswith(fused_suffix):
return [
@@ -23,8 +23,7 @@ from vllm.model_executor.parameter import SharedWeightParameter
class HadamardTransform(torch.nn.Module):
"""
Class which handles weight loading, postprocessing, and application of
"""Class which handles weight loading, postprocessing, and application of
transforms. Meant to be used with `CompressedTensorsLinearTransformMethod`
and attention transforms method (not implemented yet)
"""
@@ -76,8 +76,7 @@ class QutlassNvFP4LinearMethod(CompressedTensorsLinearTransformMethod):
@staticmethod
def _get_flashinfer_gemm_backend(kernel: NvFp4LinearKernel) -> str:
"""
Given a kernel, find the string that is needed to be passed into
"""Given a kernel, find the string that is needed to be passed into
`flashinfer_scaled_fp4_mm`, using
vllm.model_executor.kernels.linear._LINEAR_BACKEND_KERNEL_MAP as source of truth
"""
@@ -76,8 +76,7 @@ def find_matched_target(
targets: Iterable[str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
) -> str | None:
"""
Helper function to look up which "target" in the compressed-tensors
"""Helper function to look up which "target" in the compressed-tensors
config that a layer corresponds to.
Recall that a compressed-tensors configs has a concept of
@@ -98,8 +97,8 @@ def find_matched_target(
module: torch.nn.Module
targets: list of targets to match the layer against
fused_mapping: map from fused layer names to its components
"""
"""
if layer_name is None:
layer_name = ""
@@ -115,8 +114,7 @@ def find_matched_target(
def _find_first_match(
value: str, targets: Iterable[str], check_contains: bool = False
) -> str | None:
"""
Returns first element of target that matches value either
"""Returns first element of target that matches value either
exactly or as a regex after 're:'. If check_contains is set to True,
additionally checks if the target string is contained within the value.
@@ -124,8 +122,8 @@ def _find_first_match(
value: string to compare the list of targets against
targets: list of targets to match the layer against
check_contains: whether or not to do a substring match
"""
"""
for target in targets:
if is_equal_or_regex_match(value, target, check_contains=check_contains):
return target
@@ -137,8 +135,7 @@ def _match_fused_layer(
target_layers: Iterable[str],
fused_mapping: Mapping[str, list[str]],
) -> str | None:
"""
Match a fused layer name to its corresponding individual layer in
"""Match a fused layer name to its corresponding individual layer in
target_layers. Returns first value in fused_mapping which matches targets
Implements an "all" matching strategy where a fused layer matches iff
@@ -154,6 +151,7 @@ def _match_fused_layer(
target_layers = ["model.layers.0.self_attn.q_proj",
"model.layers.0.self_attn.k_proj",
"model.layers.0.self_attn.v_proj"]
"""
# find layer_name in mapping
fused = next((key for key in fused_mapping if layer_name.endswith(key)), None)
@@ -251,6 +251,7 @@ class Fp8LinearMethod(LinearMethodBase):
Args:
quant_config: The quantization config.
"""
supports_pre_processed_weights = True
@@ -490,6 +491,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
Args:
quant_config: The quantization config.
"""
supports_pre_processed_weights = True
@@ -881,9 +883,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
class Fp8KVCacheMethod(BaseKVCacheMethod):
"""
Supports loading kv-cache scaling factors from FP8 checkpoints.
"""
"""Supports loading kv-cache scaling factors from FP8 checkpoints."""
def __init__(self, quant_config: Fp8Config):
super().__init__(quant_config)
@@ -98,6 +98,7 @@ class FPQuantLinearMethod(LinearMethodBase):
Args:
quant_config: The FPQuant quantization config.
"""
def __init__(self, quant_config: FPQuantConfig):
@@ -783,8 +783,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
"""
Apply Humming-quantized MoE computation using the standard kernel flow.
"""Apply Humming-quantized MoE computation using the standard kernel flow.
This method uses FusedMoEKernel.apply() which orchestrates:
1. Preparation (quantization if needed - skipped for Humming via
@@ -86,7 +86,8 @@ class INCConfigParser:
def is_explicitly_configured(name: str) -> bool:
"""Return True if *name* has an explicit entry in extra_config,
either via exact key match or via a regex pattern key."""
either via exact key match or via a regex pattern key.
"""
if not self._config.extra_config:
return False
if name in self._config.extra_config:
@@ -48,7 +48,8 @@ class INCScheme(ABC):
layer_config: "INCLayerConfig",
) -> "FusedMoEMethodBase | None":
"""Optional. Override if this scheme supports MoE.
Default raises NotImplementedError."""
Default raises NotImplementedError.
"""
raise NotImplementedError(
f"{type(self).__name__} does not support MoE layers. "
f"Layer config: {layer_config}"
@@ -28,8 +28,7 @@ _FP8_MIN_SCALING_FACTOR = 1.0 / (_FP8_MAX * 512.0)
# --8<-- [start:quant_fp8]
@CustomOp.register("quant_fp8")
class QuantFP8(CustomOp):
"""
Quantize input tensor to FP8 (per-tensor, per-token, per-channel, or per-group).
"""Quantize input tensor to FP8 (per-tensor, per-token, per-channel, or per-group).
This CustomOp supports both static and dynamic quantization.
"""
@@ -45,18 +44,18 @@ class QuantFP8(CustomOp):
use_ue8m0: bool | None = None, # for Torch compile
compile_native: bool = True,
):
"""
Args:
static: static or dynamic quantization
group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
PER_CHANNEL, or arbitrary block size)
num_token_padding: Pad the token dimension of output to this
size
tma_aligned_scales: For group quantization, output scales in
TMA-aligned layout
column_major_scales: For group quantization, output scales in
column major format
compile_native: Manually compile forward_native if compile mode > None
"""Args:
static: static or dynamic quantization
group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
PER_CHANNEL, or arbitrary block size)
num_token_padding: Pad the token dimension of output to this
size
tma_aligned_scales: For group quantization, output scales in
TMA-aligned layout
column_major_scales: For group quantization, output scales in
column major format
compile_native: Manually compile forward_native if compile mode > None
"""
super().__init__(compile_native=compile_native)
self.static = static
@@ -41,8 +41,7 @@ class KVCacheScaleParameter(torch.nn.Parameter):
class BaseKVCacheMethod(QuantizeMethodBase):
"""
Quant method that adds `_k_scale` and `_v_scale` attributes to the
"""Quant method that adds `_k_scale` and `_v_scale` attributes to the
Attention layer to support loading those scaling factors from checkpoints.
The k/v_scale will be used to:
- quantize k/v_cache entries before saving them to the cache
@@ -50,6 +49,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
Args:
quant_config: the appropriate QuantizationConfig
"""
supports_pre_processed_weights = True
@@ -58,8 +58,7 @@ class BaseKVCacheMethod(QuantizeMethodBase):
self.quant_config = quant_config
def create_weights(self, layer: torch.nn.Module):
"""
Create "weight" (aka q_scale, k_scale and v_scale)
"""Create "weight" (aka q_scale, k_scale and v_scale)
for an attention layer.
"""
# Initialize the Q and KV cache scales to -1.0, an invalid value.
@@ -153,9 +153,7 @@ def algos_owned_by(config_name: str) -> tuple[str, ...]:
class ModelOptKVCacheMethod(BaseKVCacheMethod):
"""
Supports loading kv-cache scaling factors from FP8 or NVFP4 checkpoints.
"""
"""Supports loading kv-cache scaling factors from FP8 or NVFP4 checkpoints."""
def __init__(self, quant_config: "ModelOptQuantConfigBase"):
super().__init__(quant_config)
@@ -178,8 +176,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
self.exclude_modules: list[str] = exclude_modules
def is_layer_excluded(self, prefix: str) -> bool:
"""
Check if a layer should be excluded from quantization.
"""Check if a layer should be excluded from quantization.
Handles both exact matching (for fused layers) and ModelOpt wildcard matching.
@@ -472,8 +469,10 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
"""MoE method for ModelOpt FP8.
Supports loading FP8 checkpoints with static weight scale and
activation scale.
Args:
quant_config: The ModelOpt quantization config.
"""
def __init__(
@@ -818,10 +817,11 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
"""
MoE Method for FP4 Quantization.
"""MoE Method for FP4 Quantization.
Args:
quant_config: NVFP4 Quant Config
"""
supports_pre_processed_weights = True
@@ -851,9 +851,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
)
def uses_weight_scale_2_pattern(self) -> bool:
"""
FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales.
"""
"""FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales."""
return True
def create_weights(
@@ -973,9 +971,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
layer.register_parameter("w2_input_scale", w2_input_scale)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
"""
Convert NVFP4 MoE weights into kernel format and setup the kernel.
"""
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
if is_weights_pre_processed():
if self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM:
raise RuntimeError(
@@ -1992,7 +1988,8 @@ class KNvfp4Static(QuantKeyScheme):
class KNvfp4Dynamic(QuantKeyScheme):
"""NVFP4 activation scheme (W4A4). Has a static global input scale on disk;
the per-group scale is computed at runtime inside the kernel."""
the per-group scale is computed at runtime inside the kernel.
"""
key = kNvfp4Dynamic
@@ -2029,7 +2026,8 @@ class KNvfp4Dynamic(QuantKeyScheme):
class KFp8StaticTensor(QuantKeyScheme):
"""Plain per-tensor static FP8 — bivalent: serves BOTH the weight slot and
the activation slot (W8A8). One key in both QuantSpec slots."""
the activation slot (W8A8). One key in both QuantSpec slots.
"""
key = kFp8StaticTensorSym
@@ -2099,7 +2097,8 @@ class KFp8StaticTensor(QuantKeyScheme):
class KFp8StaticChannel(QuantKeyScheme):
"""Per-channel static FP8 weight (the 'PcPt' weight). Weight-role only —
there is no static per-channel *activation* today."""
there is no static per-channel *activation* today.
"""
key = kFp8StaticTokenSym
@@ -2140,7 +2139,8 @@ class KFp8StaticChannel(QuantKeyScheme):
class KFp8Block128(QuantKeyScheme):
"""128x128 block-static FP8 weight ('PbWo'). Weight-role only. ModelOpt
exports the scale 4-D [out_blk,1,in_blk,1]; process squeezes to 2-D.
No transpose (block kernel keeps [out,in])."""
No transpose (block kernel keeps [out,in]).
"""
key = kFp8Static128BlockSym
@@ -2196,7 +2196,8 @@ class KFp8Block128(QuantKeyScheme):
class KMxfp8Static(QuantKeyScheme):
"""MXFP8 weight: fp8-e4m3 values + per-32-block e8m0 (uint8) scale.
Weight-role only. process is validate-only plus an idempotency guard."""
Weight-role only. process is validate-only plus an idempotency guard.
"""
key = kMxfp8Static
@@ -2271,7 +2272,8 @@ class KDynamicNoParam(QuantKeyScheme):
"""Dynamic activation with no stored scale (W8A8): quantized at runtime in
the kernel. NOT the same as activation=None (weight-only) — init_fp8 needs a
non-None activation key. Activation-role only. Serves the fp8 per-token, fp8
per-block, and mxfp8 dynamic activation keys."""
per-block, and mxfp8 dynamic activation keys.
"""
def create_weights(self, layer, role, ctx, shapes, wl) -> None:
if role is not ACT:
@@ -2297,7 +2299,7 @@ SCHEME_FOR: dict[QuantKey | None, QuantKeyScheme] = {
def maybe_fuse_global_scales(layer) -> None:
"""alpha = input_global_scale * weight_global_scale, presence-gated.
"""Alpha = input_global_scale * weight_global_scale, presence-gated.
W4A4 has both -> computed; W4A16 has no input_global_scale -> skipped.
"""
@@ -229,6 +229,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
Args:
quant_config: The MOE WNA16 (W8A16/W4A16) quantization config.
"""
def __init__(self, quant_config: MoeWNA16Config, moe: "FusedMoEConfig") -> None:
@@ -79,7 +79,8 @@ class Mxfp4Config(QuantizationConfig):
def _make_moe_method(self, moe: FusedMoEConfig) -> FusedMoEMethodBase:
"""MoE method for RoutedExperts. Subclasses override to pick a
checkpoint-specific kernel family."""
checkpoint-specific kernel family.
"""
return Mxfp4MoEMethod(moe)
def get_quant_method(
@@ -139,7 +139,8 @@ def _find_matching_targets(
class OnlineQuantizationConfig(QuantizationConfig):
"""Model-level config for online quantization (quantize fp16/bf16 weights
during model loading, without requiring a pre-quantized checkpoint)."""
during model loading, without requiring a pre-quantized checkpoint).
"""
def __init__(
self,
@@ -227,6 +228,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
Returns:
The matching method class, or None when ``spec`` has no weight
quantization.
"""
if spec is None or spec.weight is None:
return None
@@ -260,6 +262,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
A tuple of source, quantization key string, target pattern, spec,
and method class. Returns None when online quantization does not
apply to the layer.
"""
quant_spec: QuantSpec | None
if self.args.targets is not None:
@@ -313,6 +316,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
A tuple of source, quantization key string, target pattern, spec,
and dispatch table. Returns None when no pattern applies or the
layer is ignored.
"""
assert self.args.targets is not None
ignored = should_ignore_layer(
@@ -114,7 +114,8 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
weights onto meta device and materializes them just-in-time.
"""
uses_meta_device: bool = True
@@ -158,7 +159,8 @@ class OnlineLinearBase(LinearMethodBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
"""
def __init__(self):
super().__init__()
@@ -259,7 +261,8 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
Loads fp16/bf16 weights and quantizes them per-block during loading.
"""
def __init__(self):
super().__init__()
@@ -430,7 +433,8 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
"""Shared base for online FP8 MoE methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
weights onto meta device and materializes them just-in-time.
"""
# Declared here for mypy; actual values are set in __init__.
fp8_backend: "Fp8MoeBackend"
@@ -551,7 +555,8 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online tensorwise FP8 MoE quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
Loads fp16/bf16 weights and quantizes them per-tensor during loading.
"""
def __init__(
self,
@@ -608,7 +613,8 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
"""Online blockwise FP8 MoE quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
Loads fp16/bf16 weights and quantizes them per-block during loading.
"""
def __init__(
self,
@@ -126,7 +126,6 @@ class QuarkConfig(QuantizationConfig):
revision: str | None = None,
):
"""Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
if hf_config is None:
return
@@ -162,13 +161,13 @@ class QuarkConfig(QuantizationConfig):
def apply_vllm_mapper( # noqa: B027
self, hf_to_vllm_mapper: "WeightsMapper"
):
"""
Interface for models to update module names referenced in
"""Interface for models to update module names referenced in
quantization configs in order to reflect the vllm model structure
Args:
hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
"""
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
@@ -657,8 +656,7 @@ class QuarkConfig(QuantizationConfig):
input_quant: dict[str, Any] | None,
allow_static_fp8: bool = False,
) -> QuantKeyMatch:
"""
This check returns True only if it is an OCP-MX weight quantization.
"""This check returns True only if it is an OCP-MX weight quantization.
The activation can be FP16/BF16, OCP MXFP4, MXFP8, FP8.
The rationale for checking only the weight type is that
the model loading concept and process primarily concerns the weights themselves.
@@ -984,8 +982,7 @@ class QuarkLinearMethod(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
"""
Use the CompressedTensorsScheme associated with each layer to create
"""Use the CompressedTensorsScheme associated with each layer to create
the necessary parameters for the layer. See LinearMethodBase for param
details
"""
@@ -1006,8 +1003,7 @@ class QuarkLinearMethod(LinearMethodBase):
x: torch.Tensor,
bias: torch.Tensor | None = None,
):
"""
Use the output of create_weights and the CompressedTensorsScheme
"""Use the output of create_weights and the CompressedTensorsScheme
associated with the layer to apply the forward pass with the
layer input. See LinearMethodBase for param details
@@ -1020,9 +1016,7 @@ class QuarkLinearMethod(LinearMethodBase):
class QuarkKVCacheMethod(BaseKVCacheMethod):
"""
Supports loading kv-cache scaling factors from quark checkpoints.
"""
"""Supports loading kv-cache scaling factors from quark checkpoints."""
def __init__(self, quant_config: QuarkConfig):
self.validate_kv_cache_config(quant_config.kv_cache_config)
@@ -1030,12 +1024,12 @@ class QuarkKVCacheMethod(BaseKVCacheMethod):
@staticmethod
def validate_kv_cache_config(kv_cache_config: dict[str, Any] | None):
"""
Validator for the kv cache configuration. Useful for controlling the
"""Validator for the kv cache configuration. Useful for controlling the
kv cache quantization schemes, that are being supported in vLLM
Args:
kv_cache_config: the quark kv cache scheme
"""
if kv_cache_config is None:
return
@@ -1748,10 +1748,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
set_weight_attrs(w2_input_scale_2, extra_weight_attrs)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
"""
Convert NVFP4 MoE weights into kernel format and setup the kernel.
"""
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
if self.moe.is_act_and_mul and not torch.allclose(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
@@ -31,8 +31,7 @@ logger = init_logger(__name__)
class QuarkNVFP4(QuarkScheme):
"""
Quark NVFP4 quantization scheme.
"""Quark NVFP4 quantization scheme.
Supports loading NVFP4 checkpoints with the following structure:
- weight: uint8, shape [out_features, in_features // 2] (packed FP4)
@@ -11,8 +11,7 @@ __all__ = ["QuarkScheme"]
class QuarkScheme(ABC):
"""
Abstract class used to describe the weight creation and forward pass
"""Abstract class used to describe the weight creation and forward pass
of different quantization schemes supported by Quark.
"""
@@ -36,25 +35,19 @@ class QuarkScheme(ABC):
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
"""
Get minimum device capability.
"""
"""Get minimum device capability."""
raise NotImplementedError
@abstractmethod
def create_weights(self, *args, **kwargs):
"""
Weight creation for the particular scheme. Inputs to this function
"""
"""Weight creation for the particular scheme. Inputs to this function."""
raise NotImplementedError
@abstractmethod
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
):
"""
Run the forward pass for the particular scheme. This is where
"""Run the forward pass for the particular scheme. This is where
scheme-specific dequant/quant steps/kernels should be applied.
Args:
@@ -62,13 +55,13 @@ class QuarkScheme(ABC):
other parameters relevant to the particular scheme.
x: input to the layer
bias: bias parameter
"""
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module):
"""
Called after weight loading is complete for any cleanup that
"""Called after weight loading is complete for any cleanup that
needs to occur.
"""
raise NotImplementedError
@@ -33,8 +33,7 @@ OCP_MX_BLOCK_SIZE = 32
class QuarkW4A8_MXFP4_FP8(QuarkScheme):
"""
- Weights: MXFP4 with E8M0 scales per block of 32
"""- Weights: MXFP4 with E8M0 scales per block of 32
- Activations: FP8 E4M3 (static per-tensor quantization)
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
@@ -31,8 +31,7 @@ def triton_scale_swizzle(
BLOCK_ROWS: tl.constexpr,
BLOCK_COLS: tl.constexpr,
):
"""
Rearranges tensor data from row-major to block-scaled swizzle format.
"""Rearranges tensor data from row-major to block-scaled swizzle format.
Args:
scale_ptr: Pointer to the input scale tensor
@@ -43,6 +42,7 @@ def triton_scale_swizzle(
output_block_stride: Stride between blocks in the output tensor
BLOCK_ROWS: Number of rows in a tile (compile-time constant)
BLOCK_COLS: Number of columns in a tile (compile-time constant)
"""
pid_row = tl.program_id(0)
pid_col = tl.program_id(1)
@@ -86,8 +86,7 @@ def triton_scale_swizzle(
@torch.library.custom_op("vllm::triton_mx_block_rearrange", mutates_args=())
def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
"""
Rearranges an E8M0 tensor scale from row-major format to
"""Rearranges an E8M0 tensor scale from row-major format to
block-scaled swizzle format.
This format is suitable for Tmem as described in NVIDIA documentation:
@@ -98,6 +97,7 @@ def triton_mx_block_rearrange(scale_tensor: torch.Tensor) -> torch.Tensor:
Returns:
Rearranged tensor in block-scaled swizzle format
"""
assert scale_tensor.element_size() == 1, (
"Expected element size to be 1 byte (8 bits)"
@@ -154,8 +154,7 @@ def _triton_mx_block_rearrange_fake(scale_tensor: torch.Tensor) -> torch.Tensor:
def to_blocked(
input_matrix: torch.Tensor, backend: Literal["torch", "triton"] = "triton"
) -> torch.Tensor:
"""
Rearrange a large matrix by breaking it into blocks and applying
"""Rearrange a large matrix by breaking it into blocks and applying
the rearrangement pattern.
See:
@@ -167,6 +166,7 @@ def to_blocked(
Returns:
Rearranged flattened tensor of size (32*cdiv(H,128) * 16*cdiv(W,4))
"""
if backend == "triton":
return triton_mx_block_rearrange(input_matrix).flatten()
@@ -74,8 +74,7 @@ def torchao_version_at_least(torchao_version: str) -> bool:
def should_skip(prefix: str, skip_modules: list[str]) -> bool:
"""
Robust skipping logic:
"""Robust skipping logic:
should_skip("model.model.layers.1.q_proj",
["model.model.layers.1.q_proj"]) # True
should_skip("model.model.layers.10.o_proj", ["o_proj"]) -> True
@@ -163,14 +162,14 @@ class TorchAOConfig(QuantizationConfig):
@staticmethod
def get_config_filenames() -> list[str]:
"""torchao doesn't require additional config files, we use
"""Torchao doesn't require additional config files, we use
`config.json` from huggingface: `model_config.hf_config`
"""
return []
@classmethod
def from_config(cls, config: dict[str, Any]) -> "TorchAOConfig":
"""Create the quant config from an hf model config"""
"""Create the quant config from an hf model config."""
try:
from torchao.core.config import config_from_dict
except ImportError as err:
@@ -287,12 +286,13 @@ class TorchAOConfig(QuantizationConfig):
def torchao_quantize_param_data(
param: torch.Tensor, torchao_config: Any
) -> torch.nn.Parameter:
"""Quantize a Tensor with torchao quantization specified by torchao_config
"""Quantize a Tensor with torchao quantization specified by torchao_config.
Args:
param: weight parameter of the linear module
torchao_config: type of quantization and their arguments we want to
use to quantize the Tensor
"""
from torchao.core.config import AOBaseConfig
from torchao.quantization import quantize_
@@ -322,6 +322,7 @@ class TorchAOLinearMethod(LinearMethodBase):
Args:
quant_config: The torchao quantization config, a string that encodes
the type of quantization and all relevant arguments.
"""
def __init__(self, quant_config: TorchAOConfig):
@@ -45,6 +45,7 @@ def solve_lloyd_max(
Returns:
centroids: Sorted tensor of 2^bits optimal centroids.
boundaries: Sorted tensor of 2^bits - 1 decision boundaries.
"""
n_levels = 2**bits
sigma2 = 1.0 / d
@@ -83,6 +83,7 @@ class TurboQuantConfig:
norm_correction: Re-normalize centroid vectors to unit norm before
inverse rotation during dequant. Fixes quantization-induced norm
distortion, improving PPL by ~0.8% at 4-bit.
"""
head_dim: int = 128
@@ -63,6 +63,7 @@ def is_shared_expert_quant_fse_compatible(
Returns:
A compatibility flag and, when incompatible, the reason.
"""
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
@@ -303,13 +304,11 @@ def is_equal_or_regex_match(
check_contains: bool = False,
use_fnmatch: bool = False,
) -> bool:
"""
Checks whether a value is exactly equal or a regex match for target
"""Checks whether a value is exactly equal or a regex match for target
if target starts with 're:'. If check_contains is set to True,
additionally checks if the target string is contained within the value.
If use_fnmatch is set, supports shell-style patterns in target.
"""
if target.startswith("re:"):
pattern = target[3:]
if re.match(pattern, value):
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path"""
"""Utility helpers for NVFP4 + FlashInfer fused-MoE path."""
from typing import TYPE_CHECKING
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path"""
"""Utility helpers for MxInt4 + FlashInfer fused-MoE path."""
import functools
@@ -37,8 +37,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
gemm2_weights: torch.Tensor,
gemm2_scales: torch.Tensor,
) -> dict[str, torch.Tensor]:
"""
Prepare MxInt4 weights for TRT-LLM kernel.
"""Prepare MxInt4 weights for TRT-LLM kernel.
Input:
gemm1_weights: [num_experts, 2*intermediate_size, hidden_size//8] int32
@@ -51,6 +50,7 @@ def prepare_static_weights_for_trtllm_mxint4_moe(
Returns:
Dict with keys 'gemm1_weights', 'gemm1_scales', 'gemm2_weights',
'gemm2_scales' containing shuffled/packed tensors ready for kernel
"""
from flashinfer import block_scale_interleave
from flashinfer.fused_moe import (
@@ -192,8 +192,7 @@ def flashinfer_trtllm_mxint4_moe(
routing_method_type: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
"""Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
Args:
x: Input hidden states. dtype: bfloat16
@@ -209,11 +208,14 @@ def flashinfer_trtllm_mxint4_moe(
ep_rank: Expert parallelism rank (default: 0)
num_expert_group: Number of expert groups (default: None -> 0)
topk_group: Top-k within groups (default: None -> 0)
routing_replay_out: Optional buffer receiving the routing decisions
so they can be replayed later
e_score_correction_bias: Optional routing bias. dtype: bfloat16
routing_method_type: FlashInfer RoutingMethodType enum value
Returns:
Output tensor from MoE layer. dtype: same as x (bfloat16)
"""
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
@@ -59,7 +59,7 @@ def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
gemm1_weights: torch.Tensor, gemm2_weights: torch.Tensor, is_gated_activation: bool
):
"""Shuffle weights for FI TRT-LLM Format"""
"""Shuffle weights for FI TRT-LLM Format."""
from flashinfer import reorder_rows_for_gated_act_gemm, shuffle_matrix_a
epilogue_tile_m = 128
@@ -210,7 +210,6 @@ def align_fp4_moe_weights_for_fi(
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
gate/up and down projection weights along the intermediate dim.
"""
# Current local intermediate size (per partition) is the K dimension of
# the down projection.
num_experts, hidden_size, intermediate = w2.shape
@@ -316,7 +315,6 @@ def align_moe_weights_for_fi(
not satisfied (e.g. with certain tensor-parallel sizes), we pad the
gate/up and down projection weights along the intermediate dim.
"""
# Current local intermediate size (per partition) is the K dimension of
# the down projection.
num_experts, hidden_size, intermediate = w2.shape
@@ -453,8 +451,7 @@ def prepare_fp8_moe_layer_for_fi(
w2_input_scale: torch.Tensor | None,
is_trtllm: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Convert Fp8 MoE weights to flashinfer kernel format
"""Convert Fp8 MoE weights to flashinfer kernel format.
Note that for trtllm we update the model state dict
with the scale format needed for these kernels.
@@ -462,7 +459,6 @@ def prepare_fp8_moe_layer_for_fi(
Note that for per-tensor, we update the layer's
intermediate size if the weights needed padding.
"""
assert hasattr(layer.moe_config, "is_act_and_mul")
block_quant = (
hasattr(layer, "weight_block_size") and layer.weight_block_size is not None
@@ -69,7 +69,8 @@ def input_to_float8(
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
"""This function quantizes input values to float8 values "
"with tensor-wise quantization."""
"with tensor-wise quantization.
"""
dtype = current_platform.fp8_dtype() if dtype is None else dtype
finfo = torch.finfo(dtype)
min_val, max_val = x.aminmax()
@@ -347,12 +348,10 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
BLOCK_N: tl.constexpr,
):
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
"""
Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
the outputs tensors at the right positions.
"""
pid_m = tl.program_id(0)
pid_n = tl.program_id(1)
N_2 = N // 2
@@ -421,8 +420,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
alpha: float = 1.0,
beta: float = 0.0,
):
"""
Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
"""
GROUP_SIZE = group_size
@@ -564,18 +562,23 @@ def per_token_group_quant_fp8(
"""Function to perform per-token-group quantization on an input tensor `x`.
It converts the tensor values into signed float8 values and returns the
quantized tensor along with the scaling factor used for quantization.
Args:
x: The input tensor with ndim >= 2.
group_size: The group size used for quantization.
eps: The minimum to avoid dividing zero.
dtype: The dtype of output tensor. Note that only `torch.float8_e4m3fn`
is supported for now.
is supported for now.
column_major_scales: Outputs scales in column major.
tma_aligned_scales: Outputs scales in TMA-aligned layout.
out_q: Optional output tensor. If not provided, function will create.
use_ue8m0: If True, round each scale down to a power of two (UE8M0
format). None defers to the platform default.
Returns:
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
scaling factor.
"""
if use_ue8m0 is None:
use_ue8m0 = is_deep_gemm_e8m0_used()
@@ -695,6 +698,7 @@ def per_token_group_quant_fp8_packed_for_deepgemm(
x_s_packed: Int32 tensor with logical shape
[mn, ceil(num_groups_per_row / 4)], laid out with
TMA-aligned stride along the packed-K dimension
"""
if use_ue8m0 is None:
use_ue8m0 = is_deep_gemm_e8m0_used()
@@ -789,7 +793,6 @@ def _w8a8_triton_block_scaled_mm(
product) on input tensors `A` and `B` with block-wise quantization, and
store the result in output tensor `C`.
"""
pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
@@ -842,14 +845,12 @@ def _w8a8_triton_block_scaled_mm(
def get_w8a8_block_fp8_configs(
N: int, K: int, block_n: int, block_k: int
) -> dict[int, Any] | None:
"""
Return optimized configurations for the w8a8 block fp8 kernel.
"""Return optimized configurations for the w8a8 block fp8 kernel.
The return value will be a dictionary that maps an irregular grid of
batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the
kernel on a given batch size bs, the closest batch size in the grid should
be picked and the associated configuration chosen to invoke the kernel.
"""
# First look up if an optimized configuration is available in the configs
# directory
device_name = get_device_name_as_file_name()
@@ -889,18 +890,20 @@ def w8a8_triton_block_scaled_mm(
quantization.
It takes two input tensors `A` and `B` with scales `As` and `Bs`.
The output is returned in the specified `output_dtype`.
Args:
A: The input tensor, e.g., activation.
B: The input tensor, e.g., weight.
As: The per-token-group quantization scale for `A`.
Bs: The per-block quantization scale for `B`.
block_size: The block size for per-block quantization. It should
be 2-dim, e.g., [128, 128].
output_dytpe: The dtype of the returned tensor.
be 2-dim, e.g., [128, 128].
output_dtype: The dtype of the returned tensor.
Returns:
torch.Tensor: The result of matmul.
"""
"""
_on_gfx1250 = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx1250
@@ -1022,6 +1025,7 @@ def requant_weight_ue8m0_inplace(
with shape `(..., M // block_size[0], K // block_size[1])`.
block_size: 2-element iterable `[block_m, block_k]` describing the
block quantisation granularity.
"""
if weight.numel() == 0:
return
@@ -1191,7 +1195,8 @@ def prepare_fp8_moe_layer_for_deepgemm(
def _maybe_pad_fp8_weight(weight: torch.Tensor) -> torch.Tensor:
"""Pad the weight tensor. This is an optimization on ROCm platform, which
can benefit from tensors located far enough from one another in memory"""
can benefit from tensors located far enough from one another in memory
"""
if (
envs.VLLM_ROCM_FP8_PADDING
and current_platform.is_rocm()
@@ -1486,7 +1491,6 @@ def process_fp8_input_tensor_strategy_moe(
enable_eplb: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Process moe input scales for tensor-wise quantization strategy."""
if not all_close_1d(w13_input_scale) or not all_close_1d(w2_input_scale):
logger.info_once(
"Found input_scales that are not equal for "
@@ -88,8 +88,7 @@ if has_humming():
def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape:
"""
Map humming group sizes to QuantKey GroupShape.
"""Map humming group sizes to QuantKey GroupShape.
group_size: elements per group along K (col); 0 means full dimension.
group_size_n: elements per group along N (row); 0 means 1 (per-row).
@@ -298,7 +297,8 @@ def _humming_input_schema_to_quant_key(
schema: "HummingInputSchema",
) -> QuantKey | None:
"""Convert a HummingInputSchema to a QuantKey. Returns None if
the schema represents unquantized (bf16/fp16) inputs."""
the schema represents unquantized (bf16/fp16) inputs.
"""
if schema.a_dtype is None or schema.a_dtype.num_bits >= 16:
return None
@@ -427,7 +427,8 @@ def convert_linear_layer_to_humming_standard(
):
"""Rename/reshape a linear layer's quantized params (the canonical MPLinear
layout: ``weight_packed`` int32 + ``weight_scale``) into the parameter names
and layout humming's weight schema expects (``weight`` / ``weight_scale``)."""
and layout humming's weight schema expects (``weight`` / ``weight_scale``).
"""
for name, checkpoint_name in name_map.items():
tensor = getattr(layer, checkpoint_name)
delattr(layer, checkpoint_name)
@@ -691,11 +692,9 @@ def select_humming_moe_experts(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> type[mk.FusedMoEExperts] | None:
"""
Select the primary Humming MoE Experts class
"""Select the primary Humming MoE Experts class
Note: Shape-specific fallbacks may still occur at runtime.
"""
if not has_humming():
return None
@@ -809,14 +808,14 @@ def _replace_layer_parameters(
tensors: dict[str, torch.Tensor],
preserve_bias: bool = False,
) -> None:
"""
Replace layer parameters for a sublayer with new tensors.
"""Replace layer parameters for a sublayer with new tensors.
Args:
layer: The RoutedExperts layer
sublayer_name: Name of the sublayer (e.g., "w13", "w2")
tensors: Dict of parameter name to tensor
preserve_bias: If True, don't delete bias parameters
"""
# Delete old parameters
for name, _ in list(layer.named_parameters()):
@@ -843,11 +842,11 @@ def _convert_sublayer_to_humming(
num_experts: int,
param_dtype: torch.dtype,
) -> tuple[Any, Any]:
"""
Convert a sublayer's weights from checkpoint format to Humming format.
"""Convert a sublayer's weights from checkpoint format to Humming format.
Returns:
Tuple of (converted_weight_schema, converted_input_schema)
"""
from vllm.utils.humming import HummingWeightSchema
@@ -931,8 +930,7 @@ def _process_single_sublayer(
param_dtype: torch.dtype,
force_weight_schema: Any | None = None,
) -> tuple[Any, Any, "LayerConfig"]:
"""
Process a single sublayer: convert, optionally requant, prepare, and transform.
"""Process a single sublayer: convert, optionally requant, prepare, and transform.
This combines the common logic from convert_to_humming_moe_kernel_format
for processing a single sublayer.
@@ -951,6 +949,7 @@ def _process_single_sublayer(
Returns:
Tuple of the final weight schema, input schema, and Humming layer config.
"""
from vllm.utils.humming import HummingWeightSchema
@@ -1005,8 +1004,7 @@ def convert_to_humming_moe_kernel_format(
input_schema: Any | None = None,
force_weight_schema: Any | None = None,
) -> dict[str, "LayerConfig"]:
"""
Convert MoE weights from checkpoint format to Humming kernel format.
"""Convert MoE weights from checkpoint format to Humming kernel format.
This function processes weights for each sublayer (w13, w2) by:
1. Converting from checkpoint format to humming format if needed
@@ -1032,8 +1030,8 @@ def convert_to_humming_moe_kernel_format(
- Modifies layer parameters in place
- Sets layer.weight_schemas and layer.input_schemas
- Sets layer.humming_configs for quant config construction
"""
"""
# Build schemas from quant_config if not provided
has_bias = layer.moe_config.has_bias
num_experts = layer.moe_config.num_local_experts
@@ -178,6 +178,7 @@ def per_token_group_quant_int8(
Returns:
tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the
scaling factor for quantization.
"""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` cannot be divisible by `group_size`"
@@ -17,8 +17,7 @@ def query_machete_supported_quant_types(zero_points: bool) -> list[ScalarType]:
def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
"""
Queries the supported group sizes for Machete based on the activation type.
"""Queries the supported group sizes for Machete based on the activation type.
Args:
act_type: The activation data type (torch.float16, torch.bfloat16).
@@ -27,6 +26,7 @@ def query_machete_supported_group_sizes(act_type: torch.dtype) -> list[int]:
A list of supported group sizes. The group size must
be divisible by `TileShapeK = 128 * 8 // num_bits(act_type)`.
-1 indicates per-channel quantization.
"""
if act_type in [torch.float16, torch.bfloat16]:
return [-1, 64, 128]
@@ -245,7 +245,8 @@ def marlin_padded_nk(size_n: int, size_k: int, group_size: int = -1) -> tuple[in
def marlin_repacked_nk(qweight: torch.Tensor, num_bits: int) -> tuple[int, int]:
"""Recover the (size_n, size_k) a Marlin weight was repacked with
(including any tile padding) from its packed shape."""
(including any tile padding) from its packed shape.
"""
pack_factor = 32 // num_bits
size_k = qweight.size(0) * GPTQ_MARLIN_TILE
size_n = qweight.size(1) * pack_factor // GPTQ_MARLIN_TILE
@@ -256,7 +257,8 @@ def marlin_pad_qweight(
qweight: torch.Tensor, size_n: int, size_k: int, padded_n: int, padded_k: int
) -> torch.Tensor:
"""Zero-pad a GPTQ-layout packed weight (size_k / pack, size_n) for
gptq_marlin_repack."""
gptq_marlin_repack.
"""
if (padded_n, padded_k) == (size_n, size_k):
return qweight
pack_factor = size_k // qweight.size(0)
@@ -274,7 +276,8 @@ def marlin_pad_scales(
group_size: int,
) -> torch.Tensor:
"""Zero-pad weight scales (num_groups, size_n); call before
marlin_permute_scales and pass the padded extents to it."""
marlin_permute_scales and pass the padded extents to it.
"""
if (padded_n, padded_k) == (size_n, size_k):
return scales
pad_rows = padded_k // group_size - scales.size(0) if group_size > 0 else 0
@@ -397,8 +400,7 @@ def check_moe_marlin_supports_layer(
def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tensor):
"""
Given Marlin packed weight matrices w1_packed, and w2_packed,
"""Given Marlin packed weight matrices w1_packed, and w2_packed,
return the MoE intermediate size N
"""
marlin_tile_size = 16
@@ -41,8 +41,8 @@ def _nvfp4_compute_scale_factor(
) -> float:
"""Compute the power-of-2 scale_factor needed so that all non-zero
values in marlin_scales * 2^7 are >= 2 after rescaling.
Returns a Python float (power of 2, >= 1.0)."""
Returns a Python float (power of 2, >= 1.0).
"""
# Since half has a smaller dynamic range compared to bfloat16,
# no rescaling is applied here if active dtype is half.
if a_dtype is not None and a_dtype == torch.half:
@@ -76,9 +76,12 @@ def nvfp4_marlin_process_scales(
given value is used directly. The caller is responsible for
dividing ``global_scale`` by the returned ``scale_factor`` to
preserve numerical correctness.
a_dtype: dtype of the activations the scales will be used with. None
keeps the dtype of ``marlin_scales``.
Returns:
A tuple of (processed_scales, scale_factor).
"""
if not (marlin_scales >= 0).all():
logger.warning_once(
@@ -373,7 +376,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w13(x: torch.Tensor) -> torch.Tensor:
"""Zero-pad each gate/up shard of a (E, num_shards * N, cols)
tensor to padded_N rows."""
tensor to padded_N rows.
"""
if padded_N == N:
return x
x = x.view(E, num_shards, N, x.size(-1))
@@ -382,7 +386,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
def pad_w2(x: torch.Tensor, packing: int) -> torch.Tensor:
"""Zero-pad the packed N (last) dim of a (E, K, N / packing)
tensor."""
tensor.
"""
if padded_N == N:
return x
return torch.nn.functional.pad(x, (0, (padded_N - N) // packing))
@@ -217,7 +217,8 @@ def prepare_fp8_layer_for_marlin(
def _moe_pad_shard_rows(x: torch.Tensor, n: int, padded_n: int) -> torch.Tensor:
"""Zero-pad each gate/up shard of a ``(E, 2 * n, ...)`` tensor to padded_n
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing."""
rows. FP8 zero decodes to 0.0, so the padded rows contribute nothing.
"""
if padded_n == n:
return x
e = x.size(0)
@@ -241,14 +242,12 @@ def prepare_fp8_moe_layer_for_marlin(
w13_weight_scale: torch.Tensor,
w2_weight_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Shuffle weights and scales into marlin format.
"""Shuffle weights and scales into marlin format.
Note that this function has the side effect of adding a `workspace`
attribute to the layer. This `workspace` does not need to be
registered as a Parameter as it is not used during weight reloading.
"""
logger.warning_once(
"Your GPU does not have native support for FP8 computation but "
"FP8 quantization is being used. Weight-only FP8 compression will "
@@ -372,9 +371,7 @@ def prepare_fp8_moe_layer_for_marlin(
def pack_fp8_to_int32(
fp8_tensor: torch.Tensor, size_k_first: bool = True
) -> torch.Tensor:
"""
Repack FP8 weights to gptq format (packed int32 elements)
"""
"""Repack FP8 weights to gptq format (packed int32 elements)."""
assert fp8_tensor.dtype == torch.float8_e4m3fn
assert fp8_tensor.ndim == 2
@@ -524,6 +521,7 @@ def prepare_mxfp8_moe_layer_for_marlin(
Returns:
(w13, w2, w13_scale, w2_scale) in Marlin format.
"""
group_size = 32
e = w13.shape[0]
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility functions used for tests and benchmarks"""
"""Utility functions used for tests and benchmarks."""
import numpy as np
import torch
@@ -28,7 +28,7 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
"""weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel"""
"""Weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel."""
assert has_triton_kernels()
import triton_kernels.matmul_ogs_details.opt_flags as opt_flags
from triton_kernels.numerics import InFlexData
@@ -374,8 +374,7 @@ def downcast_to_mxfp(
BLOCK_OUT_DIM: int = 128,
BLOCK_QUANT_DIM: int = 32,
):
"""
Convert the src weights to MXFP4. The src weight is quantized along the
"""Convert the src weights to MXFP4. The src weight is quantized along the
axis dimension into packed e2m1 values (torch.uint8, two values per
byte), so the size of that dimension in the output is half of the
logical (unpacked) size.
@@ -657,8 +657,7 @@ def ref_nvfp4_quant(x, global_scale, block_size):
def ref_nvfp4_quant_dequant(
x: torch.Tensor, global_scale: torch.Tensor, block_size: int
) -> torch.Tensor:
"""
NVFP4 quantize-dequantize operation.
"""NVFP4 quantize-dequantize operation.
`global_scale` is expected to have a single element.
"""
@@ -11,18 +11,15 @@ from vllm.utils.math_utils import round_up
def swizzle_blockscale(scale: torch.Tensor) -> torch.Tensor:
"""
Pad and block-interleave the FP4 block-scales so that they match the data
"""Pad and block-interleave the FP4 block-scales so that they match the data
layout expected by the CUTLASS / FlashInfer kernels.
Parameters
----------
scale: torch.Tensor
Args:
scale: The FP4 block-scale tensor to swizzle.
Returns:
The swizzled tensor with the same logical shape as `scale`.
Returns
-------
torch.Tensor
The swizzled tensor with the same logical shape as *scale*.
"""
assert scale.dtype == torch.float8_e4m3fn, (
"swizzle_blockscale expects the input tensor to be in "
@@ -65,8 +62,7 @@ def pad_nvfp4_weight_for_cutlass(
weight: torch.Tensor,
alignment: int = 32,
) -> tuple[torch.Tensor, int]:
"""
Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
"""Pad packed NVFP4 weights so that both N (rows) and K (columns) satisfy
the alignment constraints required by CUTLASS / FlashInfer FP4 kernels.
CUTLASS FP4 kernel requires both K and N matrix dimensions to be divisible
@@ -99,8 +95,7 @@ def pad_nvfp4_weight_for_cutlass(
def nvfp4_weight_padding_bytes(layer: torch.nn.Module) -> int:
"""
Compute K-dim padding (packed bytes) of an NVFP4 weight,
"""Compute K-dim padding (packed bytes) of an NVFP4 weight,
Aligned with `pad_nvfp4_weight_for_cutlass` above.
"""
return layer.weight.shape[1] - layer.input_size_per_partition // 2
@@ -110,8 +105,7 @@ def pad_nvfp4_activation_for_cutlass(
x_fp4: torch.Tensor,
weights_padding_bytes: int,
) -> torch.Tensor:
"""
Pad packed FP4 activations to match the K-dimension padding applied to weights.
"""Pad packed FP4 activations to match the K-dimension padding applied to weights.
The padding is in bytes (tensor dimension), not FP4 elements.
"""
if weights_padding_bytes > 0:
@@ -123,9 +117,7 @@ def slice_nvfp4_output(
out: torch.Tensor,
output_size: int,
) -> torch.Tensor:
"""
Slice the output tensor to remove padding in N dimension if weight was padded.
"""
"""Slice the output tensor to remove padding in N dimension if weight was padded."""
if out.shape[-1] != output_size:
return out[..., :output_size].contiguous()
return out
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file is used for /tests and /benchmarks"""
"""This file is used for /tests and /benchmarks."""
from collections.abc import Callable, Mapping
from dataclasses import dataclass
@@ -109,8 +109,7 @@ class _GroupShape(NamedTuple):
class GroupShape(_GroupShape):
"""
This class describes the quantization group shape.
"""This class describes the quantization group shape.
It includes static members for common shapes (per-tensor, per-token).
"""
@@ -139,8 +138,7 @@ GroupShape.PER_CHANNEL = GroupShape(-1, 1)
@dataclass(frozen=True)
class ScaleDesc:
"""
Class for describing a single quantization scaling factor.
"""Class for describing a single quantization scaling factor.
dtype: data type of the scale
static: static scale if True, dynamic if False
group_shape: group shape of the scale
@@ -166,8 +164,7 @@ class ScaleDesc:
@dataclass(frozen=True)
class QuantKey:
"""
Class for identifying the type of quantization.
"""Class for identifying the type of quantization.
dtype: quantized data type
scale: scale descriptor
scale2: second-level scale descriptor
@@ -378,8 +375,7 @@ def prep_scale_for_group_broadcast(
x: torch.Tensor,
group_shape: GroupShape | None,
) -> torch.Tensor:
"""
Prepare the input quantization scale for group broadcasting.
"""Prepare the input quantization scale for group broadcasting.
Args:
scale: The scale tensor (scalar or 1D).
@@ -388,6 +384,7 @@ def prep_scale_for_group_broadcast(
Returns:
scale reshaped for correct broadcasting.
"""
if scale.numel() == 1:
# For per-tensor quant, keep the scale as a scalar (not reshaped to (1, 1)).
@@ -432,13 +429,13 @@ def scaled_quantize(
quant_dtype: torch.dtype,
compute_dtype: torch.dtype | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Args:
x: Input tensor to quantize
group_shape: Shape of quantization groups
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
compute_dtype: Optional dtype for intermediate computations.
If None, uses input dtype. Use torch.float32 for higher precision.
"""Args:
x: Input tensor to quantize
group_shape: Shape of quantization groups
quant_dtype: Target quantized dtype (e.g., torch.float8_e4m3fn)
compute_dtype: Optional dtype for intermediate computations.
If None, uses input dtype. Use torch.float32 for higher precision.
"""
group_shape = _normalize_quant_group_shape(x, group_shape)
assert quant_dtype.is_floating_point, (
@@ -506,7 +503,7 @@ def get_attribute_fallback(obj, attributes: list[str]):
def get_and_maybe_dequant_weights(
layer: "LinearBase", out_dtype: torch.dtype = torch.float32
):
"""Return layer's unquantized weights in [out, in] layout"""
"""Return layer's unquantized weights in [out, in] layout."""
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
@@ -924,8 +921,7 @@ def awq_pack(
def convert_bf16_scales_to_fp8(
quant_fp8: Callable, scales: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
"""Convert a BF16 scale tensor into the pair of (fp8_scales, channel_scales)
expected by W4A8 GEMM kernels.
"""
assert scales.is_contiguous(), (
@@ -949,9 +945,7 @@ def convert_bf16_scales_to_fp8(
def convert_packed_uint4b8_to_signed_int4_inplace(t: torch.Tensor) -> torch.Tensor:
"""
Convert int4b8 (packed to int32) to signed int4
"""
"""Convert int4b8 (packed to int32) to signed int4."""
assert t.is_cuda, "tensor must be on gpu"
assert t.dtype == torch.int32, f"expected int32 packed weights but got {t.dtype}"