Speed up docs build (#44635)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-05 14:51:44 +00:00
committed by GitHub
parent c66b19800b
commit a80af24356
32 changed files with 234 additions and 159 deletions
@@ -599,12 +599,14 @@ class FusedMoE(PluggableLayer):
):
"""
Load grouped weight scales for group quantization or model weights
:param shard_dim: dimension to shard
:param expert_data: parameter for a particular expert
:param shard_id: either w1, w2, or w3
:param loaded_weight: checkpoint weight to load into the param
:param tp_rank: tensor parallel rank
:param load_full_w2: whether or not the w2 loaded should be sharded.
Args:
shard_dim: dimension to shard
expert_data: parameter for a particular expert
shard_id: either w1, w2, or w3
loaded_weight: checkpoint weight to load into the param
tp_rank: tensor parallel rank
load_full_w2: whether or not the w2 loaded should be sharded.
"""
if shard_id == "w2":
# In the case where we have actorder/g_idx, we do not partition the
@@ -178,8 +178,9 @@ class QuantizationConfig(ABC):
Interface for models to update module names referenced in
quantization configs in order to reflect the vllm model structure
:param hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
Args:
hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
"""
# TODO (@kylesayrs): add implementations for all subclasses
pass
@@ -267,8 +267,11 @@ class CompressedTensorsConfig(QuantizationConfig):
cls, config: dict[str, Any]
) -> tuple[dict[str, SparsityCompressionConfig], list[str]]:
"""
:param config: The `quantization_config` dictionary from config.json
:return: A tuple with two elements
Args:
config: The `quantization_config` dictionary from config.json
Returns:
A tuple with two elements
1. A dictionary mapping target layer names to their corresponding
sparsity_config
2. A list of layer names to ignore for sparsity
@@ -296,8 +299,11 @@ class CompressedTensorsConfig(QuantizationConfig):
cls, config: dict[str, Any]
) -> QUANTIZATION_SCHEME_MAP_TYPE:
"""
:param config: The `quantization_config` dictionary from config.json
:return: A dictionary mapping target layer names to their corresponding
Args:
config: The `quantization_config` dictionary from config.json
Returns:
A dictionary mapping target layer names to their corresponding
quantization_args for weights and input activations
"""
target_scheme_map: dict[str, Any] = dict()
@@ -967,7 +973,9 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
"""
Validator for the kv cache scheme. Useful for controlling the
kv cache quantization schemes, that are being supported in vLLM
:param kv_cache_scheme: the compressed-tensors kv cache scheme
Args:
kv_cache_scheme: the compressed-tensors kv cache scheme
"""
if kv_cache_scheme is None:
return
@@ -38,11 +38,11 @@ class CompressedTensorsScheme(ABC):
Run the forward pass for the particular scheme. This is where
scheme-specific dequant/quant steps/kernels should be applied.
:param layer: torch.nn.Module with the registered weights and
other parameters relevant to the particular scheme.
:param x: input to the layer
:param bias: bias parameter
Args:
layer: torch.nn.Module with the registered weights and
other parameters relevant to the particular scheme.
x: input to the layer
bias: bias parameter
"""
raise NotImplementedError()
@@ -133,12 +133,11 @@ def find_matched_target(
*All* component module names must match in order for a match to be
successful. A successful match returns the first component target
:param layer_name: layer name
:param module: torch.nn.Module
:param targets: list of targets to match the layer against
:param fused_mapping: map from fused layer names to its components
:param fused_strategy: either "all" or "any". If using "all", fused
layers match if "all" of its components match
Args:
layer_name: layer name
module: torch.nn.Module
targets: list of targets to match the layer against
fused_mapping: map from fused layer names to its components
"""
if layer_name is None:
@@ -161,9 +160,10 @@ def _find_first_match(
exactly or as a regex after 're:'. If check_contains is set to True,
additionally checks if the target string is contained within the value.
:param value: string to compare the list of targets against
:param targets: list of targets to match the layer against
:param check_contains: whether or not to do a substring match
Args:
value: string to compare the list of targets against
targets: list of targets to match the layer against
check_contains: whether or not to do a substring match
"""
for target in targets:
@@ -205,9 +205,10 @@ def _match_fused_layer(
Implements an "all" matching strategy where a fused layer matches iff
"all" of its components match
:param layer_name: layer name
:param target_layers: list of targets to match the layer against
:param fused_mapping: map from fused layer names to its components
Args:
layer_name: layer name
target_layers: list of targets to match the layer against
fused_mapping: map from fused layer names to its components
Examples:
layer_name = "model.layers.0.self_attn.qkv_proj"
@@ -114,8 +114,9 @@ class GGUFConfig(QuantizationConfig):
Interface for models to update module names referenced in
quantization configs in order to reflect the vllm model structure
:param hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
Args:
hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
"""
if self.unquantized_modules is not None:
self.unquantized_modules = hf_to_vllm_mapper.apply_list(
@@ -46,16 +46,17 @@ class QuantFP8(CustomOp):
compile_native: bool = True,
):
"""
:param static: static or dynamic quantization
:param group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
PER_CHANNEL, or arbitrary block size)
:param num_token_padding: Pad the token dimension of output to this
size
:param tma_aligned_scales: For group quantization, output scales in
TMA-aligned layout
:param column_major_scales: For group quantization, output scales in
column major format
:param compile_native: Manually compile forward_native if compile mode > None
Args:
static: static or dynamic quantization
group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
PER_CHANNEL, or arbitrary block size)
num_token_padding: Pad the token dimension of output to this
size
tma_aligned_scales: For group quantization, output scales in
TMA-aligned layout
column_major_scales: For group quantization, output scales in
column major format
compile_native: Manually compile forward_native if compile mode > None
"""
super().__init__(compile_native=compile_native)
self.static = static
@@ -47,7 +47,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
- quantize k/v_cache entries before saving them to the cache
- dequantize k/v_cache entries before fetching them from the cache
:param quant_config: the appropriate QuantizationConfig
Args:
quant_config: the appropriate QuantizationConfig
"""
def __init__(self, quant_config: QuantizationConfig):
@@ -87,8 +87,9 @@ class QuarkConfig(QuantizationConfig):
Interface for models to update module names referenced in
quantization configs in order to reflect the vllm model structure
:param hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
Args:
hf_to_vllm_mapper: maps from hf model structure (the assumed
structure of the qconfig) to vllm model structure
"""
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
@@ -724,7 +725,9 @@ class QuarkKVCacheMethod(BaseKVCacheMethod):
"""
Validator for the kv cache configuration. Useful for controlling the
kv cache quantization schemes, that are being supported in vLLM
:param kv_cache_config: the quark kv cache scheme
Args:
kv_cache_config: the quark kv cache scheme
"""
if kv_cache_config is None:
return
@@ -38,11 +38,11 @@ class QuarkScheme(ABC):
Run the forward pass for the particular scheme. This is where
scheme-specific dequant/quant steps/kernels should be applied.
:param layer: torch.nn.Module with the registered weights and
other parameters relevant to the particular scheme.
:param x: input to the layer
:param bias: bias parameter
Args:
layer: torch.nn.Module with the registered weights and
other parameters relevant to the particular scheme.
x: input to the layer
bias: bias parameter
"""
raise NotImplementedError
@@ -123,7 +123,8 @@ def initialize_online_processing(layer: torch.nn.Module):
Called by either `initialize_layerwise_reload` or an online quantization scheme,
prevents double wrapping in the case of online quantization + reloading
:param layer: layer whose parameter weight loaders will be wrapped
Args:
layer: layer whose parameter weight loaders will be wrapped
"""
info = get_layerwise_info(layer)
@@ -222,8 +223,9 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
This function should be applied after `initialize_layerwise_reload` is applied
unwrap the layerwise weight loaders.
:param model: model to finalize processing for
:param model_config: config needed for applying processing to attention layers
Args:
model: model to finalize processing for
model_config: config needed for applying processing to attention layers
"""
if hasattr(model, "_original_do_torchao_reload"):
model._do_torchao_reload = model._original_do_torchao_reload
@@ -175,9 +175,12 @@ def get_numel_loaded(
"""
Determine how many elements would be loaded by a weight loader call.
:param weight loader: used to load weights
:param args: bound arguments to weight loader
:return: number of elements loaded by the weight loader, the return value of the
Args:
weight_loader: used to load weights
args: bound arguments to weight loader
Returns:
number of elements loaded by the weight loader, the return value of the
weight loader
"""
with CopyCounter() as counter:
@@ -20,9 +20,12 @@ def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.T
tensors will reference layers, and the WeakKeyDictionary will never evict entries,
even when the model is deleted.
:param tensor: tensor to be sanitized
:param layer: layer whose references should be removed
:return: sanitized tensor
Args:
tensor: tensor to be sanitized
layer: layer whose references should be removed
Returns:
sanitized tensor
"""
for key, value in tensor.__dict__.items():
if isinstance(value, MethodType) and value.__self__ is layer:
@@ -38,10 +41,12 @@ def restore_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Te
Used by `restore_layer_on_meta` to add back layer references, allowing for proper
weight loading.
:param tensor: tensor to be sanitized
:param layer: layer whose references should be removed
:return: sanitized tensor
Args:
tensor: tensor to be sanitized
layer: layer whose references should be removed
Returns:
sanitized tensor
"""
for key, value in tensor.__dict__.items():
if isinstance(value, MethodType) and value.__self__ is layer_ref_sentinel:
@@ -49,8 +49,11 @@ def has_device_tensors(bound_args: BoundArguments) -> bool:
"""
Return True if the loaded weights exist on an accelerator device
:param bound_args: args to load weights
:return: True if weights are on accelerator device
Args:
bound_args: args to load weights
Returns:
True if weights are on accelerator device
"""
return any(
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
@@ -62,8 +65,11 @@ def get_info_size(info: LayerReloadingInfo) -> int:
"""
Calculate the number of bytes used by loaded weights for a given layer
:param info: layerwise info to get size of
:return: number of bytes used by loaded weights
Args:
info: layerwise info to get size of
Returns:
number of bytes used by loaded weights
"""
return sum(
value.nbytes
+2 -1
View File
@@ -277,7 +277,8 @@ class OlmoModel(nn.Module):
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
"""
:param input_ids: A tensor of shape `(batch_size, seq_len)`.
Args:
input_ids: A tensor of shape `(batch_size, seq_len)`.
"""
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
+2 -1
View File
@@ -314,7 +314,8 @@ class Olmo2Model(nn.Module):
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
"""
:param input_ids: A tensor of shape `(batch_size, seq_len)`.
Args:
input_ids: A tensor of shape `(batch_size, seq_len)`.
"""
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
+10 -9
View File
@@ -3,6 +3,7 @@
from collections.abc import Callable, Hashable
from fractions import Fraction
from typing import Any
from weakref import WeakValueDictionary
import torch
@@ -42,10 +43,9 @@ class BasevLLMParameter(Parameter):
"""
Initialize the BasevLLMParameter
:param data: torch tensor with the parameter data
:param weight_loader: weight loader callable
:returns: a torch.nn.parameter
Args:
data: torch tensor with the parameter data
weight_loader: weight loader callable
"""
# During weight loading, we often do something like:
@@ -445,15 +445,16 @@ class SharedWeightParameter(BasevLLMParameter):
"currently support tensor parallelism"
)
def add_partition(self, index: int, data_key: Hashable, *args, **kwargs):
def add_partition(self, index: int, data_key: Hashable, *args: Any, **kwargs: Any):
"""
Add a partition to the weight parameter. Partitions whose `data_key`
is the same will share tensor data
:param index: index of partition to add
:param data_key: hashable key used to key shared tensors
:param *args: arguments for `torch.empty`
:param **kwargs: keyword arguments for `torch.empty`
Args:
index: index of partition to add
data_key: hashable key used to key shared tensors
*args: arguments for `torch.empty`
**kwargs: keyword arguments for `torch.empty`
"""
# load (shared) tensor using `data_key`
if data_key not in self.tensors_registry: