Speed up docs build (#44635)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -599,12 +599,14 @@ class FusedMoE(PluggableLayer):
|
||||
):
|
||||
"""
|
||||
Load grouped weight scales for group quantization or model weights
|
||||
:param shard_dim: dimension to shard
|
||||
:param expert_data: parameter for a particular expert
|
||||
:param shard_id: either w1, w2, or w3
|
||||
:param loaded_weight: checkpoint weight to load into the param
|
||||
:param tp_rank: tensor parallel rank
|
||||
:param load_full_w2: whether or not the w2 loaded should be sharded.
|
||||
|
||||
Args:
|
||||
shard_dim: dimension to shard
|
||||
expert_data: parameter for a particular expert
|
||||
shard_id: either w1, w2, or w3
|
||||
loaded_weight: checkpoint weight to load into the param
|
||||
tp_rank: tensor parallel rank
|
||||
load_full_w2: whether or not the w2 loaded should be sharded.
|
||||
"""
|
||||
if shard_id == "w2":
|
||||
# In the case where we have actorder/g_idx, we do not partition the
|
||||
|
||||
@@ -178,8 +178,9 @@ class QuantizationConfig(ABC):
|
||||
Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the vllm model structure
|
||||
|
||||
:param hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
Args:
|
||||
hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
"""
|
||||
# TODO (@kylesayrs): add implementations for all subclasses
|
||||
pass
|
||||
|
||||
@@ -267,8 +267,11 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
cls, config: dict[str, Any]
|
||||
) -> tuple[dict[str, SparsityCompressionConfig], list[str]]:
|
||||
"""
|
||||
:param config: The `quantization_config` dictionary from config.json
|
||||
:return: A tuple with two elements
|
||||
Args:
|
||||
config: The `quantization_config` dictionary from config.json
|
||||
|
||||
Returns:
|
||||
A tuple with two elements
|
||||
1. A dictionary mapping target layer names to their corresponding
|
||||
sparsity_config
|
||||
2. A list of layer names to ignore for sparsity
|
||||
@@ -296,8 +299,11 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
cls, config: dict[str, Any]
|
||||
) -> QUANTIZATION_SCHEME_MAP_TYPE:
|
||||
"""
|
||||
:param config: The `quantization_config` dictionary from config.json
|
||||
:return: A dictionary mapping target layer names to their corresponding
|
||||
Args:
|
||||
config: The `quantization_config` dictionary from config.json
|
||||
|
||||
Returns:
|
||||
A dictionary mapping target layer names to their corresponding
|
||||
quantization_args for weights and input activations
|
||||
"""
|
||||
target_scheme_map: dict[str, Any] = dict()
|
||||
@@ -967,7 +973,9 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Validator for the kv cache scheme. Useful for controlling the
|
||||
kv cache quantization schemes, that are being supported in vLLM
|
||||
:param kv_cache_scheme: the compressed-tensors kv cache scheme
|
||||
|
||||
Args:
|
||||
kv_cache_scheme: the compressed-tensors kv cache scheme
|
||||
"""
|
||||
if kv_cache_scheme is None:
|
||||
return
|
||||
|
||||
+5
-5
@@ -38,11 +38,11 @@ class CompressedTensorsScheme(ABC):
|
||||
Run the forward pass for the particular scheme. This is where
|
||||
scheme-specific dequant/quant steps/kernels should be applied.
|
||||
|
||||
:param layer: torch.nn.Module with the registered weights and
|
||||
other parameters relevant to the particular scheme.
|
||||
:param x: input to the layer
|
||||
:param bias: bias parameter
|
||||
|
||||
Args:
|
||||
layer: torch.nn.Module with the registered weights and
|
||||
other parameters relevant to the particular scheme.
|
||||
x: input to the layer
|
||||
bias: bias parameter
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@@ -133,12 +133,11 @@ def find_matched_target(
|
||||
*All* component module names must match in order for a match to be
|
||||
successful. A successful match returns the first component target
|
||||
|
||||
:param layer_name: layer name
|
||||
:param module: torch.nn.Module
|
||||
:param targets: list of targets to match the layer against
|
||||
:param fused_mapping: map from fused layer names to its components
|
||||
:param fused_strategy: either "all" or "any". If using "all", fused
|
||||
layers match if "all" of its components match
|
||||
Args:
|
||||
layer_name: layer name
|
||||
module: torch.nn.Module
|
||||
targets: list of targets to match the layer against
|
||||
fused_mapping: map from fused layer names to its components
|
||||
"""
|
||||
|
||||
if layer_name is None:
|
||||
@@ -161,9 +160,10 @@ def _find_first_match(
|
||||
exactly or as a regex after 're:'. If check_contains is set to True,
|
||||
additionally checks if the target string is contained within the value.
|
||||
|
||||
:param value: string to compare the list of targets against
|
||||
:param targets: list of targets to match the layer against
|
||||
:param check_contains: whether or not to do a substring match
|
||||
Args:
|
||||
value: string to compare the list of targets against
|
||||
targets: list of targets to match the layer against
|
||||
check_contains: whether or not to do a substring match
|
||||
"""
|
||||
|
||||
for target in targets:
|
||||
@@ -205,9 +205,10 @@ def _match_fused_layer(
|
||||
Implements an "all" matching strategy where a fused layer matches iff
|
||||
"all" of its components match
|
||||
|
||||
:param layer_name: layer name
|
||||
:param target_layers: list of targets to match the layer against
|
||||
:param fused_mapping: map from fused layer names to its components
|
||||
Args:
|
||||
layer_name: layer name
|
||||
target_layers: list of targets to match the layer against
|
||||
fused_mapping: map from fused layer names to its components
|
||||
|
||||
Examples:
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
|
||||
@@ -114,8 +114,9 @@ class GGUFConfig(QuantizationConfig):
|
||||
Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the vllm model structure
|
||||
|
||||
:param hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
Args:
|
||||
hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
"""
|
||||
if self.unquantized_modules is not None:
|
||||
self.unquantized_modules = hf_to_vllm_mapper.apply_list(
|
||||
|
||||
@@ -46,16 +46,17 @@ class QuantFP8(CustomOp):
|
||||
compile_native: bool = True,
|
||||
):
|
||||
"""
|
||||
:param static: static or dynamic quantization
|
||||
:param group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
|
||||
PER_CHANNEL, or arbitrary block size)
|
||||
:param num_token_padding: Pad the token dimension of output to this
|
||||
size
|
||||
:param tma_aligned_scales: For group quantization, output scales in
|
||||
TMA-aligned layout
|
||||
:param column_major_scales: For group quantization, output scales in
|
||||
column major format
|
||||
:param compile_native: Manually compile forward_native if compile mode > None
|
||||
Args:
|
||||
static: static or dynamic quantization
|
||||
group_shape: quantization group shape (PER_TOKEN, PER_TENSOR,
|
||||
PER_CHANNEL, or arbitrary block size)
|
||||
num_token_padding: Pad the token dimension of output to this
|
||||
size
|
||||
tma_aligned_scales: For group quantization, output scales in
|
||||
TMA-aligned layout
|
||||
column_major_scales: For group quantization, output scales in
|
||||
column major format
|
||||
compile_native: Manually compile forward_native if compile mode > None
|
||||
"""
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.static = static
|
||||
|
||||
@@ -47,7 +47,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
- quantize k/v_cache entries before saving them to the cache
|
||||
- dequantize k/v_cache entries before fetching them from the cache
|
||||
|
||||
:param quant_config: the appropriate QuantizationConfig
|
||||
Args:
|
||||
quant_config: the appropriate QuantizationConfig
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: QuantizationConfig):
|
||||
|
||||
@@ -87,8 +87,9 @@ class QuarkConfig(QuantizationConfig):
|
||||
Interface for models to update module names referenced in
|
||||
quantization configs in order to reflect the vllm model structure
|
||||
|
||||
:param hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
Args:
|
||||
hf_to_vllm_mapper: maps from hf model structure (the assumed
|
||||
structure of the qconfig) to vllm model structure
|
||||
"""
|
||||
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
|
||||
|
||||
@@ -724,7 +725,9 @@ class QuarkKVCacheMethod(BaseKVCacheMethod):
|
||||
"""
|
||||
Validator for the kv cache configuration. Useful for controlling the
|
||||
kv cache quantization schemes, that are being supported in vLLM
|
||||
:param kv_cache_config: the quark kv cache scheme
|
||||
|
||||
Args:
|
||||
kv_cache_config: the quark kv cache scheme
|
||||
"""
|
||||
if kv_cache_config is None:
|
||||
return
|
||||
|
||||
@@ -38,11 +38,11 @@ class QuarkScheme(ABC):
|
||||
Run the forward pass for the particular scheme. This is where
|
||||
scheme-specific dequant/quant steps/kernels should be applied.
|
||||
|
||||
:param layer: torch.nn.Module with the registered weights and
|
||||
other parameters relevant to the particular scheme.
|
||||
:param x: input to the layer
|
||||
:param bias: bias parameter
|
||||
|
||||
Args:
|
||||
layer: torch.nn.Module with the registered weights and
|
||||
other parameters relevant to the particular scheme.
|
||||
x: input to the layer
|
||||
bias: bias parameter
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -123,7 +123,8 @@ def initialize_online_processing(layer: torch.nn.Module):
|
||||
Called by either `initialize_layerwise_reload` or an online quantization scheme,
|
||||
prevents double wrapping in the case of online quantization + reloading
|
||||
|
||||
:param layer: layer whose parameter weight loaders will be wrapped
|
||||
Args:
|
||||
layer: layer whose parameter weight loaders will be wrapped
|
||||
"""
|
||||
info = get_layerwise_info(layer)
|
||||
|
||||
@@ -222,8 +223,9 @@ def finalize_layerwise_processing(model: torch.nn.Module, model_config: ModelCon
|
||||
This function should be applied after `initialize_layerwise_reload` is applied
|
||||
unwrap the layerwise weight loaders.
|
||||
|
||||
:param model: model to finalize processing for
|
||||
:param model_config: config needed for applying processing to attention layers
|
||||
Args:
|
||||
model: model to finalize processing for
|
||||
model_config: config needed for applying processing to attention layers
|
||||
"""
|
||||
if hasattr(model, "_original_do_torchao_reload"):
|
||||
model._do_torchao_reload = model._original_do_torchao_reload
|
||||
|
||||
@@ -175,9 +175,12 @@ def get_numel_loaded(
|
||||
"""
|
||||
Determine how many elements would be loaded by a weight loader call.
|
||||
|
||||
:param weight loader: used to load weights
|
||||
:param args: bound arguments to weight loader
|
||||
:return: number of elements loaded by the weight loader, the return value of the
|
||||
Args:
|
||||
weight_loader: used to load weights
|
||||
args: bound arguments to weight loader
|
||||
|
||||
Returns:
|
||||
number of elements loaded by the weight loader, the return value of the
|
||||
weight loader
|
||||
"""
|
||||
with CopyCounter() as counter:
|
||||
|
||||
@@ -20,9 +20,12 @@ def sanitize_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.T
|
||||
tensors will reference layers, and the WeakKeyDictionary will never evict entries,
|
||||
even when the model is deleted.
|
||||
|
||||
:param tensor: tensor to be sanitized
|
||||
:param layer: layer whose references should be removed
|
||||
:return: sanitized tensor
|
||||
Args:
|
||||
tensor: tensor to be sanitized
|
||||
layer: layer whose references should be removed
|
||||
|
||||
Returns:
|
||||
sanitized tensor
|
||||
"""
|
||||
for key, value in tensor.__dict__.items():
|
||||
if isinstance(value, MethodType) and value.__self__ is layer:
|
||||
@@ -38,10 +41,12 @@ def restore_layer_refs(tensor: torch.Tensor, layer: torch.nn.Module) -> torch.Te
|
||||
Used by `restore_layer_on_meta` to add back layer references, allowing for proper
|
||||
weight loading.
|
||||
|
||||
:param tensor: tensor to be sanitized
|
||||
:param layer: layer whose references should be removed
|
||||
:return: sanitized tensor
|
||||
Args:
|
||||
tensor: tensor to be sanitized
|
||||
layer: layer whose references should be removed
|
||||
|
||||
Returns:
|
||||
sanitized tensor
|
||||
"""
|
||||
for key, value in tensor.__dict__.items():
|
||||
if isinstance(value, MethodType) and value.__self__ is layer_ref_sentinel:
|
||||
|
||||
@@ -49,8 +49,11 @@ def has_device_tensors(bound_args: BoundArguments) -> bool:
|
||||
"""
|
||||
Return True if the loaded weights exist on an accelerator device
|
||||
|
||||
:param bound_args: args to load weights
|
||||
:return: True if weights are on accelerator device
|
||||
Args:
|
||||
bound_args: args to load weights
|
||||
|
||||
Returns:
|
||||
True if weights are on accelerator device
|
||||
"""
|
||||
return any(
|
||||
isinstance(value, torch.Tensor) and value.device.type not in ("meta", "cpu")
|
||||
@@ -62,8 +65,11 @@ def get_info_size(info: LayerReloadingInfo) -> int:
|
||||
"""
|
||||
Calculate the number of bytes used by loaded weights for a given layer
|
||||
|
||||
:param info: layerwise info to get size of
|
||||
:return: number of bytes used by loaded weights
|
||||
Args:
|
||||
info: layerwise info to get size of
|
||||
|
||||
Returns:
|
||||
number of bytes used by loaded weights
|
||||
"""
|
||||
return sum(
|
||||
value.nbytes
|
||||
|
||||
@@ -277,7 +277,8 @@ class OlmoModel(nn.Module):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
"""
|
||||
:param input_ids: A tensor of shape `(batch_size, seq_len)`.
|
||||
Args:
|
||||
input_ids: A tensor of shape `(batch_size, seq_len)`.
|
||||
"""
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
|
||||
@@ -314,7 +314,8 @@ class Olmo2Model(nn.Module):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
"""
|
||||
:param input_ids: A tensor of shape `(batch_size, seq_len)`.
|
||||
Args:
|
||||
input_ids: A tensor of shape `(batch_size, seq_len)`.
|
||||
"""
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from collections.abc import Callable, Hashable
|
||||
from fractions import Fraction
|
||||
from typing import Any
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
import torch
|
||||
@@ -42,10 +43,9 @@ class BasevLLMParameter(Parameter):
|
||||
"""
|
||||
Initialize the BasevLLMParameter
|
||||
|
||||
:param data: torch tensor with the parameter data
|
||||
:param weight_loader: weight loader callable
|
||||
|
||||
:returns: a torch.nn.parameter
|
||||
Args:
|
||||
data: torch tensor with the parameter data
|
||||
weight_loader: weight loader callable
|
||||
"""
|
||||
|
||||
# During weight loading, we often do something like:
|
||||
@@ -445,15 +445,16 @@ class SharedWeightParameter(BasevLLMParameter):
|
||||
"currently support tensor parallelism"
|
||||
)
|
||||
|
||||
def add_partition(self, index: int, data_key: Hashable, *args, **kwargs):
|
||||
def add_partition(self, index: int, data_key: Hashable, *args: Any, **kwargs: Any):
|
||||
"""
|
||||
Add a partition to the weight parameter. Partitions whose `data_key`
|
||||
is the same will share tensor data
|
||||
|
||||
:param index: index of partition to add
|
||||
:param data_key: hashable key used to key shared tensors
|
||||
:param *args: arguments for `torch.empty`
|
||||
:param **kwargs: keyword arguments for `torch.empty`
|
||||
Args:
|
||||
index: index of partition to add
|
||||
data_key: hashable key used to key shared tensors
|
||||
*args: arguments for `torch.empty`
|
||||
**kwargs: keyword arguments for `torch.empty`
|
||||
"""
|
||||
# load (shared) tensor using `data_key`
|
||||
if data_key not in self.tensors_registry:
|
||||
|
||||
Reference in New Issue
Block a user