Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -150,8 +150,7 @@ class AttentionConfig:
|
||||
return self.indexer_kv_dtype
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
"""Provide a hash that uniquely identifies all the configs
|
||||
that affect the structure of the computation
|
||||
graph from input ids/embeddings to the final hidden states,
|
||||
excluding anything before input ids/embeddings and after
|
||||
|
||||
@@ -248,8 +248,7 @@ class CacheConfig:
|
||||
KV offloading is only activated when kv_offloading_size is set."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+15
-24
@@ -36,7 +36,8 @@ logger = init_logger(__name__)
|
||||
|
||||
class CompilationMode(enum.IntEnum):
|
||||
"""The compilation approach used for torch.compile-based compilation of the
|
||||
model."""
|
||||
model.
|
||||
"""
|
||||
|
||||
NONE = 0
|
||||
"""No torch.compile compilation is applied, model runs in fully eager pytorch mode.
|
||||
@@ -192,12 +193,10 @@ class PassConfig:
|
||||
# TODO(luka) better pass enabling system.
|
||||
|
||||
def flashinfer_max_size(self, world_size: int) -> int | None:
|
||||
"""
|
||||
Returns the max communication size in bytes for flashinfer
|
||||
"""Returns the max communication size in bytes for flashinfer
|
||||
allreduce fusion for the given world size. Returns None if world size
|
||||
is not supported by configs as it's not supported by flashinfer.
|
||||
"""
|
||||
|
||||
MiB = 1024 * 1024
|
||||
FI_SUPPORTED_WORLD_SIZES = [2, 4, 8, 16]
|
||||
if world_size not in FI_SUPPORTED_WORLD_SIZES:
|
||||
@@ -223,12 +222,10 @@ class PassConfig:
|
||||
return FI_ALLREDUCE_FUSION_MAX_SIZE_MB.get(capability.to_int(), {})
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
"""Produces a hash unique to the pass configuration.
|
||||
Any new fields that affect compilation should be added to the hash.
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
"""
|
||||
|
||||
return hash_factors(get_hash_factors(self, set()))
|
||||
|
||||
@field_validator(
|
||||
@@ -317,8 +314,7 @@ class PassConfig:
|
||||
self.fuse_rope_kvcache_cat_mla = False
|
||||
|
||||
def log_enabled_passes(self) -> None:
|
||||
"""
|
||||
Log the enabled custom fusion passes.
|
||||
"""Log the enabled custom fusion passes.
|
||||
This is called at the end of VLLMConfig post_init,
|
||||
after all defaults are finalized.
|
||||
TODO also log the compile ranges for which this is enabled.
|
||||
@@ -341,7 +337,8 @@ class PassConfig:
|
||||
class DynamicShapesType(str, enum.Enum):
|
||||
"""Types of dynamic shapes handling in torch.compile().
|
||||
see Dynamic shapes and vllm guard dropping in torch_compile.md
|
||||
for more details."""
|
||||
for more details.
|
||||
"""
|
||||
|
||||
BACKED = "backed"
|
||||
"""Use backed dynamic shapes. torch.compile() guards on backed dynamic
|
||||
@@ -391,10 +388,7 @@ class DynamicShapesConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash for DynamicShapesConfig
|
||||
"""
|
||||
|
||||
"""Provide a hash for DynamicShapesConfig."""
|
||||
from vllm.config.utils import get_hash_factors, hash_factors
|
||||
|
||||
factors = get_hash_factors(self, set())
|
||||
@@ -790,8 +784,7 @@ class CompilationConfig:
|
||||
]
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
"""Provide a hash that uniquely identifies all the configs
|
||||
that affect the structure of the computation
|
||||
graph from input ids/embeddings to the final hidden states,
|
||||
excluding anything before input ids/embeddings and after
|
||||
@@ -856,8 +849,7 @@ class CompilationConfig:
|
||||
@field_validator("mode", mode="before")
|
||||
@classmethod
|
||||
def validate_mode_before(cls, value: Any) -> Any:
|
||||
"""
|
||||
Enable parsing the `mode` field from string mode names.
|
||||
"""Enable parsing the `mode` field from string mode names.
|
||||
Accepts both integers (0-3) and string names, like NONE, STOCK_TORCH_COMPILE,
|
||||
DYNAMO_TRACE_ONCE, VLLM_COMPILE.
|
||||
"""
|
||||
@@ -1079,15 +1071,17 @@ class CompilationConfig:
|
||||
prefix: str = "",
|
||||
is_encoder: bool = False,
|
||||
) -> str | Callable:
|
||||
"""
|
||||
Initialize the backend for the compilation config from a vllm config.
|
||||
"""Initialize the backend for the compilation config from a vllm config.
|
||||
|
||||
Arguments:
|
||||
vllm_config: The vllm config to initialize the backend from.
|
||||
prefix: Cache directory prefix for this compiled module.
|
||||
is_encoder: Whether this module is used in an encoder (as
|
||||
opposed to a text backbone).
|
||||
|
||||
Returns:
|
||||
The backend for the compilation config.
|
||||
|
||||
"""
|
||||
if self.mode is None:
|
||||
raise ValueError(
|
||||
@@ -1122,7 +1116,6 @@ class CompilationConfig:
|
||||
configs are set. This includes:
|
||||
- initialize compile_sizes
|
||||
"""
|
||||
|
||||
computed_compile_sizes: list[int] = []
|
||||
if self.compile_sizes is not None:
|
||||
# de-duplicate the sizes provided by the config
|
||||
@@ -1321,13 +1314,11 @@ class CompilationConfig:
|
||||
return self.backend == "inductor" and self.mode != CompilationMode.NONE
|
||||
|
||||
def custom_op_log_check(self):
|
||||
"""
|
||||
This method logs the enabled/disabled custom ops and checks that the
|
||||
"""This method logs the enabled/disabled custom ops and checks that the
|
||||
passed custom_ops field only contains relevant ops.
|
||||
It is called at the end of set_current_vllm_config,
|
||||
after the custom ops have been instantiated.
|
||||
"""
|
||||
|
||||
if len(self.enabled_custom_ops) + len(self.disabled_custom_ops) == 0:
|
||||
logger.debug("No custom ops found in model.")
|
||||
return
|
||||
|
||||
@@ -28,8 +28,7 @@ class DeviceConfig:
|
||||
`__post_init__`."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -31,8 +31,7 @@ class EncoderCacheManagerConfig:
|
||||
|
||||
|
||||
class EncoderCacheManagerMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate between the
|
||||
"""Abstract Metadata used to communicate between the
|
||||
Scheduler EncoderCacheManager and Worker EncoderCacheManager.
|
||||
"""
|
||||
|
||||
|
||||
@@ -63,8 +63,7 @@ class ECTransferConfig:
|
||||
Only supported in V1."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+7
-15
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@config
|
||||
class IrOpPriorityConfig:
|
||||
"""
|
||||
Configuration for vLLM IR op priority for dispatching/lowering during the
|
||||
"""Configuration for vLLM IR op priority for dispatching/lowering during the
|
||||
forward pass. Each member is a list of strings, which will be installed
|
||||
in worker init via vllm.ir.ops.<op_name>.set_default().
|
||||
A single comma-separated string is accepted as well,
|
||||
@@ -38,8 +37,7 @@ class IrOpPriorityConfig:
|
||||
"""Priority list for vllm.ir.ops.gelu_and_mul_sparse"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
"""Produces a hash unique to the pass configuration.
|
||||
Any new fields that affect compilation should be added to the hash.
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
|
||||
@@ -71,8 +69,7 @@ class IrOpPriorityConfig:
|
||||
return value
|
||||
|
||||
def _iter_op_priorities(self):
|
||||
"""
|
||||
Yield (IrOp, priority_list) for each field, after importing platform
|
||||
"""Yield (IrOp, priority_list) for each field, after importing platform
|
||||
kernels and validating each entry.
|
||||
"""
|
||||
from vllm.ir.op import IrOp
|
||||
@@ -89,16 +86,13 @@ class IrOpPriorityConfig:
|
||||
yield IrOp.registry[field.name], op_priority
|
||||
|
||||
def set_default(self) -> None:
|
||||
"""
|
||||
Permanently set the IR op priority for all op members.
|
||||
"""
|
||||
"""Permanently set the IR op priority for all op members."""
|
||||
for ir_op, op_priority in self._iter_op_priorities():
|
||||
ir_op.set_default(op_priority)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_priority(self):
|
||||
"""
|
||||
Context manager to set the IR op priority for all op members.
|
||||
"""Context manager to set the IR op priority for all op members.
|
||||
It also imports IR kernel implementations for the current platform
|
||||
to ensure all implementations are made available.
|
||||
"""
|
||||
@@ -111,8 +105,7 @@ class IrOpPriorityConfig:
|
||||
def with_default(
|
||||
cls, default: list[str], /, **kwargs: list[str]
|
||||
) -> "IrOpPriorityConfig":
|
||||
"""
|
||||
A helper to create an IrOpPriorityConfig where fields not specified in kwargs
|
||||
"""A helper to create an IrOpPriorityConfig where fields not specified in kwargs
|
||||
use the given default list.
|
||||
"""
|
||||
for field in fields(cls): # type: ignore[arg-type]
|
||||
@@ -355,8 +348,7 @@ class KernelConfig:
|
||||
return value
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Produces a hash unique to the pass configuration.
|
||||
"""Produces a hash unique to the pass configuration.
|
||||
Any new fields that affect compilation should be added to the hash.
|
||||
Any future fields that don't affect compilation should be excluded.
|
||||
"""
|
||||
|
||||
@@ -106,8 +106,7 @@ class KVTransferConfig:
|
||||
'fail': immediately fail the request with an error finish reason (default)"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+1
-2
@@ -118,8 +118,7 @@ class LoadConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+1
-2
@@ -86,8 +86,7 @@ class LoRAConfig:
|
||||
MoE models whose adapters use this layout; ignored otherwise."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+15
-21
@@ -418,8 +418,7 @@ class ModelConfig:
|
||||
mm_processor_device: InitVar[MMProcessorDevice | None] = None
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -938,7 +937,7 @@ class ModelConfig:
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_model_config_after(self: "ModelConfig") -> "ModelConfig":
|
||||
"""Called after __post_init__"""
|
||||
"""Called after __post_init__."""
|
||||
if not isinstance(self.tokenizer, str):
|
||||
raise ValueError(
|
||||
f"tokenizer must be a string, got "
|
||||
@@ -1006,7 +1005,8 @@ class ModelConfig:
|
||||
|
||||
def _get_transformers_backend_cls(self) -> str:
|
||||
"""Determine which Transformers modeling backend class will be used if
|
||||
`model_impl` is set to `transformers` or `auto`."""
|
||||
`model_impl` is set to `transformers` or `auto`.
|
||||
"""
|
||||
cls = "Transformers"
|
||||
# If 'hf_config is not hf_text_config' it's a nested config, i.e. multimodal
|
||||
cls += "MultiModal" if self.hf_config is not self.hf_text_config else ""
|
||||
@@ -1083,8 +1083,8 @@ class ModelConfig:
|
||||
Args:
|
||||
model: Model name or path
|
||||
tokenizer: Tokenizer name or path
|
||||
"""
|
||||
|
||||
"""
|
||||
# Skip if model_weights is already set (model already pulled)
|
||||
if self.model_weights:
|
||||
return
|
||||
@@ -1662,9 +1662,7 @@ class ModelConfig:
|
||||
raise AssertionError(f"Unsupported block type: {block_type}")
|
||||
|
||||
def get_mamba_chunk_size(self) -> int:
|
||||
"""
|
||||
Returns the mamba chunk size if it exists
|
||||
"""
|
||||
"""Returns the mamba chunk size if it exists."""
|
||||
# used by e.g. Bamba, FalconH1, Granite
|
||||
chunk_size = getattr(self.hf_text_config, "mamba_chunk_size", None)
|
||||
if chunk_size is None:
|
||||
@@ -1679,11 +1677,11 @@ class ModelConfig:
|
||||
return chunk_size
|
||||
|
||||
def get_multimodal_config(self) -> MultiModalConfig:
|
||||
"""
|
||||
Get the multimodal configuration of the model.
|
||||
"""Get the multimodal configuration of the model.
|
||||
|
||||
Raises:
|
||||
ValueError: If the model is not multimodal.
|
||||
|
||||
"""
|
||||
if self.multimodal_config is None:
|
||||
raise ValueError("The model is not multimodal.")
|
||||
@@ -1691,8 +1689,7 @@ class ModelConfig:
|
||||
return self.multimodal_config
|
||||
|
||||
def try_get_generation_config(self) -> dict[str, Any]:
|
||||
"""
|
||||
This method attempts to retrieve the non-default values of the
|
||||
"""This method attempts to retrieve the non-default values of the
|
||||
generation config for this model.
|
||||
|
||||
The generation config can contain information about special tokens, as
|
||||
@@ -1701,6 +1698,7 @@ class ModelConfig:
|
||||
|
||||
Returns:
|
||||
A dictionary containing the non-default generation config.
|
||||
|
||||
"""
|
||||
if self.generation_config in {"auto", "vllm"}:
|
||||
config = try_get_generation_config(
|
||||
@@ -1726,8 +1724,7 @@ class ModelConfig:
|
||||
return config.to_diff_dict()
|
||||
|
||||
def get_diff_sampling_param(self) -> dict[str, Any]:
|
||||
"""
|
||||
This method returns a dictionary containing the non-default sampling
|
||||
"""This method returns a dictionary containing the non-default sampling
|
||||
parameters with `override_generation_config` applied.
|
||||
|
||||
The default sampling parameters are:
|
||||
@@ -1739,6 +1736,7 @@ class ModelConfig:
|
||||
|
||||
Returns:
|
||||
A dictionary containing the non-default sampling parameters.
|
||||
|
||||
"""
|
||||
src = self.generation_config
|
||||
|
||||
@@ -1870,8 +1868,7 @@ class ModelConfig:
|
||||
|
||||
@property
|
||||
def score_type(self) -> ScoreType:
|
||||
"""
|
||||
Scoring API handles score/rerank for:
|
||||
"""Scoring API handles score/rerank for:
|
||||
|
||||
- "classify" task (score_type: cross-encoder models)
|
||||
- "embed" task (score_type: bi-encoder models)
|
||||
@@ -1964,8 +1961,7 @@ class ModelConfig:
|
||||
|
||||
@property
|
||||
def head_dtype(self) -> torch.dtype:
|
||||
"""
|
||||
"head" refers to the last Linear layer(s) of an LLM,
|
||||
"""The "head" refers to the last Linear layer(s) of an LLM,
|
||||
such as the lm_head in a generation model,
|
||||
or the score or classifier in a classification model.
|
||||
|
||||
@@ -1976,7 +1972,6 @@ class ModelConfig:
|
||||
fp32, which is required for RL training-inference consistency
|
||||
(the trainer computes logits in fp32).
|
||||
"""
|
||||
|
||||
head_dtype = _get_head_dtype(
|
||||
config=self.hf_config, dtype=self.dtype, runner_type=self.runner_type
|
||||
)
|
||||
@@ -2174,8 +2169,7 @@ class ModelConfig:
|
||||
|
||||
|
||||
def get_served_model_name(model: str, served_model_name: str | list[str] | None):
|
||||
"""
|
||||
If the input is a non-empty list, the first model_name in
|
||||
"""If the input is a non-empty list, the first model_name in
|
||||
`served_model_name` is taken.
|
||||
If the input is a non-empty string, it is used directly.
|
||||
For cases where the input is either an empty string or an
|
||||
|
||||
@@ -14,9 +14,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
|
||||
class ModelArchitectureConfig:
|
||||
"""
|
||||
Configuration for model architecture that required by vLLM runtime
|
||||
"""
|
||||
"""Configuration for model architecture that required by vLLM runtime."""
|
||||
|
||||
architectures: list[str]
|
||||
"""List of model architecture class names (e.g., ['LlamaForCausalLM']).
|
||||
|
||||
@@ -357,6 +357,7 @@ class MultiModalConfig:
|
||||
Returns:
|
||||
The kwargs to build the config with, unchanged unless the flag adds
|
||||
a `device`.
|
||||
|
||||
"""
|
||||
if mm_processor_device in (None, "auto"):
|
||||
return mm_processor_kwargs
|
||||
@@ -390,6 +391,7 @@ class MultiModalConfig:
|
||||
ValueError: If `device` is not something `torch.device` accepts.
|
||||
`validate_mm_processor_device` is what surfaces this during
|
||||
startup, so the value is only parsed once.
|
||||
|
||||
"""
|
||||
device = (self.mm_processor_kwargs or {}).get("device")
|
||||
if device is None:
|
||||
@@ -418,6 +420,7 @@ class MultiModalConfig:
|
||||
ValueError: If the requested device is not a torch device, or if it
|
||||
is the accelerator on an instance that also runs the language
|
||||
model.
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -452,8 +455,7 @@ class MultiModalConfig:
|
||||
)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -476,8 +478,7 @@ class MultiModalConfig:
|
||||
return hash_str
|
||||
|
||||
def get_limit_per_prompt(self, modality: str) -> int:
|
||||
"""
|
||||
Get the maximum number of input items allowed per prompt
|
||||
"""Get the maximum number of input items allowed per prompt
|
||||
for the given modality (backward compatible).
|
||||
"""
|
||||
if self.language_model_only:
|
||||
@@ -495,8 +496,7 @@ class MultiModalConfig:
|
||||
self,
|
||||
inference_kwargs: Mapping[str, object],
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Get the keyword arguments to pass to the multi-modal processor
|
||||
"""Get the keyword arguments to pass to the multi-modal processor
|
||||
according to the extra arguments passed during inference.
|
||||
"""
|
||||
kwargs = self.mm_processor_kwargs or {}
|
||||
@@ -524,7 +524,8 @@ class MultiModalConfig:
|
||||
|
||||
def get_video_pruning_spec(self) -> tuple[VideoPruningMethod, float] | None:
|
||||
"""Return `(method, rate)` when video pruning is enabled, else None.
|
||||
`rate` is the fraction of video tokens to prune."""
|
||||
`rate` is the fraction of video tokens to prune.
|
||||
"""
|
||||
if self.video_pruning_rate is not None and self.video_pruning_rate > 0:
|
||||
return (self.video_pruning_method, float(self.video_pruning_rate))
|
||||
return None
|
||||
|
||||
@@ -112,8 +112,7 @@ class ObservabilityConfig:
|
||||
)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -157,7 +156,8 @@ class ObservabilityConfig:
|
||||
cls, value: list[DetailedTraceModules] | None
|
||||
) -> list[DetailedTraceModules] | None:
|
||||
"""Handle the legacy case where users might provide a comma-separated
|
||||
string instead of a list of strings."""
|
||||
string instead of a list of strings.
|
||||
"""
|
||||
if value is not None and len(value) == 1 and "," in value[0]:
|
||||
value = cast(list[DetailedTraceModules], value[0].split(","))
|
||||
return value
|
||||
|
||||
@@ -137,8 +137,7 @@ class OffloadConfig:
|
||||
return self
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the offload configs.
|
||||
"""Provide a hash that uniquely identifies all the offload configs.
|
||||
|
||||
All fields are included because PrefetchOffloader patches module
|
||||
forwards and inserts custom ops (wait_prefetch, start_prefetch)
|
||||
|
||||
@@ -602,15 +602,13 @@ class ParallelConfig:
|
||||
|
||||
@property
|
||||
def local_engines_only(self) -> bool:
|
||||
"""
|
||||
Client manages local+remote EngineCores in pure internal LB case.
|
||||
"""Client manages local+remote EngineCores in pure internal LB case.
|
||||
Client manages local EngineCores in hybrid and external LB case.
|
||||
"""
|
||||
return self.data_parallel_external_lb or self.data_parallel_hybrid_lb
|
||||
|
||||
def get_next_dp_init_port(self) -> int:
|
||||
"""
|
||||
We might need to initialize process groups in multiple
|
||||
"""We might need to initialize process groups in multiple
|
||||
processes that is related to data parallelism,
|
||||
e.g. both in the worker and in the engine, which
|
||||
can live in different processes. To avoid port conflicts, we
|
||||
@@ -794,6 +792,7 @@ class ParallelConfig:
|
||||
|
||||
Returns:
|
||||
(has_unfinished_global, pause_consensus)
|
||||
|
||||
"""
|
||||
tensor = torch.tensor(
|
||||
[int(has_unfinished), int(pending_pause)], dtype=torch.int32, device="cpu"
|
||||
@@ -815,8 +814,7 @@ class ParallelConfig:
|
||||
return tensor.item()
|
||||
|
||||
def compute_hash(self):
|
||||
"""
|
||||
Provide a hash that uniquely identifies all the configs
|
||||
"""Provide a hash that uniquely identifies all the configs
|
||||
that affect the structure of the computation
|
||||
graph from input ids/embeddings to the final hidden states,
|
||||
excluding anything before input ids/embeddings and after
|
||||
|
||||
@@ -176,8 +176,7 @@ class PoolerConfig:
|
||||
return self.tok_pooling_type
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -151,8 +151,7 @@ class ProfilerConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -47,13 +47,15 @@ class ReasoningConfig:
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""Returns True if reasoning is enabled (i.e. if token IDs have been
|
||||
initialized), False otherwise."""
|
||||
initialized), False otherwise.
|
||||
"""
|
||||
return self._enabled
|
||||
|
||||
@property
|
||||
def reasoning_start_token_ids(self) -> list[int] | None:
|
||||
"""Token IDs derived from `reasoning_start_str`. Set automatically by
|
||||
`initialize_token_ids`. Not intended to be configured directly."""
|
||||
`initialize_token_ids`. Not intended to be configured directly.
|
||||
"""
|
||||
return self._reasoning_start_token_ids
|
||||
|
||||
@property
|
||||
|
||||
@@ -200,9 +200,7 @@ class SchedulerConfig:
|
||||
|
||||
@staticmethod
|
||||
def default_factory(**kwargs):
|
||||
"""
|
||||
Factory method to create `SchedulerConfig` with default values for `InitVar`s.
|
||||
"""
|
||||
"""Create a `SchedulerConfig` with default values for its `InitVar`s."""
|
||||
if "max_model_len" not in kwargs:
|
||||
kwargs["max_model_len"] = 8192
|
||||
if "is_encoder_decoder" not in kwargs:
|
||||
@@ -233,8 +231,7 @@ class SchedulerConfig:
|
||||
return resolve_obj_by_qualname(self.scheduler_cls)
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
@@ -543,7 +543,8 @@ class SpeculativeConfig:
|
||||
@staticmethod
|
||||
def _acceptance_length_to_rates(length: float, n: int) -> list[float]:
|
||||
"""Mean acceptance length to unconditional per-position rates, using
|
||||
the minimum-variance schedule."""
|
||||
the minimum-variance schedule.
|
||||
"""
|
||||
num_drafts = length - 1 # expected number of accepted draft tokens
|
||||
num_full = int(num_drafts)
|
||||
return (
|
||||
@@ -557,7 +558,8 @@ class SpeculativeConfig:
|
||||
length: float | None,
|
||||
) -> list[float]:
|
||||
"""Return per-position unconditional acceptance rates from exactly one
|
||||
of `rates` or `length` (validates range, length, and monotonicity)."""
|
||||
of `rates` or `length` (validates range, length, and monotonicity).
|
||||
"""
|
||||
if (rates is None) == (length is None):
|
||||
raise ValueError(
|
||||
"rejection_sample_method='synthetic' requires exactly one of "
|
||||
@@ -598,8 +600,7 @@ class SpeculativeConfig:
|
||||
top-k base-logit candidates. Requires draft tensor parallel size 1."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -1627,7 +1628,6 @@ class SpeculativeConfig:
|
||||
speculative_max_model_len is mainly used for testing that sequences can
|
||||
skip speculation.
|
||||
"""
|
||||
|
||||
if speculative_max_model_len is not None:
|
||||
if speculative_max_model_len > draft_max_model_len:
|
||||
raise ValueError(
|
||||
@@ -1671,6 +1671,7 @@ class SpeculativeConfig:
|
||||
Args:
|
||||
draft_hf_config: The draft model's HF config, mutated in place.
|
||||
target_max_model_len: The target model's max_model_len.
|
||||
|
||||
"""
|
||||
draft_max_position_embeddings = getattr(
|
||||
draft_hf_config, "max_position_embeddings", None
|
||||
@@ -1695,8 +1696,7 @@ class SpeculativeConfig:
|
||||
speculative_draft_tensor_parallel_size: int | None,
|
||||
draft_hf_config: PretrainedConfig,
|
||||
) -> int:
|
||||
"""
|
||||
Verifies and adjusts the tensor parallel size for a draft model
|
||||
"""Verifies and adjusts the tensor parallel size for a draft model
|
||||
specified using speculative_draft_tensor_parallel_size.
|
||||
"""
|
||||
# If speculative_draft_tensor_parallel_size is unset then set it
|
||||
@@ -1725,8 +1725,7 @@ class SpeculativeConfig:
|
||||
return speculative_draft_tensor_parallel_size
|
||||
|
||||
def update_arch_(self):
|
||||
"""
|
||||
EagleConfig and ExtractHiddenStatesConfig update architectures, so update all
|
||||
"""EagleConfig and ExtractHiddenStatesConfig update architectures, so update all
|
||||
architectures-related fields in self.draft_model_config
|
||||
"""
|
||||
self.draft_model_config.hf_text_config = get_hf_text_config(
|
||||
|
||||
@@ -42,8 +42,7 @@ class StructuredOutputsConfig:
|
||||
"""Whether to use structured input for reasoning."""
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
|
||||
+14
-14
@@ -64,7 +64,9 @@ def config(
|
||||
cls: The class to decorate
|
||||
config: The pydantic ConfigDict to use. If provided, it will be merged with
|
||||
the default config.
|
||||
**kwargs: Additional arguments to pass to pydantic.dataclass."""
|
||||
**kwargs: Additional arguments to pass to pydantic.dataclass.
|
||||
|
||||
"""
|
||||
# Extra fields are forbidden by default
|
||||
merged_config = ConfigDict(extra="forbid")
|
||||
if config is not None:
|
||||
@@ -82,7 +84,8 @@ def config(
|
||||
|
||||
def get_field(cls: ConfigType, name: str) -> Any:
|
||||
"""Get the default factory field of a dataclass by name. Used for getting
|
||||
default factory fields in `EngineArgs`."""
|
||||
default factory fields in `EngineArgs`.
|
||||
"""
|
||||
if not is_dataclass(cls):
|
||||
raise TypeError("The given class is not a dataclass.")
|
||||
try:
|
||||
@@ -119,7 +122,8 @@ def is_init_field(cls: ConfigType, name: str) -> bool:
|
||||
def replace(dataclass_instance: ConfigT, /, **kwargs) -> ConfigT:
|
||||
"""Like [`dataclasses.replace`](https://docs.python.org/3/library/dataclasses.html#dataclasses.replace),
|
||||
but compatible with Pydantic dataclasses which use `pydantic.fields.Field` instead
|
||||
of `dataclasses.field`"""
|
||||
of `dataclasses.field`
|
||||
"""
|
||||
cls = type(dataclass_instance)
|
||||
dataclass_dict = dataclass_instance.__dict__
|
||||
dataclass_dict = {k: v for k, v in dataclass_dict.items() if is_init_field(cls, k)}
|
||||
@@ -134,8 +138,7 @@ def getattr_iter(
|
||||
default_factory: Callable[[], Any] | None = None,
|
||||
warn: bool = False,
|
||||
) -> Any:
|
||||
"""
|
||||
A helper function that retrieves an attribute from an object which may
|
||||
"""A helper function that retrieves an attribute from an object which may
|
||||
have multiple possible names. This is useful when fetching attributes from
|
||||
arbitrary `transformers.PretrainedConfig` instances.
|
||||
|
||||
@@ -158,12 +161,10 @@ def getattr_iter(
|
||||
|
||||
|
||||
def get_attr_docs(cls: type[Any]) -> dict[str, str]:
|
||||
"""
|
||||
Get any docstrings placed after attribute assignments in a class body.
|
||||
"""Get any docstrings placed after attribute assignments in a class body.
|
||||
|
||||
https://davidism.com/mit-license/
|
||||
"""
|
||||
|
||||
cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0]
|
||||
|
||||
if not isinstance(cls_node, ast.ClassDef):
|
||||
@@ -385,8 +386,7 @@ def hash_factors(items: dict[str, object]) -> str:
|
||||
|
||||
@dataclass
|
||||
class Range:
|
||||
"""
|
||||
A range of numbers.
|
||||
"""A range of numbers.
|
||||
Inclusive of start, inclusive of end.
|
||||
"""
|
||||
|
||||
@@ -445,8 +445,7 @@ def get_from_deprecated_env_if_set(
|
||||
removal_version: str,
|
||||
field_name: str | None = None,
|
||||
) -> str | None:
|
||||
"""
|
||||
Get value from deprecated environment variable with warning.
|
||||
"""Get value from deprecated environment variable with warning.
|
||||
|
||||
Args:
|
||||
env_name: Name of the deprecated environment variable
|
||||
@@ -455,6 +454,7 @@ def get_from_deprecated_env_if_set(
|
||||
|
||||
Returns:
|
||||
The environment variable value if set, None otherwise
|
||||
|
||||
"""
|
||||
if envs.is_set(env_name):
|
||||
value = os.environ.get(env_name)
|
||||
@@ -477,8 +477,7 @@ def set_from_deprecated_env_if_set(
|
||||
to_bool: bool = False,
|
||||
to_int: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Set object field from deprecated environment variable with warning.
|
||||
"""Set object field from deprecated environment variable with warning.
|
||||
|
||||
Args:
|
||||
config: Config object to set the field on
|
||||
@@ -489,6 +488,7 @@ def set_from_deprecated_env_if_set(
|
||||
to_int: Whether to convert the environment variable value to integer
|
||||
Returns:
|
||||
None
|
||||
|
||||
"""
|
||||
if to_bool and to_int:
|
||||
raise ValueError("Cannot convert to both boolean and integer.")
|
||||
|
||||
+19
-28
@@ -153,8 +153,8 @@ IS_DENSE = False
|
||||
|
||||
def enable_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if either RMS norm or quant FP8 custom op is active;
|
||||
otherwise Inductor handles fusion."""
|
||||
|
||||
otherwise Inductor handles fusion.
|
||||
"""
|
||||
return (
|
||||
cfg.compilation_config.is_custom_op_enabled("rms_norm")
|
||||
or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
|
||||
@@ -163,8 +163,7 @@ def enable_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
|
||||
def enable_act_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""
|
||||
Enable if either SiLU+Mul or quant FP8 custom op is active;
|
||||
"""Enable if either SiLU+Mul or quant FP8 custom op is active;
|
||||
otherwise Inductor handles fusion.
|
||||
Also enable for FP4 models as FP4 quant is always custom so Inductor cannot fuse it.
|
||||
"""
|
||||
@@ -220,7 +219,6 @@ def enable_rope_kvcache_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
def enable_rope_kvcache_mla_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if use_inductor_graph_partition is enabled."""
|
||||
|
||||
return (
|
||||
cfg.compilation_config.use_inductor_graph_partition
|
||||
or not cfg.compilation_config.splitting_ops_contain_kv_cache_update()
|
||||
@@ -229,7 +227,6 @@ def enable_rope_kvcache_mla_fusion(cfg: "VllmConfig") -> bool:
|
||||
|
||||
def enable_norm_pad_fusion(cfg: "VllmConfig") -> bool:
|
||||
"""Enable if using AITER RMSNorm and hidden size is 2880 i.e. gpt-oss."""
|
||||
|
||||
return (
|
||||
cfg.kernel_config.ir_op_priority.fused_add_rms_norm[0] == "aiter"
|
||||
and cfg.model_config is not None
|
||||
@@ -460,8 +457,7 @@ class VllmConfig:
|
||||
"""
|
||||
|
||||
def compute_hash(self, include_version: bool = True) -> str:
|
||||
"""
|
||||
WARNING: Whenever a new field is added to this config,
|
||||
"""WARNING: Whenever a new field is added to this config,
|
||||
ensure that it is included in the factors list if
|
||||
it affects the computation graph.
|
||||
|
||||
@@ -473,6 +469,7 @@ class VllmConfig:
|
||||
|
||||
Args:
|
||||
include_version: Include the vLLM version in the hash.
|
||||
|
||||
"""
|
||||
factors: list[Any] = []
|
||||
|
||||
@@ -747,7 +744,8 @@ class VllmConfig:
|
||||
|
||||
def _is_dflash2_draft(self) -> bool:
|
||||
"""Whether the DFlash draft is a DFlash2 one, by the architecture the
|
||||
speculator selects on (v1/worker/gpu/spec_decode/__init__.py)."""
|
||||
speculator selects on (v1/worker/gpu/spec_decode/__init__.py).
|
||||
"""
|
||||
spec = self.speculative_config
|
||||
if spec is None or spec.method != "dflash":
|
||||
return False
|
||||
@@ -798,8 +796,7 @@ class VllmConfig:
|
||||
|
||||
@property
|
||||
def needs_dp_coordinator(self) -> bool:
|
||||
"""
|
||||
Determine if the DPCoordinator process is needed.
|
||||
"""Determine if the DPCoordinator process is needed.
|
||||
|
||||
The DPCoordinator is needed in two cases:
|
||||
1. For MoE models with DP > 1: to handle wave coordination
|
||||
@@ -809,8 +806,8 @@ class VllmConfig:
|
||||
|
||||
Returns:
|
||||
True if DPCoordinator process is needed, False otherwise.
|
||||
"""
|
||||
|
||||
"""
|
||||
# For non-MoE models, only need coordinator in internal/hybrid LB mode
|
||||
# (for stats collection).
|
||||
return self.parallel_config.data_parallel_size > 1 and (
|
||||
@@ -820,8 +817,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
def enable_trace_function_call_for_thread(self) -> None:
|
||||
"""
|
||||
Set up function tracing for the current thread,
|
||||
"""Set up function tracing for the current thread,
|
||||
if enabled via the `VLLM_TRACE_FUNCTION` environment variable.
|
||||
"""
|
||||
if envs.VLLM_TRACE_FUNCTION:
|
||||
@@ -955,6 +951,7 @@ class VllmConfig:
|
||||
config_obj: Configuration object to update.
|
||||
key: Attribute name.
|
||||
value: Default value (static or callable).
|
||||
|
||||
"""
|
||||
if getattr(config_obj, key) is None:
|
||||
# Some config values are known before initialization and are
|
||||
@@ -976,6 +973,7 @@ class VllmConfig:
|
||||
|
||||
Args:
|
||||
defaults: Dictionary of default values to apply.
|
||||
|
||||
"""
|
||||
|
||||
def apply_recursive(config_obj: Any, config_defaults: dict[str, Any]) -> None:
|
||||
@@ -1276,7 +1274,6 @@ class VllmConfig:
|
||||
|
||||
def __post_init__(self):
|
||||
"""Verify configs are valid & consistent with each other."""
|
||||
|
||||
# To give each torch profile run a unique instance name.
|
||||
self.instance_id = f"{time.time_ns()}"
|
||||
|
||||
@@ -2133,8 +2130,7 @@ class VllmConfig:
|
||||
]
|
||||
|
||||
def _set_max_num_scheduled_tokens(self):
|
||||
"""
|
||||
In most cases, the scheduler may schedule a batch with as many tokens as the
|
||||
"""In most cases, the scheduler may schedule a batch with as many tokens as the
|
||||
worker is configured to handle.
|
||||
"""
|
||||
if self.speculative_config is not None:
|
||||
@@ -2172,8 +2168,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
def _set_cudagraph_sizes(self):
|
||||
"""
|
||||
vLLM defines the default candidate list of batch sizes for CUDA graph
|
||||
"""VLLM defines the default candidate list of batch sizes for CUDA graph
|
||||
capture as:
|
||||
|
||||
```python
|
||||
@@ -2219,8 +2214,8 @@ class VllmConfig:
|
||||
padded CUDA graph will be used.
|
||||
- If batch size > largest `cudagraph_capture_sizes`, cudagraph will
|
||||
not be used.
|
||||
"""
|
||||
|
||||
"""
|
||||
if (
|
||||
self.model_config is not None
|
||||
and not self.model_config.enforce_eager
|
||||
@@ -2441,9 +2436,7 @@ class VllmConfig:
|
||||
self.compilation_config.post_init_cudagraph_sizes()
|
||||
|
||||
def _set_compile_ranges(self):
|
||||
"""
|
||||
Set the compile ranges for the compilation config.
|
||||
"""
|
||||
"""Set the compile ranges for the compilation config."""
|
||||
compilation_config = self.compilation_config
|
||||
computed_compile_ranges_endpoints = []
|
||||
|
||||
@@ -3310,8 +3303,7 @@ _current_prefix: str | None = None
|
||||
def set_current_vllm_config(
|
||||
vllm_config: VllmConfig, check_compile=False, prefix: str | None = None
|
||||
):
|
||||
"""
|
||||
Temporarily set the current vLLM config.
|
||||
"""Temporarily set the current vLLM config.
|
||||
Used during model initialization.
|
||||
We save the current vLLM config in a global variable,
|
||||
so that all modules can access it, e.g. custom ops
|
||||
@@ -3392,15 +3384,14 @@ def get_layers_from_vllm_config(
|
||||
layer_type: type[T],
|
||||
layer_names: Iterable[str] | None = None,
|
||||
) -> dict[str, T]:
|
||||
"""
|
||||
Get layers from the vLLM config.
|
||||
"""Get layers from the vLLM config.
|
||||
|
||||
Args:
|
||||
vllm_config: The vLLM config.
|
||||
layer_type: The type of the layer to get.
|
||||
layer_names: The names of the layers to get. If None, return all layers.
|
||||
"""
|
||||
|
||||
"""
|
||||
forward_context = vllm_config.compilation_config.static_forward_context
|
||||
if layer_names is None:
|
||||
layer_names = forward_context.keys()
|
||||
|
||||
Reference in New Issue
Block a user